From 37f3a829e711f28c25c37d2b27cff10173beebfd Mon Sep 17 00:00:00 2001 From: James <83447078+xiaofan-luan@users.noreply.github.com> Date: Thu, 17 Sep 2026 09:43:08 +0800 Subject: [PATCH] fix: make local storage keys complete paths at every layer (#53145) issue: #53051, #53052, #53152 ## Summary Align local storage around one path contract: a key is a complete absolute filesystem path, and the local loon/Arrow filesystem is rooted at `/`. Previously, some writers used `minio.rootPath` while others supplied complete local paths to an already-prefixed filesystem, placing files in different physical locations from those expected by readers and GC. Remote keys remain complete bucket-relative object keys; their filesystem-root convention is unchanged. ## Changes - Unify Go/C++ local filesystem roots and StorageV3 manifest-base construction across writers. Preserve the existing base/version pair when continuing a growing-source manifest. - Fix local JSON/TEXT stats reads, rejected-stats cleanup, and explore temporary paths. Exercise resolved V3 stats paths through the actual `LocalChunkManager.MultiRead` interface. - Resolve legacy local manifest bases in loaded metadata and locally referenced snapshots, without eager catalog or manifest-file writes. Preserve manifest CAS compatibility and pathless V3 summaries; copy/restore writes use the target storage root. - Require an absolute, nonempty `localStorage.path` at initialization, including on remote deployments where it is a cache directory. Freeze forbidden parameters against configuration-source refresh while retaining explicit runtime/test overrides, and derive mmap, pprof, and disk-capacity defaults from the same canonical startup value. - Make segment deletion prefixes segment-boundary aware, retain root/identity and snapshot checks, and add BM25 orphan scanning. These shared GC changes also apply to remote storage; this PR does not claim every remote code path is untouched. - Remove the local test-only root override and unused local-only property initialization chain; use the production storage-config initialization path. ## Automatic local upgrade Let `R = localStorage.path` and `M = minio.rootPath`. Before standalone components start, local deployments check the known legacy locations: | Existing data | Upgrade handling | | --- | --- | | Canonical files under `R/` | Leave in place. | | Double-root files under `R//` | Move into `R/` for `insert_log`, `text_log`, `json_stats`, `index_files`, and `index_v1`. | | Recognized late-2.6 unified packed indexes under the process CWD | Move into the corresponding directory under `R`, if visible to the new process. | | Legacy manifest base `M/insert_log//

/`, with files under `R/M/insert_log//

/` | **Do not move the files.** Resolve the loaded manifest base to that existing absolute location. New segment layouts use the canonical root; updates to an existing legacy manifest can retain its base. | Migration prefers directory/subtree rename and recursively merges overlapping directories. Cross-filesystem moves fall back to copy-then-remove. Existing same-name files cause a conflict rather than content comparison or overwrite, except a published copy recognized as this migration's own interrupted work. The copy path synchronizes the file and target directory chain before deleting the source. Remaining sources are rediscovered on retry; empty legacy directories are pruned. Once migrated, restarts probe the known old locations without rescanning canonical data. The initial pass walks legacy file metadata but does not hash file contents. There is no configured migration timeout. SIGTERM/SIGINT cancels the migration and prevents etcd or components from starting; directory walks and cross-filesystem copies observe cancellation between filesystem reads. A canceled partial copy is never published and is recopied from the intact source on the next start. No user migration script, offline command, reverse migration, journal, or backup is provided. Successfully moved sources are removed. ## Compatibility and limits - Preserve the absolute local root, data mounts, and legacy `minio.rootPath` when upgrading. Canonical 2.6 data stays in place; misplaced late-2.6 indexes and affected 3.0.0/3.0.1 layouts are handled as above. Relative roots from older versions are unsupported. - Automatic migration is standalone/local-only. If old CWD indexes disappeared with an old container, they cannot be discovered: preserve/make those files visible or rebuild the derived indexes. Cross-filesystem copy has no configured timeout and can extend the first startup, but a shutdown signal cancels it and the next start resumes from the remaining source. - Downgrade to the affected old layout is unsupported after migration. Symlink layouts require manual resolution. - Known legacy LOB/orphan GC gaps under `R/M/insert_log`, pending old external-refresh paths, and rare interrupted-copy publication residue remain outside scope. This is not a claim that every historical file is recovered or reclaimable. ## Validation At head `898dba4b41`, `ci-v2/build-ut-cov` passed Build, Code Check, Integration UT, Go UT, and C++ UT (9,145/9,145). Diff coverage was 100% for C++ and 81.2% for Go; total patch coverage was 82.8%. Focused local tests for canonical/frozen `localStorage.path` derivation and migration cancellation/deadline/copy recovery passed with `-tags dynamic,test -gcflags='all=-N -l' -count=1`, and `git diff --check` passed before publication. Full paramtable execution remains locally blocked by tests attempting to create `/var/lib/milvus/data`; roles/native-dependent Go tests remain build-blocked by local C++ header/library mismatches. A real 3.0.0/3.0.1 version-to-version upgrade was not executed locally. `ci-v2/e2e-default` and `ci-v2/go-sdk` are still pending for this head. Signed-off-by: xiaofanluan Co-authored-by: xiaofanluan --- CLAUDE.md | 1 + cmd/roles/local_layout.go | 143 ++++ cmd/roles/local_layout_test.go | 224 +++++ cmd/roles/roles.go | 4 + cmd/roles/roles_test.go | 4 +- configs/milvus.yaml | 2 + docs/agent_guides/README.md | 2 + docs/agent_guides/storage/path_contract.md | 113 +++ internal/compaction/lob_compaction.go | 82 +- internal/compaction/lob_compaction_test.go | 84 ++ .../src/clustering/KmeansClusteringTest.cpp | 11 +- .../src/common/VectorArrayStorageV2Test.cpp | 16 +- .../exec/expression/LikeConjunctExprTest.cpp | 25 +- internal/core/src/index/FMIndexTest.cpp | 29 +- internal/core/src/index/Meta.h | 4 - .../core/src/index/NgramInvertedIndexTest.cpp | 14 +- .../ChunkedSegmentSealedStorageV2Test.cpp | 17 +- .../segcore/flush_growing_segment_test.cpp | 20 +- internal/core/src/storage/FileManager.h | 8 +- .../core/src/storage/StorageV2FSCache.cpp | 12 +- .../storage/StorageV2FSCacheLocalRootTest.cpp | 104 +++ .../src/storage/loon_ffi/property_singleton.h | 10 - internal/core/src/storage/loon_ffi/util.cpp | 29 +- internal/core/src/storage/loon_ffi/util.h | 22 +- internal/core/unittest/init_gtest.cpp | 1 - .../unittest/test_utils/ManifestTestUtil.h | 14 +- internal/datacoord/copy_segment_task_test.go | 2 +- .../external_collection_refresh_manager.go | 18 +- ...xternal_collection_refresh_manager_test.go | 71 ++ internal/datacoord/garbage_collector.go | 59 +- .../datacoord/garbage_collector_bm25_test.go | 163 ++++ internal/datacoord/garbage_collector_lob.go | 39 +- .../datacoord/garbage_collector_lob_test.go | 30 +- internal/datacoord/garbage_collector_test.go | 322 ++++++- internal/datacoord/manifest_index.go | 8 +- .../datacoord/manifest_index_review_test.go | 7 +- internal/datacoord/meta.go | 14 + internal/datacoord/meta_local_manifest.go | 58 ++ .../meta_local_manifest_integration_test.go | 438 ++++++++++ .../datacoord/meta_local_manifest_test.go | 306 +++++++ internal/datacoord/segment_manager.go | 7 +- .../Meta.cpp => datacoord/segment_path.go} | 18 +- internal/datacoord/segment_path_test.go | 44 + internal/datacoord/server.go | 1 - .../datacoord/snapshot_local_manifest_test.go | 197 +++++ internal/datacoord/snapshot_meta.go | 17 + .../task_refresh_external_collection_test.go | 22 +- internal/datacoord/task_stats.go | 177 +++- internal/datacoord/task_stats_test.go | 598 ++++++++++++- .../bump_schema_version_compactor.go | 78 +- .../bump_schema_version_compactor_test.go | 23 +- .../compactor/bump_schema_version_ut_test.go | 113 ++- .../compactor/clustering_compactor.go | 38 +- .../clustering_compactor_storage_v2_test.go | 8 +- .../clustering_compactor_storage_v3_test.go | 37 +- .../datanode/compactor/compactor_common.go | 26 + internal/datanode/compactor/merge_sort.go | 6 +- internal/datanode/compactor/mix_compactor.go | 28 +- .../mix_compactor_storage_v2_test.go | 8 +- .../datanode/compactor/mix_compactor_test.go | 46 +- .../datanode/compactor/sort_compaction.go | 44 +- .../compactor/sort_compaction_test.go | 42 +- internal/datanode/external/manager_test.go | 52 ++ internal/datanode/external/task_update.go | 23 +- .../datanode/importv2/copy_segment_utils.go | 253 +++--- .../importv2/copy_segment_utils_test.go | 783 +++++++++++++++--- .../importv2/task_copy_segment_test.go | 15 +- internal/datanode/importv2/util.go | 5 +- .../flushcommon/syncmgr/growing_source.go | 41 +- .../syncmgr/growing_source_test.go | 124 +++ .../flushcommon/syncmgr/meta_writer_test.go | 6 +- .../syncmgr/pack_writer_v3_test.go | 28 +- internal/flushcommon/syncmgr/task_test.go | 3 +- .../flushcommon/writebuffer/write_buffer.go | 6 +- .../querynodev2/segments/segment_loader.go | 4 +- .../segments/segment_loader_test.go | 34 + .../snapshotio/storage/snapshot_data_test.go | 51 +- internal/storage/binlog_record_writer.go | 32 +- .../storage/binlog_record_writer_v3_test.go | 64 ++ internal/storage/local_chunk_manager.go | 24 +- internal/storage/local_chunk_manager_test.go | 159 ++++ .../localmigrate/displaced_empty_test.go | 112 +++ .../storage/localmigrate/durability_test.go | 158 ++++ .../storage/localmigrate/empty_dirs_test.go | 191 +++++ .../storage/localmigrate/legacy_cwd_test.go | 155 ++++ .../storage/localmigrate/merge_copy_test.go | 339 ++++++++ internal/storage/localmigrate/migrate.go | 556 +++++++++++++ internal/storage/localmigrate/migrate_test.go | 189 +++++ internal/storage/localmigrate/move.go | 412 +++++++++ .../storage/localmigrate/progress_test.go | 140 ++++ internal/storage/record_reader.go | 45 +- internal/storage/remote_chunk_manager.go | 8 + internal/storage/rw.go | 42 + internal/storage/rw_test.go | 16 + internal/storage/types.go | 15 +- internal/storagev2/filesystem_metrics.go | 4 +- internal/storagev2/filesystem_metrics_test.go | 40 +- internal/storagev2/loon_root.go | 42 + internal/storagev2/loon_root_test.go | 38 + internal/storagev2/packed/ffi_common.go | 12 +- .../storagev2/packed/ffi_root_path_test.go | 47 ++ .../storagev2/packed/manifest_commit_test.go | 32 +- .../storagev2/packed/manifest_index_test.go | 20 +- .../packed/packed_reader_ffi_test.go | 7 +- .../packed/packed_writer_ffi_test.go | 13 +- .../storagev2/packed/segment_reader_ffi.go | 183 ++++ internal/storagev2/packed/transaction_test.go | 2 +- .../wal/vchannel/segment/pack_writer.go | 9 +- .../wal/vchannel/segment/pack_writer_test.go | 71 ++ internal/util/segcore/segment.go | 7 +- internal/util/segcore/segment_test.go | 54 ++ pkg/util/metautil/binlog.go | 7 +- pkg/util/metautil/binlog_test.go | 16 + pkg/util/paramtable/component_param.go | 15 +- pkg/util/paramtable/component_param_test.go | 34 + pkg/util/paramtable/param_item.go | 18 +- pkg/util/paramtable/param_item_test.go | 16 + pkg/util/paramtable/service_param.go | 27 +- pkg/util/paramtable/service_param_test.go | 101 +++ scripts/segcore_error_boundary_baseline.txt | 1 + 120 files changed, 8326 insertions(+), 683 deletions(-) create mode 100644 cmd/roles/local_layout.go create mode 100644 cmd/roles/local_layout_test.go create mode 100644 docs/agent_guides/storage/path_contract.md create mode 100644 internal/core/src/storage/StorageV2FSCacheLocalRootTest.cpp create mode 100644 internal/datacoord/garbage_collector_bm25_test.go create mode 100644 internal/datacoord/meta_local_manifest.go create mode 100644 internal/datacoord/meta_local_manifest_integration_test.go create mode 100644 internal/datacoord/meta_local_manifest_test.go rename internal/{core/src/index/Meta.cpp => datacoord/segment_path.go} (61%) create mode 100644 internal/datacoord/segment_path_test.go create mode 100644 internal/datacoord/snapshot_local_manifest_test.go create mode 100644 internal/storage/localmigrate/displaced_empty_test.go create mode 100644 internal/storage/localmigrate/durability_test.go create mode 100644 internal/storage/localmigrate/empty_dirs_test.go create mode 100644 internal/storage/localmigrate/legacy_cwd_test.go create mode 100644 internal/storage/localmigrate/merge_copy_test.go create mode 100644 internal/storage/localmigrate/migrate.go create mode 100644 internal/storage/localmigrate/migrate_test.go create mode 100644 internal/storage/localmigrate/move.go create mode 100644 internal/storage/localmigrate/progress_test.go create mode 100644 internal/storagev2/loon_root.go create mode 100644 internal/storagev2/loon_root_test.go create mode 100644 internal/storagev2/packed/ffi_root_path_test.go create mode 100644 internal/storagev2/packed/segment_reader_ffi.go diff --git a/CLAUDE.md b/CLAUDE.md index 8536ac7a93..ffc4fd90ae 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -34,6 +34,7 @@ NEVER answer based on documentation alone or code alone. NEVER skip Step 2 — t - [**Observability**](docs/agent_guides/observability/README.md): Logging, metrics, tracing, and observability debug workflows. - [**Streaming System**](docs/agent_guides/streaming-system/streaming-system.md): Write path, WAL, DDL/DCL execution, replication && CDC. +- [**Storage Paths**](docs/agent_guides/storage/path_contract.md): Read before changing storage paths, manifests, local migration, or GC. ## Testing diff --git a/cmd/roles/local_layout.go b/cmd/roles/local_layout.go new file mode 100644 index 0000000000..286b661cf7 --- /dev/null +++ b/cmd/roles/local_layout.go @@ -0,0 +1,143 @@ +// Licensed to the LF AI & Data foundation under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package roles + +import ( + "context" + "time" + + "github.com/milvus-io/milvus/internal/storage/localmigrate" + "github.com/milvus-io/milvus/pkg/v3/mlog" + "github.com/milvus-io/milvus/pkg/v3/util/paramtable" +) + +// maxLoggedConflicts bounds the conflict sample in the failure log. The full set +// stays in the returned Report; a startup log must not grow to the size of the +// storage root. +const maxLoggedConflicts = 100 + +// migrateLocalStorageLayoutOrDie moves the double-joined 3.0 local layout and +// recognized 2.6 working-directory index directories before any component opens +// data. It is enabled automatically for standalone with common.storageType=local. +// Directories are merged without overwriting files; cross-filesystem moves copy. +func (mr *MilvusRoles) migrateLocalStorageLayoutOrDie(ctx context.Context, params *paramtable.ComponentParam) bool { + if !mr.Local || params.CommonCfg.StorageType.GetValue() != "local" { + return true + } + migrationCtx, cancel := mr.localStorageMigrationContext(ctx) + defer cancel() + + report, err := mr.migrateLocalStorageLayout(migrationCtx, params) + if err != nil { + if migrationCtx.Err() != nil { + mlog.Info(ctx, "local storage layout migration canceled before component startup", + mlog.String("root", params.LocalStorageCfg.Path.GetValue()), + mlog.Err(err)) + return false + } + mlog.Fatal(ctx, "local storage layout migration failed, refusing to start", + mlog.String("root", params.LocalStorageCfg.Path.GetValue()), + mlog.String("report", report.String()), + mlog.Int("conflictCount", len(report.Conflicts)), + mlog.Strings("conflictSample", firstStrings(report.Conflicts, maxLoggedConflicts)), + mlog.Err(err)) + return false + } + // A shutdown can race with the final migration operation. Do not continue + // into etcd/component startup after the signal handler has requested exit. + if migrationCtx.Err() != nil { + mlog.Info(ctx, "local storage layout migration completed during shutdown; skipping component startup", + mlog.String("root", params.LocalStorageCfg.Path.GetValue())) + return false + } + // Always report the outcome. "Checked the root and found nothing to move" + // and "never ran" must not look identical in a startup log. + mlog.Info(ctx, "local storage layout migration finished", + mlog.String("report", report.String())) + return true +} + +func (mr *MilvusRoles) localStorageMigrationContext(parent context.Context) (context.Context, context.CancelFunc) { + ctx, cancel := context.WithCancel(parent) + if mr.closed == nil { + return ctx, cancel + } + go func() { + select { + case <-mr.closed: + cancel() + case <-ctx.Done(): + } + }() + return ctx, cancel +} + +func (mr *MilvusRoles) migrateLocalStorageLayout(ctx context.Context, params *paramtable.ComponentParam) (*localmigrate.Report, error) { + if !mr.Local || params.CommonCfg.StorageType.GetValue() != "local" { + return nil, nil + } + root := params.LocalStorageCfg.Path.GetValue() + + logger := mlog.With(mlog.String("root", root)) + logger.Info(ctx, "checking local storage layout for legacy directories to migrate") + + planned, started, completed := 0, 0, 0 + return localmigrate.Migrate(ctx, root, localmigrate.Options{ + LegacyPrefix: params.MinioCfg.RootPath.GetValue(), + OnPlan: func(dirs []localmigrate.Dir) { + planned = len(dirs) + files, bytes := 0, int64(0) + for _, dir := range dirs { + files += dir.Files + bytes += dir.Bytes + } + logger.Info(ctx, "local storage layout migration plan decided", + mlog.Int("directories", planned), + mlog.Int("files", files), + mlog.Int64("bytes", bytes)) + // Report the full plan before moving any directory. + for index, dir := range dirs { + logger.Info(ctx, "local storage layout migration will move directory", + mlog.Int("index", index+1), mlog.Int("total", planned), + mlog.String("source", dir.Source), mlog.String("target", dir.Target), + mlog.Int("files", dir.Files), mlog.Int64("bytes", dir.Bytes)) + } + }, + OnDirStart: func(dir localmigrate.Dir) { + started++ + logger.Info(ctx, "moving local storage layout directory", + mlog.Int("index", started), mlog.Int("total", planned), + mlog.String("source", dir.Source), mlog.String("target", dir.Target), + mlog.Int("files", dir.Files), mlog.Int64("bytes", dir.Bytes)) + }, + OnDirDone: func(dir localmigrate.Dir, elapsed time.Duration) { + completed++ + logger.Info(ctx, "moved local storage layout directory", + mlog.Int("index", completed), mlog.Int("total", planned), + mlog.String("source", dir.Source), mlog.String("target", dir.Target), + mlog.Int("files", dir.Files), mlog.Int64("bytes", dir.Bytes), + mlog.Duration("elapsed", elapsed)) + }, + }) +} + +func firstStrings(values []string, limit int) []string { + if len(values) <= limit { + return values + } + return values[:limit] +} diff --git a/cmd/roles/local_layout_test.go b/cmd/roles/local_layout_test.go new file mode 100644 index 0000000000..4e3870edd0 --- /dev/null +++ b/cmd/roles/local_layout_test.go @@ -0,0 +1,224 @@ +// Licensed to the LF AI & Data foundation under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package roles + +import ( + "context" + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/milvus-io/milvus/internal/storage/localmigrate" + "github.com/milvus-io/milvus/pkg/v3/util/merr" + "github.com/milvus-io/milvus/pkg/v3/util/paramtable" +) + +func TestLocalStorageMigrationContextCancelsOnShutdown(t *testing.T) { + mr := NewMilvusRoles() + ctx, cancel := mr.localStorageMigrationContext(context.Background()) + defer cancel() + + close(mr.closed) + select { + case <-ctx.Done(): + require.ErrorIs(t, ctx.Err(), context.Canceled) + case <-time.After(time.Second): + t.Fatal("migration context was not canceled after shutdown") + } +} + +func localLayoutTestParams(t *testing.T, storageType, root string) *paramtable.ComponentParam { + t.Helper() + base := paramtable.NewBaseTable(paramtable.SkipRemote(true), paramtable.SkipEnv(true)) + require.NoError(t, base.Save("common.storageType", storageType)) + require.NoError(t, base.Save("localStorage.path", root)) + params := ¶mtable.ComponentParam{} + params.Init(base) + return params +} + +func writeLocalLayoutTestFile(t *testing.T, root, key, contents string) { + t.Helper() + filename := filepath.Join(root, key) + require.NoError(t, os.MkdirAll(filepath.Dir(filename), 0o755)) + require.NoError(t, os.WriteFile(filename, []byte(contents), 0o600)) +} + +func TestMigrateLocalStorageLayoutAutomaticSources(t *testing.T) { + root, cwd := t.TempDir(), t.TempDir() + params := localLayoutTestParams(t, "local", root) + mr := &MilvusRoles{Local: true} + t.Chdir(cwd) + keys := map[string]string{ + cwd: "text_log/7/0/1/2/3/100/milvus_packed_text_index.v3", + localmigrate.DisplacedRoots(root)[0]: "insert_log/1/2/3/_data/a.parquet", + } + for sourceRoot, key := range keys { + writeLocalLayoutTestFile(t, sourceRoot, key, key) + } + report, err := mr.migrateLocalStorageLayout(t.Context(), params) + require.NoError(t, err) + require.Equal(t, 2, report.Renamed) + for sourceRoot, key := range keys { + require.NoFileExists(t, filepath.Join(sourceRoot, key)) + contents, err := os.ReadFile(filepath.Join(root, key)) + require.NoError(t, err) + require.Equal(t, key, string(contents)) + } + // A new startup finds no old directories. + mr = &MilvusRoles{Local: true} + params = localLayoutTestParams(t, "local", root) + report, err = mr.migrateLocalStorageLayout(t.Context(), params) + require.NoError(t, err) + require.Zero(t, report.Renamed) + require.Zero(t, report.Bytes) + // Simulate normal GC. A restart does not resurrect deleted canonical data. + for _, key := range keys { + require.NoError(t, os.Remove(filepath.Join(root, key))) + } + report, err = mr.migrateLocalStorageLayout(t.Context(), params) + require.NoError(t, err) + require.Zero(t, report.Renamed) + for _, key := range keys { + require.NoFileExists(t, filepath.Join(root, key)) + } +} + +func TestMigrateLocalStorageLayoutDisabled(t *testing.T) { + for _, test := range []struct { + name string + standalone bool + storageType string + }{ + {name: "remote standalone", standalone: true, storageType: "remote"}, + {name: "local distributed", storageType: "local"}, + {name: "remote distributed", storageType: "remote"}, + } { + t.Run(test.name, func(t *testing.T) { + root, cwd := t.TempDir(), t.TempDir() + params := localLayoutTestParams(t, test.storageType, root) + mr := &MilvusRoles{Local: test.standalone} + t.Chdir(cwd) + key := "text_log/7/0/1/2/3/100/milvus_packed_text_index.v3" + writeLocalLayoutTestFile(t, cwd, key, "old-index") + writeLocalLayoutTestFile(t, localmigrate.DisplacedRoots(root)[0], key, "displaced-index") + report, err := mr.migrateLocalStorageLayout(t.Context(), params) + require.NoError(t, err) + require.Nil(t, report) + require.NoFileExists(t, filepath.Join(root, key)) + require.NoFileExists(t, filepath.Join(root, ".milvus-local-layout.lock")) + require.NoDirExists(t, filepath.Join(root, ".milvus-local-layout")) + require.FileExists(t, filepath.Join(cwd, key)) + require.FileExists(t, filepath.Join(localmigrate.DisplacedRoots(root)[0], key)) + }) + } +} + +func TestMigrateLocalStorageLayoutConflictStopsStartup(t *testing.T) { + root, cwd := t.TempDir(), t.TempDir() + params := localLayoutTestParams(t, "local", root) + mr := &MilvusRoles{Local: true} + t.Chdir(cwd) + key := "index_files/7/0/2/3/milvus_packed_inverted_index.v3" + writeLocalLayoutTestFile(t, cwd, key, "old-index") + writeLocalLayoutTestFile(t, root, key, "new-index") + report, err := mr.migrateLocalStorageLayout(t.Context(), params) + require.ErrorIs(t, err, merr.ErrDataIntegrity) + require.Equal(t, []string{filepath.Join(root, key)}, report.Conflicts) + contents, err := os.ReadFile(filepath.Join(root, key)) + require.NoError(t, err) + require.Equal(t, "new-index", string(contents)) +} + +func TestMigrateLocalStorageLayoutWithoutLegacyFiles(t *testing.T) { + for _, layout := range []string{"existing-root", "missing-root"} { + t.Run(layout, func(t *testing.T) { + root, cwd := t.TempDir(), t.TempDir() + if layout == "missing-root" { + root = filepath.Join(root, "new-install") + } + params := localLayoutTestParams(t, "local", root) + if layout == "missing-root" { + // Configuration creates this empty directory when reading disk + // capacity. Remove it so migration really receives a missing root. + require.NoError(t, os.Remove(root)) + require.NoDirExists(t, root) + } + mr := &MilvusRoles{Local: true} + t.Chdir(cwd) + _, err := mr.migrateLocalStorageLayout(t.Context(), params) + require.NoError(t, err) + require.NoDirExists(t, filepath.Join(root, ".milvus-local-layout")) + if layout == "missing-root" { + require.NoDirExists(t, root, "migration must not create a missing storage root") + require.NoFileExists(t, filepath.Join(root, ".milvus-local-layout.lock")) + } + }) + } +} + +func TestMigrateLocalStorageLayoutRetriesAfterCopyFailure(t *testing.T) { + root, cwd := t.TempDir(), t.TempDir() + t.Chdir(cwd) + key := "index_files/7/0/2/3/milvus_packed_inverted_index.v3" + writeLocalLayoutTestFile(t, cwd, key, "old-index") + // A regular file in place of the destination directory makes rename fail. + blocker := filepath.Join(root, "index_files") + require.NoError(t, os.WriteFile(blocker, []byte("not-a-directory"), 0o600)) + mr := &MilvusRoles{Local: true} + _, err := mr.migrateLocalStorageLayout(t.Context(), localLayoutTestParams(t, "local", root)) + require.Error(t, err) + source, err := os.ReadFile(filepath.Join(cwd, key)) + require.NoError(t, err) + require.Equal(t, "old-index", string(source)) + + // Once the filesystem problem is resolved, a fresh standalone startup + // recovers directly from the same visible source without a shell handoff. + require.NoError(t, os.Remove(blocker)) + mr = &MilvusRoles{Local: true} + report, err := mr.migrateLocalStorageLayout(t.Context(), localLayoutTestParams(t, "local", root)) + require.NoError(t, err) + require.Equal(t, 1, report.Renamed) + require.Equal(t, 1, report.Renamed) + require.NoFileExists(t, filepath.Join(cwd, key)) + contents, err := os.ReadFile(filepath.Join(root, key)) + require.NoError(t, err) + require.Equal(t, "old-index", string(contents)) +} + +func TestMigrateLocalStorageLayoutProtectsLegacyManifestNamespace(t *testing.T) { + root, cwd := t.TempDir(), t.TempDir() + t.Chdir(cwd) + params := localLayoutTestParams(t, "local", root) + // This configured legacy prefix resolves to the same physical namespace as + // the double-root source. A relative manifest still reads there after reload. + relativeRoot, err := filepath.Rel(string(filepath.Separator), root) + require.NoError(t, err) + require.NoError(t, params.Save("minio.rootPath", filepath.ToSlash(relativeRoot))) + key := "insert_log/1/2/3/_data/a.parquet" + sourceRoot := localmigrate.DisplacedRoots(root)[0] + writeLocalLayoutTestFile(t, sourceRoot, key, "still-referenced") + mr := &MilvusRoles{Local: true} + _, err = mr.migrateLocalStorageLayout(t.Context(), params) + require.Error(t, err) + require.FileExists(t, filepath.Join(sourceRoot, key)) + require.NoFileExists(t, filepath.Join(root, key)) + require.NoDirExists(t, filepath.Join(root, ".milvus-local-layout", "backups")) +} diff --git a/cmd/roles/roles.go b/cmd/roles/roles.go index 4a84c2bb2c..6a4264bb23 100644 --- a/cmd/roles/roles.go +++ b/cmd/roles/roles.go @@ -435,6 +435,10 @@ func (mr *MilvusRoles) Run() { } params := paramtable.Get() + // Recover visible legacy local files before etcd and components start. + if !mr.migrateLocalStorageLayoutOrDie(ctx, params) { + return + } if params.EtcdCfg.UseEmbedEtcd.GetAsBool() { // Start etcd server. if err := etcd.InitEtcdServer( diff --git a/cmd/roles/roles_test.go b/cmd/roles/roles_test.go index 6db6477b1b..152e9df1f6 100644 --- a/cmd/roles/roles_test.go +++ b/cmd/roles/roles_test.go @@ -77,7 +77,9 @@ func TestFilesystemMetricsRegisteredWithRolesRegistry(t *testing.T) { "milvus_storage_filesystem_multi_part_upload_created": {}, "milvus_storage_filesystem_multi_part_upload_finished": {}, } - expectedDisplayKeyPrefix := "file://" + dir + "#fs:" + // Local storage shares one loon filesystem rooted at "/", whatever the + // configured localStorage.path is (see storagev2.LoonFSRootPath). + expectedDisplayKeyPrefix := "file://" + storagev2.LoonLocalFSRootPath + "#fs:" for _, family := range gathered { if _, ok := missingFamilies[family.GetName()]; !ok { continue diff --git a/configs/milvus.yaml b/configs/milvus.yaml index 55fd35f9e1..e3b486ec72 100644 --- a/configs/milvus.yaml +++ b/configs/milvus.yaml @@ -87,6 +87,8 @@ tikv: localStorage: # Local path to where vector data are stored during a search or a query to avoid repetitve access to MinIO or S3 service. + # Must be an absolute filesystem path. Milvus refuses to start if this value is relative or empty. + # Migration of data written using a relative path in older versions is not supported. # Caution: Changing this parameter after using Milvus for a period of time will affect your access to old data. # It is recommended to change this parameter before starting Milvus for the first time. path: /var/lib/milvus/data/ diff --git a/docs/agent_guides/README.md b/docs/agent_guides/README.md index ccea123700..b6b0ce4a6f 100644 --- a/docs/agent_guides/README.md +++ b/docs/agent_guides/README.md @@ -8,6 +8,8 @@ changing that subsystem. observability debugging. - [Streaming system](streaming-system/streaming-system.md): WAL, channels, DDL/DCL execution, recovery storage, and replication. +- [Storage path contract](storage/path_contract.md): complete keys, local upgrade + compatibility, and GC ownership boundaries. Update these guides with the code when an invariant, interface, or ownership boundary changes. diff --git a/docs/agent_guides/storage/path_contract.md b/docs/agent_guides/storage/path_contract.md new file mode 100644 index 0000000000..13bdb7d623 --- /dev/null +++ b/docs/agent_guides/storage/path_contract.md @@ -0,0 +1,113 @@ +# Storage path contract + +Read this before changing storage paths, manifests, local migration, or GC. +Update it with the code when a path or ownership rule changes. + +## Complete keys + +ChunkManager receives complete keys; it does not prepend its root. + +| Backend | Key | Example | +| --- | --- | --- | +| Local | Absolute filesystem path including `localStorage.path` | `/var/lib/milvus/data/insert_log/1/2/3/...` | +| Remote | Bucket-relative object key, normally including `minio.rootPath` | `files/insert_log/1/2/3/...` | + +**New local segment/index namespaces never use minio.rootPath.** Updates to an +existing legacy manifest retain its base path, including the historical prefix; +do not rebase that manifest while appending data or stats. A remote key does not include +the bucket name or a URI scheme. Local loon/Arrow filesystems use `/` as their +filesystem root so a complete key is not joined to the storage root twice. + +Explore is a separate layout: local temporary manifests live under +`localStorage.path/__explore_temp__/`; remote manifests live under +`__explore_temp__/` at the bucket root, without `minio.rootPath`. + +## Path ownership + +- Writers use shared layout helpers: `storage.SegmentManifestBasePath`, + `SegmentPartitionBasePath`, and the binlog/index helpers in `metautil`. +- Readers consume the complete key recorded in metadata or the manifest. + They must not prepend another root. +- Some API fields describe relative file names rather than complete keys. + TEXT stats results contain complete keys; JSON stats results contain names + relative to their field's stats directory. Resolve those names once at the + consuming boundary. +- Cleanup uses the same layout as the producer. Preserve remote keys exactly: + `.`, `..`, and repeated separators are literal object-key components. + +## Local upgrade compatibility + +Let `R = localStorage.path` and `M = the legacy minio.rootPath`. + +| Existing layout | Upgrade behavior | +| --- | --- | +| `R//...` | Use directly | +| Manifest base `M/insert_log/...`, files at `R/M/insert_log/...` | Resolve the loaded manifest to the existing absolute path; read in place | +| `R///...` | Move displaced layouts into the standard root at standalone startup | +| Unified indexes under the old process CWD | Move recognized index directories if they remain accessible | + +Migration uses rename, merges overlapping directories, and falls back to copy +on cross-filesystem moves. It does not overwrite conflicting files. Restart +resumes from remaining sources. Shutdown cancellation is checked while walking +and between file-copy reads; partial temporary copies are never published. There +is no configured migration timeout, reverse migration, or backup. +Old relative `localStorage.path` configurations are not supported. + +`normalizeLocalManifestPath` applies only to locally owned segments, not +remote data or foreign/self-contained snapshots. It updates the loaded view, +without writing the catalog; a later ordinary update may persist the path. +Keep the old `minio.rootPath` unchanged while relative legacy manifests remain. +Compatibility rules must not become new-write rules. + +TEXT LOB references contain a file ID and row offset, not their source storage +prefix. Compaction may reuse those encoded references only when every source +manifest and the output manifest have the same partition base. When a legacy +local source under `R/M/insert_log//

` is compacted into a canonical output +under `R/insert_log//

`, the source reader must decode TEXT through the +legacy LOB directory and the output writer must create new LOB files under the +canonical directory. Copying references or merely registering the old LOB file +metadata in the output manifest is not sufficient: readers reconstruct the LOB +directory from the output manifest base. + +## Necessary checks, at clear boundaries + +- Configuration initialization requires an absolute local root and freezes it + against runtime configuration updates. +- Metadata loading resolves recognized legacy paths and checks their identity. +- Deletion validates its segment/field ownership and applicable root boundary. + Recursive segment prefixes end in `/`, so `2001` cannot match `20010`. +- Avoid repeated URI or path-spelling blacklists. Check the actual range or + ownership constraint, and share a helper when the semantics are identical. +- Migration recognizes old CWD indexes by layout and + `milvus_packed__index.v3` names, rather than moving arbitrary CWD data. + Packed files move last during a merge so interrupted leaves stay recognizable. + Symlinks inside migration paths are refused; automatic alias repair is not + supported. Root confinement alone does not protect an in-root live alias. + +## GC coverage and limits + +- V1/V2: delete recorded files and periodically scan standard layouts for + orphans, including independent `bm25_stats` files. +- Dropped V3 segments: remove the manifest base directory, including data and + stats, subject to retention and snapshot protection. +- V3 orphan scan: enumerate `insert_log` and reclaim files without segment + metadata after the protection period. Files belonging to registered V3 + segments are skipped, even if an individual file is absent from the manifest. +- Therefore an uncommitted stats file inside a live V3 segment is not guaranteed + to be reclaimed by DataCoord's orphan scan. +- Legacy `R/M/insert_log` orphan/LOB layouts and old Explore temporary paths + have separate cleanup gaps. Do not claim this PR repairs all such leftovers. + Startup cannot recover indexes hidden with an old container's CWD. + +## Implementation and regression tests + +- Roots and writers: `internal/storagev2/loon_root.go`, + `internal/core/src/storage/loon_ffi/util.cpp`, + `internal/storage/binlog_record_writer.go`. +- Upgrade: `cmd/roles/local_layout.go`, `internal/storage/localmigrate/*_test.go`. +- Compatibility: `internal/datacoord/meta_local_manifest*.go` and + `snapshot_local_manifest_test.go`. +- Cleanup: `internal/datacoord/garbage_collector*.go` and `task_stats*.go`. + +Source review and unit tests do not replace a real version-to-version upgrade +test. Report which versions, layouts, and failure cases were actually exercised. diff --git a/internal/compaction/lob_compaction.go b/internal/compaction/lob_compaction.go index 3fe79c9a8f..dcd08f07d0 100644 --- a/internal/compaction/lob_compaction.go +++ b/internal/compaction/lob_compaction.go @@ -19,6 +19,7 @@ package compaction import ( "context" "fmt" + "path" "github.com/milvus-io/milvus-proto/go-api/v3/schemapb" "github.com/milvus-io/milvus/internal/storagev2/packed" @@ -43,10 +44,12 @@ const ( // // forced strategies: // - clustering compaction: always REWRITE_ALL (data is repartitioned) -// - sort compaction: always REUSE_ALL (row order changes but same data) +// - sort compaction: REUSE_ALL when source and output share a partition base +// (row order changes but the LOB namespace stays valid) // - schema-bump compaction: always REUSE_ALL (1->1, only materializes a new // non-LOB output column + bumps schema version; existing TEXT LOB data is -// unchanged, so its refs are carried as-is — never REWRITE_ALL) +// unchanged). Compactors must override this to REWRITE_ALL when a full +// rewrite moves the output to a different partition base. // - mix compaction with multiple outputs (N->M, M>1): always REWRITE_ALL // (LOB refs cannot be duplicated across output manifests without inflating ValidRows) // - L0 delete compaction: always SKIP (only applies delete logs, segment LOB refs unchanged) @@ -271,6 +274,10 @@ type LOBCompactionContext struct { ForcedStrategy LOBCompactionStrategy // isForced indicates whether strategy is forced (skip hole ratio calculation) IsForced bool + // DecodeTextFromSource is set when source and output partition bases differ. + // The compactor must decode references with the source base before writing + // new LOB files under the output base. + DecodeTextFromSource bool } // SegmentRowStats tracks row statistics for a segment during compaction @@ -355,6 +362,75 @@ func (ctx *LOBCompactionContext) ComputeStrategies(textFieldIDs []int64, thresho } } +// ForceRewriteAll overrides the normal compaction-type or hole-ratio decision. +// It is used when encoded TEXT references would cross a partition namespace: +// a reference contains only file ID and row offset, while readers reconstruct +// its LOB directory from the output manifest's partition base. +func (ctx *LOBCompactionContext) ForceRewriteAll(textFieldIDs []int64) { + if ctx == nil { + return + } + ctx.ForcedStrategy = LOBStrategyRewriteAll + ctx.IsForced = true + for _, fieldID := range textFieldIDs { + decision := ctx.Decisions[fieldID] + decision.FieldID = fieldID + decision.Strategy = LOBStrategyRewriteAll + ctx.Decisions[fieldID] = decision + } +} + +// ForceRewriteAllAcrossPartitionBases selects source-side TEXT decoding and +// canonical output rewriting in addition to overriding the field strategies. +func (ctx *LOBCompactionContext) ForceRewriteAllAcrossPartitionBases(textFieldIDs []int64) { + ctx.ForceRewriteAll(textFieldIDs) + if ctx != nil { + ctx.DecodeTextFromSource = true + } +} + +// LOBSourcePartitionBaseMismatch reports whether any source manifest resolves +// TEXT LOB references relative to a different partition base than the output. +// REUSE_ALL is safe only when every source and the output share this base. +func LOBSourcePartitionBaseMismatch(sourceManifests map[int64]string, outputPartitionBase string) (bool, error) { + outputPartitionBase = path.Clean(outputPartitionBase) + for _, manifestPath := range sourceManifests { + if manifestPath == "" { + continue + } + segmentBasePath, _, err := packed.UnmarshalManifestPath(manifestPath) + if err != nil { + return false, err + } + if path.Clean(path.Dir(segmentBasePath)) != outputPartitionBase { + return true, nil + } + } + return false, nil +} + +// GetSourceTextColumnConfigs derives the source LOB directories used by the +// TEXT-decoding SegmentReader for a single manifest. +func (ctx *LOBCompactionContext) GetSourceTextColumnConfigs(manifestPath string) ([]packed.TextColumnConfig, error) { + if ctx == nil || !ctx.DecodeTextFromSource { + return nil, nil + } + segmentBasePath, _, err := packed.UnmarshalManifestPath(manifestPath) + if err != nil { + return nil, err + } + partitionBasePath := path.Dir(segmentBasePath) + fieldIDs := ctx.GetRewriteAllFieldIDs() + configs := make([]packed.TextColumnConfig, 0, len(fieldIDs)) + for _, fieldID := range fieldIDs { + configs = append(configs, packed.TextColumnConfig{ + FieldID: fieldID, + LobBasePath: path.Join(partitionBasePath, "lobs", fmt.Sprint(fieldID)), + }) + } + return configs, nil +} + // GetStrategy returns the strategy for a specific field. func (ctx *LOBCompactionContext) GetStrategy(fieldID int64) LOBCompactionStrategy { if decision, ok := ctx.Decisions[fieldID]; ok { @@ -655,7 +731,7 @@ func (ctx *LOBCompactionContext) GetTextColumnConfigs(partitionBasePath string, InlineThreshold: inlineThreshold, MaxLobFileBytes: maxLobFileBytes, FlushThresholdBytes: flushThresholdBytes, - RewriteMode: true, + RewriteMode: !ctx.DecodeTextFromSource, }) } diff --git a/internal/compaction/lob_compaction_test.go b/internal/compaction/lob_compaction_test.go index 516ca2d94b..6e1b8ab5ba 100644 --- a/internal/compaction/lob_compaction_test.go +++ b/internal/compaction/lob_compaction_test.go @@ -26,6 +26,90 @@ import ( "github.com/milvus-io/milvus/pkg/v3/proto/datapb" ) +func TestLOBSourcePartitionBaseMismatch(t *testing.T) { + outputPartitionBase := "/var/lib/milvus/data/insert_log/1/2" + tests := []struct { + name string + manifests map[int64]string + want bool + wantErr bool + }{ + { + name: "same canonical partition", + manifests: map[int64]string{ + 10: packed.MarshalManifestPath(outputPartitionBase+"/10", 1), + 11: packed.MarshalManifestPath(outputPartitionBase+"/11", 2), + }, + }, + { + name: "legacy partition", + manifests: map[int64]string{ + 10: packed.MarshalManifestPath("/var/lib/milvus/data/files/insert_log/1/2/10", 1), + }, + want: true, + }, + { + name: "mixed partitions", + manifests: map[int64]string{ + 10: packed.MarshalManifestPath(outputPartitionBase+"/10", 1), + 11: packed.MarshalManifestPath("/var/lib/milvus/data/files/insert_log/1/2/11", 1), + }, + want: true, + }, + { + name: "invalid manifest", + manifests: map[int64]string{10: "not-json"}, + wantErr: true, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got, err := LOBSourcePartitionBaseMismatch(tc.manifests, outputPartitionBase) + if tc.wantErr { + assert.Error(t, err) + return + } + assert.NoError(t, err) + assert.Equal(t, tc.want, got) + }) + } +} + +func TestLOBCompactionContext_ForceRewriteAll(t *testing.T) { + ctx := NewLOBCompactionContext() + ctx.SetCompactionType(datapb.CompactionType_SortCompaction, 1, 1) + ctx.ComputeStrategies([]int64{101, 102}, DefaultLOBHoleRatioThreshold) + + ctx.ForceRewriteAll([]int64{101, 102}) + + assert.True(t, ctx.IsForced) + assert.Equal(t, LOBStrategyRewriteAll, ctx.ForcedStrategy) + assert.False(t, ctx.HasReuseAllFields()) + assert.True(t, ctx.ShouldRewriteAnyField()) + assert.Equal(t, LOBStrategyRewriteAll, ctx.GetStrategy(101)) + assert.Equal(t, LOBStrategyRewriteAll, ctx.GetStrategy(102)) +} + +func TestLOBCompactionContext_CrossPartitionRewriteConfigs(t *testing.T) { + ctx := NewLOBCompactionContext() + ctx.SetCompactionType(datapb.CompactionType_SortCompaction, 1, 1) + ctx.ComputeStrategies([]int64{101}, DefaultLOBHoleRatioThreshold) + ctx.ForceRewriteAllAcrossPartitionBases([]int64{101}) + + sourceManifest := packed.MarshalManifestPath("/data/files/insert_log/1/2/10", 3) + sourceConfigs, err := ctx.GetSourceTextColumnConfigs(sourceManifest) + assert.NoError(t, err) + if assert.Len(t, sourceConfigs, 1) { + assert.Equal(t, "/data/files/insert_log/1/2/lobs/101", sourceConfigs[0].LobBasePath) + } + outputConfigs := ctx.GetTextColumnConfigs("/data/insert_log/1/2", 64, 1024, 512) + if assert.Len(t, outputConfigs, 1) { + assert.Equal(t, "/data/insert_log/1/2/lobs/101", outputConfigs[0].LobBasePath) + assert.False(t, outputConfigs[0].RewriteMode, "input is already decoded by the source reader") + } +} + func TestGetForcedStrategy_AllTypes(t *testing.T) { tests := []struct { name string diff --git a/internal/core/src/clustering/KmeansClusteringTest.cpp b/internal/core/src/clustering/KmeansClusteringTest.cpp index 8d0c6fa12e..3ed2cf324b 100644 --- a/internal/core/src/clustering/KmeansClusteringTest.cpp +++ b/internal/core/src/clustering/KmeansClusteringTest.cpp @@ -386,11 +386,9 @@ TEST(KmeansClusteringTest, ReadFromManifestStorageV3) { ASSERT_EQ(v3.TotalRows(), num_rows); ASSERT_EQ(v3.NumColumnGroups(), 1); - // The fixture wrote to the LOCAL filesystem, and - // MakeInternalPropertiesFromStorageConfig copies storage_type/root_path - // verbatim into the loon properties. So the storage config must be - // local-typed and rooted at root_path (ManifestPathJson() returns a - // relative base_path; loon resolves it against PROPERTY_FS_ROOT_PATH). + // The fixture wrote to the local filesystem and ManifestPathJson() returns + // an absolute base_path. MakeInternalPropertiesFromStorageConfig uses "/" + // as the local filesystem root, leaving that complete path unchanged. // Leaving StorageConfig at its default ("minio") would send the manifest // reader to object storage. auto storage_config = gen_local_storage_config(root_path); @@ -424,8 +422,7 @@ TEST(KmeansClusteringTest, ReadFromManifestStorageV3) { pb_storage_config->set_root_path(storage_config.root_path); // Mirror analyze_c.cpp: the field schema is carried in FieldDataMeta, and - // the loon properties come from the storage config through the very - // production helper under test - not from MakeInternalLocalProperies. + // the loon properties come from the production storage-config helper. milvus::storage::FieldDataMeta field_data_meta{ collection_id, partition_id, 0, vec_fid.get(), info.field_schema()}; milvus::storage::IndexMeta index_meta{ diff --git a/internal/core/src/common/VectorArrayStorageV2Test.cpp b/internal/core/src/common/VectorArrayStorageV2Test.cpp index f8b540b90a..4f5b09ad2c 100644 --- a/internal/core/src/common/VectorArrayStorageV2Test.cpp +++ b/internal/core/src/common/VectorArrayStorageV2Test.cpp @@ -138,8 +138,9 @@ class TestVectorArrayStorageV2 : public testing::Test { auto fs = milvus::segcore::GetDefaultArrowFileSystem(); // Prepare paths and column groups - std::vector paths = {"test_data/0/10000.parquet", - "test_data/101/10001.parquet"}; + std::vector paths = { + TestLocalPath + "test_data/0/10000.parquet", + TestLocalPath + "test_data/101/10001.parquet"}; // Create directories for the parquet files for (const auto& path : paths) { @@ -293,7 +294,7 @@ class TestVectorArrayStorageV2 : public testing::Test { void TearDown() override { auto fs = milvus::segcore::GetDefaultArrowFileSystem(); - (void)fs->DeleteDir("test_data"); + (void)fs->DeleteDir(TestLocalPath + "test_data"); } protected: @@ -312,7 +313,8 @@ TEST_F(TestVectorArrayStorageV2, BuildEmbListHNSWIndex) { ASSERT_TRUE(segment_->HasFieldData(vector_array_field_id)); // Get the storage v2 parquet file paths that were already written in SetUp - std::vector paths = {"test_data/101/10001.parquet"}; + std::vector paths = {TestLocalPath + + "test_data/101/10001.parquet"}; // Use the existing Arrow file system from SetUp auto fs = milvus::segcore::GetDefaultArrowFileSystem(); @@ -427,7 +429,8 @@ TEST_F(TestVectorArrayStorageV2, BuildEmbListHNSWIndexWithMmap) { ASSERT_TRUE(segment_->HasFieldData(vector_array_field_id)); // Get the storage v2 parquet file paths that were already written in SetUp - std::vector paths = {"test_data/101/10001.parquet"}; + std::vector paths = {TestLocalPath + + "test_data/101/10001.parquet"}; // Use the existing Arrow file system from SetUp auto fs = milvus::segcore::GetDefaultArrowFileSystem(); @@ -565,7 +568,8 @@ TEST_F(TestVectorArrayStorageV2, BuildEncodedEmbListHNSWIndexWithMmap) { auto vector_array_field_id = fields_["vector_array"]; ASSERT_TRUE(segment_->HasFieldData(vector_array_field_id)); - std::vector paths = {"test_data/101/10001.parquet"}; + std::vector paths = {TestLocalPath + + "test_data/101/10001.parquet"}; auto fs = milvus::segcore::GetDefaultArrowFileSystem(); auto storage_config = gen_local_storage_config(TestLocalPath); auto cm = CreateChunkManager(storage_config); diff --git a/internal/core/src/exec/expression/LikeConjunctExprTest.cpp b/internal/core/src/exec/expression/LikeConjunctExprTest.cpp index eb48eabb33..b6753f7e0f 100644 --- a/internal/core/src/exec/expression/LikeConjunctExprTest.cpp +++ b/internal/core/src/exec/expression/LikeConjunctExprTest.cpp @@ -32,6 +32,7 @@ #include "common/Types.h" #include "common/protobuf_utils.h" #include "common/type_c.h" +#include "exec/expression/ExprBatchTestUtils.h" #include "exec/expression/function/FunctionFactory.h" #include "expr/ITypeExpr.h" #include "filemanager/InputStream.h" @@ -86,7 +87,7 @@ TEST(LikeConjunctExpr, TestMultiFieldMultiLikeWithRetrieve) { int64_t segment_id = 3; int64_t index_version = 4000; - EXEC_EVAL_EXPR_BATCH_SIZE.store(3); + milvus::test::ExprBatchSizeGuard batch_size_guard(3); auto schema = std::make_shared(); auto pk_fid = schema->AddDebugField("pk", DataType::INT64); @@ -94,8 +95,9 @@ TEST(LikeConjunctExpr, TestMultiFieldMultiLikeWithRetrieve) { auto content_fid = schema->AddDebugField("content", DataType::VARCHAR); schema->set_primary_field_id(pk_fid); - // Use TestLocalPath to match LocalChunkManagerSingleton initialized in init_gtest.cpp - auto storage_config = gen_local_storage_config(TestLocalPath); + // AppendIndexV2 loads through RemoteChunkManagerSingleton, so build the + // index under the same TestRemotePath root. + auto storage_config = get_default_local_storage_config(); auto cm = CreateChunkManager(storage_config); auto fs = storage::InitArrowFileSystem(storage_config); @@ -202,13 +204,14 @@ TEST(LikeConjunctExpr, TestMultiFieldMultiLikeWithRetrieve) { insert_data.SetTimestamps(0, 100); auto serialized_bytes = insert_data.Serialize(storage::Remote); - auto log_path = fmt::format("{}{}/{}/{}/{}/{}", - TestLocalPath, - collection_id, - partition_id, - segment_id, - field_id.get(), - 0); + auto log_path = + fmt::format("{}insert_log/like_conjunct/{}/{}/{}/{}/{}", + storage_config.root_path, + collection_id, + partition_id, + segment_id, + field_id.get(), + 0); cm_w.Write( log_path, serialized_bytes.data(), serialized_bytes.size()); @@ -358,6 +361,4 @@ TEST(LikeConjunctExpr, TestMultiFieldMultiLikeWithRetrieve) { } EXPECT_EQ(actual_pks, expected_pks); - - EXEC_EVAL_EXPR_BATCH_SIZE.store(DEFAULT_EXEC_EVAL_EXPR_BATCH_SIZE); } diff --git a/internal/core/src/index/FMIndexTest.cpp b/internal/core/src/index/FMIndexTest.cpp index 4e4dc1b542..c2ef922bdb 100644 --- a/internal/core/src/index/FMIndexTest.cpp +++ b/internal/core/src/index/FMIndexTest.cpp @@ -1067,7 +1067,7 @@ TEST(FMIndex, ExecutorPathDeclinedOpsFallBackToScan) { false); auto index_meta = gen_index_meta( segment_id, field_id.get(), index_build_id, index_version); - auto storage_config = gen_local_storage_config(TestLocalPath); + auto storage_config = get_default_local_storage_config(); auto cm = CreateChunkManager(storage_config); auto fs = storage::InitArrowFileSystem(storage_config); @@ -1096,8 +1096,8 @@ TEST(FMIndex, ExecutorPathDeclinedOpsFallBackToScan) { insert_data.SetFieldDataMeta(field_meta); insert_data.SetTimestamps(0, 100); auto serialized_bytes = insert_data.Serialize(storage::Remote); - auto log_path = fmt::format("{}{}/{}/{}/{}/{}", - TestLocalPath, + auto log_path = fmt::format("{}insert_log/fm_index/{}/{}/{}/{}/{}", + storage_config.root_path, collection_id, partition_id, segment_id, @@ -1256,7 +1256,7 @@ TEST(FMIndex, ExecutorPathMatchRechecksVarchar) { false); auto index_meta = gen_index_meta( segment_id, field_id.get(), index_build_id, index_version); - auto storage_config = gen_local_storage_config(TestLocalPath); + auto storage_config = get_default_local_storage_config(); auto cm = CreateChunkManager(storage_config); auto fs = storage::InitArrowFileSystem(storage_config); @@ -1297,8 +1297,8 @@ TEST(FMIndex, ExecutorPathMatchRechecksVarchar) { insert_data.SetFieldDataMeta(field_meta); insert_data.SetTimestamps(0, 100); auto serialized_bytes = insert_data.Serialize(storage::Remote); - auto log_path = fmt::format("{}{}/{}/{}/{}/{}", - TestLocalPath, + auto log_path = fmt::format("{}insert_log/fm_index/{}/{}/{}/{}/{}", + storage_config.root_path, collection_id, partition_id, segment_id, @@ -1419,9 +1419,8 @@ struct SealedFMMatch { FieldId varchar_id; FieldId int_id; std::unique_ptr segment; - // Owns the insert-log write and keeps TestLocalPath alive until the - // loaded segment is destroyed. ChunkManagerWrapper::dtor wipes the - // chunk-manager root. + // Owns the insert-log write and cleans the remote test root with this + // fixture. ChunkManagerWrapper::dtor wipes the chunk-manager root. std::unique_ptr cm_w; }; @@ -1488,11 +1487,11 @@ LoadSealedFMMatch(int64_t collection_id, /*max_length=*/65535); auto index_meta = gen_index_meta( segment_id, out.varchar_id.get(), index_build_id, index_build_id); - auto storage_config = gen_local_storage_config(TestLocalPath); + auto storage_config = get_default_local_storage_config(); auto cm = CreateChunkManager(storage_config); - // Same FS handle as ExecutorPathMatchRechecksVarchar. UploadUnified writes - // through StorageV2FSCache; AppendIndexV2 reads through the process Arrow - // FS. Both are rooted at TestLocalPath, so the packed file is visible. + // AppendIndexV2 loads through RemoteChunkManagerSingleton, so build the + // index under that singleton's TestRemotePath root as well. TestLocalPath + // remains the mmap/local-temp root. auto fs = storage::InitArrowFileSystem(storage_config); out.cm_w = std::make_unique(cm); @@ -1544,8 +1543,8 @@ LoadSealedFMMatch(int64_t collection_id, insert_data.SetFieldDataMeta(field_meta); insert_data.SetTimestamps(0, 100); auto serialized_bytes = insert_data.Serialize(storage::Remote); - auto log_path = fmt::format("{}{}/{}/{}/{}/{}", - TestLocalPath, + auto log_path = fmt::format("{}insert_log/fm_index/{}/{}/{}/{}/{}", + storage_config.root_path, collection_id, partition_id, segment_id, diff --git a/internal/core/src/index/Meta.h b/internal/core/src/index/Meta.h index 01d00a6aff..3acbf1f061 100644 --- a/internal/core/src/index/Meta.h +++ b/internal/core/src/index/Meta.h @@ -133,8 +133,4 @@ constexpr const char* DISK_ANN_QUERY_BEAMWIDTH = "beamwidth"; // HNSW query params constexpr const char* HNSW_QUERY_EF = "ef"; -// UT-only: when non-empty, overrides rcm_->GetRootPath() in -// GetRemoteIndexObjectPrefix/GetRemoteTextLogPrefix to avoid -// absolute path duplication with SubTreeFileSystem in tests. -extern std::string kOverrideRootPathForUT; } // namespace milvus::index diff --git a/internal/core/src/index/NgramInvertedIndexTest.cpp b/internal/core/src/index/NgramInvertedIndexTest.cpp index b9be6dd5b9..4030c422d3 100644 --- a/internal/core/src/index/NgramInvertedIndexTest.cpp +++ b/internal/core/src/index/NgramInvertedIndexTest.cpp @@ -147,8 +147,7 @@ test_ngram_with_data(const boost::container::vector& data, auto index_meta = gen_index_meta( segment_id, field_id.get(), index_build_id, index_version); - std::string root_path = TestLocalPath; - auto storage_config = gen_local_storage_config(root_path); + auto storage_config = get_default_local_storage_config(); auto cm = CreateChunkManager(storage_config); auto fs = storage::InitArrowFileSystem(storage_config); @@ -176,8 +175,8 @@ test_ngram_with_data(const boost::container::vector& data, auto serialized_bytes = insert_data.Serialize(storage::Remote); auto get_binlog_path = [=](int64_t log_id) { - return fmt::format("{}{}/{}/{}/{}/{}", - TestLocalPath, + return fmt::format("{}insert_log/ngram_index/{}/{}/{}/{}/{}", + storage_config.root_path, collection_id, partition_id, segment_id, @@ -466,8 +465,7 @@ TEST(NgramIndex, TestNonLikeExpressionsWithNgram) { auto index_meta = gen_index_meta( segment_id, field_id.get(), index_build_id, index_version); - std::string root_path = TestLocalPath; - auto storage_config = gen_local_storage_config(root_path); + auto storage_config = get_default_local_storage_config(); auto cm = CreateChunkManager(storage_config); auto fs = storage::InitArrowFileSystem(storage_config); @@ -495,8 +493,8 @@ TEST(NgramIndex, TestNonLikeExpressionsWithNgram) { auto serialized_bytes = insert_data.Serialize(storage::Remote); auto get_binlog_path = [=](int64_t log_id) { - return fmt::format("{}{}/{}/{}/{}/{}", - TestLocalPath, + return fmt::format("{}insert_log/ngram_index/{}/{}/{}/{}/{}", + storage_config.root_path, collection_id, partition_id, segment_id, diff --git a/internal/core/src/segcore/ChunkedSegmentSealedStorageV2Test.cpp b/internal/core/src/segcore/ChunkedSegmentSealedStorageV2Test.cpp index cd635c5acc..deef6e138c 100644 --- a/internal/core/src/segcore/ChunkedSegmentSealedStorageV2Test.cpp +++ b/internal/core/src/segcore/ChunkedSegmentSealedStorageV2Test.cpp @@ -22,6 +22,8 @@ #include #include #include + +#include "test_utils/Constants.h" #include #include #include @@ -246,7 +248,8 @@ TEST(ChunkedSegmentSealedStorageV2, auto schema = Schema::ParseFrom(schema_proto); auto fs = milvus::segcore::GetDefaultArrowFileSystem(); - const std::string dir = "test_data/storage_v2_direct_warmup"; + const std::string dir = + TestLocalPath + "test_data/storage_v2_direct_warmup"; StorageV2TempDirGuard dir_guard(fs, dir); const std::string path = dir + "/vec.parquet"; ASSERT_TRUE(fs->CreateDir(dir).ok()); @@ -365,9 +368,10 @@ class TestChunkSegmentStorageV2 : public testing::TestWithParam { auto fs = milvus::segcore::GetDefaultArrowFileSystem(); // Prepare paths and column groups - std::vector paths = {"test_data/0/10000.parquet", - "test_data/102/10001.parquet", - "test_data/103/10002.parquet"}; + std::vector paths = { + TestLocalPath + "test_data/0/10000.parquet", + TestLocalPath + "test_data/102/10001.parquet", + TestLocalPath + "test_data/103/10002.parquet"}; // Create directories for the parquet files for (const auto& path : paths) { @@ -496,7 +500,7 @@ class TestChunkSegmentStorageV2 : public testing::TestWithParam { } // Clean up test data directory auto fs = milvus::segcore::GetDefaultArrowFileSystem(); - auto status = fs->DeleteDir("test_data"); + auto status = fs->DeleteDir(TestLocalPath + "test_data"); ASSERT_TRUE(status.ok()); } @@ -909,7 +913,8 @@ TEST(TestChunkSegmentStorageV2Regression, schema->set_primary_field_id(right_fid); auto fs = milvus::segcore::GetDefaultArrowFileSystem(); - const std::string root = "test_compare_expr_misaligned_storage_v2"; + const std::string root = + TestLocalPath + "test_compare_expr_misaligned_storage_v2"; auto cleanup_status = fs->DeleteDir(root); (void)cleanup_status; ASSERT_TRUE(fs->CreateDir(root + "/0").ok()); diff --git a/internal/core/src/segcore/flush_growing_segment_test.cpp b/internal/core/src/segcore/flush_growing_segment_test.cpp index 9e8fbc6406..b01196adf0 100644 --- a/internal/core/src/segcore/flush_growing_segment_test.cpp +++ b/internal/core/src/segcore/flush_growing_segment_test.cpp @@ -14,7 +14,6 @@ #include #include #include -#include #include #include #include @@ -28,6 +27,7 @@ #include "test_utils/c_api_test_utils.h" #include "test_utils/DataGen.h" #include "test_utils/SegcoreConfigUtils.h" +#include "test_utils/TmpPath.h" #include "storage/Util.h" #include "storage/loon_ffi/property_singleton.h" #include "knowhere/index/index_factory.h" @@ -38,8 +38,6 @@ using namespace milvus; using namespace milvus::segcore; -namespace fs = std::filesystem; - class FlushGrowingSegmentTest : public ::testing::Test { protected: struct ParsedBM25Stats { @@ -50,21 +48,13 @@ class FlushGrowingSegmentTest : public ::testing::Test { void SetUp() override { - // create a temporary directory for test output - test_dir_ = "/tmp/flush_growing_test_" + std::to_string(time(nullptr)); - fs::create_directories(test_dir_); - + // Each fixture owns a unique directory under the shard-specific + // TestLocalPath. TmpPath also cleans up only this fixture's files. + test_dir_ = temp_path_.get().string(); // Arrow filesystem is initialized by init_gtest.cpp } - void - TearDown() override { - // cleanup test directory - if (fs::exists(test_dir_)) { - fs::remove_all(test_dir_); - } - } - + milvus::test::TmpPath temp_path_; std::string test_dir_; std::vector diff --git a/internal/core/src/storage/FileManager.h b/internal/core/src/storage/FileManager.h index d6906f50d6..af3c6f35db 100644 --- a/internal/core/src/storage/FileManager.h +++ b/internal/core/src/storage/FileManager.h @@ -309,9 +309,7 @@ class FileManagerImpl : public milvus::FileManager { virtual std::string GetRemoteIndexObjectPrefix() const { - boost::filesystem::path prefix = index::kOverrideRootPathForUT.empty() - ? rcm_->GetRootPath() - : index::kOverrideRootPathForUT; + boost::filesystem::path prefix = rcm_->GetRootPath(); if (index_meta_.index_store_path_version >= ::milvus::proto::index::IndexStorePathVersion:: INDEX_STORE_PATH_VERSION_COLLECTION_ROOTED) { @@ -336,9 +334,7 @@ class FileManagerImpl : public milvus::FileManager { if (!stats_base_path_.empty()) { return stats_base_path_; } - boost::filesystem::path prefix = index::kOverrideRootPathForUT.empty() - ? rcm_->GetRootPath() - : index::kOverrideRootPathForUT; + boost::filesystem::path prefix = rcm_->GetRootPath(); boost::filesystem::path path = std::string(TEXT_LOG_ROOT_PATH); boost::filesystem::path path1 = std::to_string(index_meta_.build_id) + "/" + diff --git a/internal/core/src/storage/StorageV2FSCache.cpp b/internal/core/src/storage/StorageV2FSCache.cpp index 3da8425b61..453ff0036e 100644 --- a/internal/core/src/storage/StorageV2FSCache.cpp +++ b/internal/core/src/storage/StorageV2FSCache.cpp @@ -16,6 +16,7 @@ #include "milvus-storage/filesystem/fs.h" #include "milvus-storage/properties.h" #include "log/Log.h" +#include "storage/loon_ffi/util.h" namespace milvus::storage { @@ -39,7 +40,11 @@ StorageV2FSCache::Get(const Key& key) { props[PROPERTY_FS_BUCKET_NAME] = key.bucket_name; props[PROPERTY_FS_ACCESS_KEY_ID] = key.access_key_id; props[PROPERTY_FS_ACCESS_KEY_VALUE] = key.access_key_value; - props[PROPERTY_FS_ROOT_PATH] = key.root_path; + // Keys handed to this filesystem already carry the storage prefix, so the + // local filesystem is rooted at "/" (remote is rooted at the bucket by + // milvus-storage regardless of root_path). See LoonFSRootPath. + const auto fs_root_path = LoonFSRootPath(key.storage_type, key.root_path); + props[PROPERTY_FS_ROOT_PATH] = fs_root_path; props[PROPERTY_FS_STORAGE_TYPE] = key.storage_type; props[PROPERTY_FS_CLOUD_PROVIDER] = key.cloud_provider; props[PROPERTY_FS_IAM_ENDPOINT] = key.iam_endpoint; @@ -60,11 +65,12 @@ StorageV2FSCache::Get(const Key& key) { props[PROPERTY_FS_USE_CRC32C_CHECKSUM] = key.use_crc32c_checksum; LOG_INFO( - "StorageV2FSCache::Get: address={}, bucket={}, root_path={}, " - "storage_type={}", + "StorageV2FSCache::Get: address={}, bucket={}, key_prefix={}, " + "fs_root_path={}, storage_type={}", key.address, key.bucket_name, key.root_path, + fs_root_path, key.storage_type); auto result = milvus_storage::FilesystemCache::getInstance().get(props, ""); diff --git a/internal/core/src/storage/StorageV2FSCacheLocalRootTest.cpp b/internal/core/src/storage/StorageV2FSCacheLocalRootTest.cpp new file mode 100644 index 0000000000..07d4dc1f10 --- /dev/null +++ b/internal/core/src/storage/StorageV2FSCacheLocalRootTest.cpp @@ -0,0 +1,104 @@ +// Licensed to the LF AI & Data foundation under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include + +#include +#include + +#include "common/type_c.h" +#include "milvus-storage/properties.h" +#include "storage/StorageV2FSCache.h" +#include "storage/loon_ffi/util.h" + +namespace { + +std::string +MakeTempRoot(const std::string& name) { + auto root = + std::filesystem::temp_directory_path() / + ("milvus_local_root_" + name + "_" + std::to_string(::getpid())); + std::filesystem::remove_all(root); + std::filesystem::create_directories(root); + return root.string(); +} + +} // namespace + +// Milvus hands complete filesystem paths (already prefixed with +// localStorage.path) to the local Arrow / loon filesystem, so that filesystem +// must be rooted at "/". If it were rooted at localStorage.path again, every +// write would land under //... (milvus-storage #351, #53051). +TEST(StorageV2FSCacheLocalRootTest, AbsolutePathsAreNotJoinedWithRootTwice) { + const auto root = MakeTempRoot("fs"); + + milvus::storage::StorageV2FSCache::Key key; + key.root_path = root; + key.storage_type = "local"; + auto fs = milvus::storage::StorageV2FSCache::Instance().Get(key); + ASSERT_NE(fs, nullptr); + + const auto dir = root + "/insert_log/1/2/3/_data"; + const auto file = dir + "/0.parquet"; + ASSERT_TRUE(fs->CreateDir(dir, /*recursive=*/true).ok()); + auto out = fs->OpenOutputStream(file); + ASSERT_TRUE(out.ok()) << out.status().ToString(); + ASSERT_TRUE(out.ValueOrDie()->Write("x", 1).ok()); + ASSERT_TRUE(out.ValueOrDie()->Close().ok()); + + // Physical location is exactly the key. + EXPECT_TRUE(std::filesystem::exists(file)); + // And not the double-joined location. + EXPECT_FALSE(std::filesystem::exists(root + root + "/insert_log")); + + // Reading back through the same filesystem with the same key works. + auto in = fs->OpenInputFile(file); + ASSERT_TRUE(in.ok()) << in.status().ToString(); + auto size = in.ValueOrDie()->GetSize(); + ASSERT_TRUE(size.ok()); + EXPECT_EQ(size.ValueOrDie(), 1); + + std::filesystem::remove_all(root); +} + +TEST(StorageV2FSCacheLocalRootTest, LoonFSRootPathRule) { + EXPECT_EQ(LoonFSRootPath("local", "/var/lib/milvus/data"), "/"); + EXPECT_EQ(LoonFSRootPath("local", ""), "/"); + EXPECT_EQ(LoonFSRootPath("remote", "files"), "files"); + EXPECT_EQ(LoonFSRootPath("minio", "files"), "files"); + EXPECT_EQ(LoonFSRootPath("", "files"), "files"); +} + +TEST(StorageV2FSCacheLocalRootTest, InternalPropertiesUseSlashForLocal) { + CStorageConfig c_config{}; + c_config.root_path = "/var/lib/milvus/data"; + c_config.storage_type = "local"; + auto props = MakeInternalPropertiesFromStorageConfig(c_config); + auto root = milvus_storage::api::GetValue( + *props, PROPERTY_FS_ROOT_PATH); + ASSERT_TRUE(root.ok()); + EXPECT_EQ(root.ValueOrDie(), "/"); + + CStorageConfig remote{}; + remote.root_path = "files"; + remote.storage_type = "remote"; + auto remote_props = MakeInternalPropertiesFromStorageConfig(remote); + auto remote_root = milvus_storage::api::GetValue( + *remote_props, PROPERTY_FS_ROOT_PATH); + ASSERT_TRUE(remote_root.ok()); + EXPECT_EQ(remote_root.ValueOrDie(), "files"); +} diff --git a/internal/core/src/storage/loon_ffi/property_singleton.h b/internal/core/src/storage/loon_ffi/property_singleton.h index 9d8e6c20f6..c742546076 100644 --- a/internal/core/src/storage/loon_ffi/property_singleton.h +++ b/internal/core/src/storage/loon_ffi/property_singleton.h @@ -65,16 +65,6 @@ class LoonFFIPropertiesSingleton { } } - void - Init(const char* root_path) { - std::unique_lock lck(mutex_); - - if (properties_ == nullptr) { - properties_ = MakeInternalLocalProperies(root_path); - ApplyArrowReaderConfig(*properties_); - } - } - void SetArrowReaderConfig(int64_t hole_size_limit_bytes, int64_t range_size_limit_bytes) { diff --git a/internal/core/src/storage/loon_ffi/util.cpp b/internal/core/src/storage/loon_ffi/util.cpp index 4c265dddab..35e1abceb7 100644 --- a/internal/core/src/storage/loon_ffi/util.cpp +++ b/internal/core/src/storage/loon_ffi/util.cpp @@ -74,9 +74,17 @@ MakeInternalPropertiesFromStorageConfig(CStorageConfig c_storage_config) { PROPERTY_FS_ACCESS_KEY_VALUE, c_storage_config.access_key_value); } - if (c_storage_config.root_path != nullptr) { + const std::string storage_type = c_storage_config.storage_type != nullptr + ? c_storage_config.storage_type + : ""; + if (c_storage_config.root_path != nullptr || storage_type == "local") { + const std::string root_path = c_storage_config.root_path != nullptr + ? c_storage_config.root_path + : ""; milvus_storage::api::SetValue( - *properties_map, PROPERTY_FS_ROOT_PATH, c_storage_config.root_path); + *properties_map, + PROPERTY_FS_ROOT_PATH, + LoonFSRootPath(storage_type, root_path).c_str()); } if (c_storage_config.storage_type != nullptr) { milvus_storage::api::SetValue(*properties_map, @@ -801,17 +809,12 @@ InjectExternalSpecProperties(milvus_storage::api::Properties& properties, } } -std::shared_ptr -MakeInternalLocalProperies(const char* c_path) { - auto properties_map = std::make_shared(); - - milvus_storage::api::SetValue( - *properties_map, PROPERTY_FS_STORAGE_TYPE, "local"); - - milvus_storage::api::SetValue( - *properties_map, PROPERTY_FS_ROOT_PATH, c_path); - - return properties_map; +std::string +LoonFSRootPath(const std::string& storage_type, const std::string& root_path) { + if (storage_type == "local") { + return kLoonLocalFSRootPath; + } + return root_path; } CStorageConfig diff --git a/internal/core/src/storage/loon_ffi/util.h b/internal/core/src/storage/loon_ffi/util.h index 11c2e0ab05..870a5ec8f6 100644 --- a/internal/core/src/storage/loon_ffi/util.h +++ b/internal/core/src/storage/loon_ffi/util.h @@ -34,16 +34,22 @@ std::shared_ptr MakeInternalPropertiesFromStorageConfig(CStorageConfig c_storage_config); /** - * @brief Create Properties for local filesystem storage + * @brief fs.root_path handed to the loon / Arrow filesystem for a storage + * config. * - * Creates a minimal Properties object configured for local file storage - * with the specified path as the root. - * - * @param c_path Local filesystem path to use as storage root - * @return Shared pointer to Properties configured for local storage + * Milvus keys already carry the storage prefix: localStorage.path for local + * (`/var/lib/milvus/data/insert_log/...`) and minio.rootPath for remote + * (`files/insert_log/...`). The filesystem therefore has to be rooted at the + * namespace root, which is the bucket for remote (milvus-storage never + * applies root_path there) and "/" for local. Rooting the local + * SubTreeFileSystem at localStorage.path joins the prefix twice + * (milvus-storage #351, milvus #53051), making complete keys differ from their + * physical locations. */ -std::shared_ptr -MakeInternalLocalProperies(const char* c_path); +inline constexpr const char* kLoonLocalFSRootPath = "/"; + +std::string +LoonFSRootPath(const std::string& storage_type, const std::string& root_path); /** * @brief Convert StorageConfig to C-style CStorageConfig diff --git a/internal/core/unittest/init_gtest.cpp b/internal/core/unittest/init_gtest.cpp index 4149536929..062a9e865e 100644 --- a/internal/core/unittest/init_gtest.cpp +++ b/internal/core/unittest/init_gtest.cpp @@ -136,6 +136,5 @@ main(int argc, char** argv) { std::chrono::milliseconds(0), std::chrono::milliseconds(-1)); - milvus::index::kOverrideRootPathForUT = "files"; return RUN_ALL_TESTS(); } diff --git a/internal/core/unittest/test_utils/ManifestTestUtil.h b/internal/core/unittest/test_utils/ManifestTestUtil.h index 5485a47cda..01eceb2597 100644 --- a/internal/core/unittest/test_utils/ManifestTestUtil.h +++ b/internal/core/unittest/test_utils/ManifestTestUtil.h @@ -16,6 +16,7 @@ #pragma once +#include "storage/loon_ffi/util.h" #include #include #include @@ -88,9 +89,12 @@ class V3SegmentTestData { int64_t dim, const std::string& root_path, const std::string& base_path) - : schema_(schema), base_path_(base_path), root_path_(root_path) { - std::filesystem::create_directories(std::filesystem::path(root_path) / - std::filesystem::path(base_path)); + : schema_(schema), + // Keys are complete paths: the loon local filesystem is rooted at + // "/" and base_path lives under root_path (see LoonFSRootPath). + base_path_((std::filesystem::path(root_path) / base_path).string()), + root_path_(root_path) { + std::filesystem::create_directories(base_path_); // Convert schema to Arrow schemas auto arrow_schema = schema_->ConvertToArrowSchema(); @@ -118,7 +122,7 @@ class V3SegmentTestData { milvus_storage::api::SetValue( props, PROPERTY_FS_STORAGE_TYPE, LOON_FS_TYPE_LOCAL); milvus_storage::api::SetValue( - props, PROPERTY_FS_ROOT_PATH, root_path.c_str()); + props, PROPERTY_FS_ROOT_PATH, kLoonLocalFSRootPath); milvus_storage::api::SetValue(props, PROPERTY_WRITER_POLICY, LOON_COLUMN_GROUP_POLICY_SCHEMA_BASED); @@ -201,7 +205,7 @@ class V3SegmentTestData { milvus_storage::api::SetValue( reader_props, PROPERTY_FS_STORAGE_TYPE, LOON_FS_TYPE_LOCAL); milvus_storage::api::SetValue( - reader_props, PROPERTY_FS_ROOT_PATH, root_path_.c_str()); + reader_props, PROPERTY_FS_ROOT_PATH, kLoonLocalFSRootPath); auto reader = milvus_storage::api::Reader::create( column_groups_, loon_schema_, needed_columns, reader_props); auto result = reader->get_chunk_reader(cg_index); diff --git a/internal/datacoord/copy_segment_task_test.go b/internal/datacoord/copy_segment_task_test.go index c8fd866074..df7c14b332 100644 --- a/internal/datacoord/copy_segment_task_test.go +++ b/internal/datacoord/copy_segment_task_test.go @@ -1588,7 +1588,7 @@ func (s *CopySegmentTaskSuite) TestSyncCopySegmentTask_IndexWritePlacementMatrix s.True(present) s.Equal(tc.enabled, persistedPlacement) - manifestPath := packed.MarshalManifestPath("files/insert_log/100/10/2001", 3) + manifestPath := packed.MarshalManifestPath("/tmp/test-restart/files/insert_log/100/10/2001", 3) var manifestBuildIDs []int64 if tc.enabled { store.revisions[manifestPath] = []packed.ManifestIndexInfo{{ diff --git a/internal/datacoord/external_collection_refresh_manager.go b/internal/datacoord/external_collection_refresh_manager.go index bae5a220fd..7a4c2b137e 100644 --- a/internal/datacoord/external_collection_refresh_manager.go +++ b/internal/datacoord/external_collection_refresh_manager.go @@ -19,6 +19,7 @@ package datacoord import ( "context" "fmt" + "path" "sync" "time" @@ -71,6 +72,19 @@ func exploreTempDirForAttempt(jobID, attemptID int64) string { return fmt.Sprintf("%s/attempt_%d", exploreTempDirForJob(jobID), attemptID) } +// exploreDirForChunkManager turns an explore temp dir into the complete key +// for the primary chunk manager. On remote storage explore manifests have +// always lived at the bucket root, outside minio.rootPath, so the key is the +// dir itself. On local storage every key is a complete filesystem path, so the +// dir is placed under localStorage.path. Producer (explore) and cleanup must +// agree on this or cleanup silently misses the files. +func (m *externalCollectionRefreshManager) exploreDirForChunkManager(dir string) string { + if _, local := m.chunkManager.(*storage.LocalChunkManager); local { + return path.Join(m.chunkManager.RootPath(), dir) + } + return dir +} + // External Collection Refresh Manager // // The manager is the facade for external collection refresh operations. It encapsulates @@ -317,7 +331,7 @@ func (m *externalCollectionRefreshManager) cleanupExploreTempForJob(jobID int64) if m.chunkManager == nil { return } - exploreBaseDir := exploreTempDirForJob(jobID) + exploreBaseDir := m.exploreDirForChunkManager(exploreTempDirForJob(jobID)) explorePrefix := exploreBaseDir + "/" // Derive from m.ctx so shutdown cancels in-flight cleanup instead of // blocking Stop() on a slow object-store call. @@ -1169,7 +1183,7 @@ func (m *externalCollectionRefreshManager) exploreExternalFiles( if err != nil { return nil, "", merr.Wrap(err, "allocate external refresh Explore attempt ID") } - exploreBaseDir := exploreTempDirForAttempt(job.GetJobId(), attemptID) + exploreBaseDir := m.exploreDirForChunkManager(exploreTempDirForAttempt(job.GetJobId(), attemptID)) fileInfos, manifestPath, err := packed.ExploreFilesReturnManifestPath( columns, spec.Format, diff --git a/internal/datacoord/external_collection_refresh_manager_test.go b/internal/datacoord/external_collection_refresh_manager_test.go index aea773bb38..7ce829b4b9 100644 --- a/internal/datacoord/external_collection_refresh_manager_test.go +++ b/internal/datacoord/external_collection_refresh_manager_test.go @@ -19,6 +19,8 @@ package datacoord import ( "context" "fmt" + "path" + "strings" "sync" "testing" "time" @@ -33,6 +35,7 @@ import ( "github.com/milvus-io/milvus-proto/go-api/v3/schemapb" "github.com/milvus-io/milvus/internal/storage" "github.com/milvus-io/milvus/internal/storagev2/packed" + "github.com/milvus-io/milvus/pkg/v3/objectstorage" "github.com/milvus-io/milvus/pkg/v3/proto/datapb" "github.com/milvus-io/milvus/pkg/v3/proto/indexpb" "github.com/milvus-io/milvus/pkg/v3/util/merr" @@ -1841,6 +1844,8 @@ func TestHandleJobFinished_SourceChangedOnly(t *testing.T) { type recordingChunkManager struct { storage.ChunkManager mu sync.Mutex + rootPath string + objects map[string][]byte prefixCalls []string removeCalls []string prefixErr error @@ -1848,6 +1853,18 @@ type recordingChunkManager struct { prefixBlockUntil <-chan struct{} } +func (r *recordingChunkManager) RootPath() string { return r.rootPath } + +func (r *recordingChunkManager) Write(_ context.Context, key string, value []byte) error { + r.mu.Lock() + defer r.mu.Unlock() + if r.objects == nil { + r.objects = make(map[string][]byte) + } + r.objects[key] = append([]byte(nil), value...) + return nil +} + func (r *recordingChunkManager) RemoveWithPrefix(ctx context.Context, prefix string) error { if r.prefixBlockUntil != nil { select { @@ -1858,6 +1875,13 @@ func (r *recordingChunkManager) RemoveWithPrefix(ctx context.Context, prefix str } r.mu.Lock() r.prefixCalls = append(r.prefixCalls, prefix) + if r.prefixErr == nil { + for key := range r.objects { + if strings.HasPrefix(key, prefix) { + delete(r.objects, key) + } + } + } r.mu.Unlock() return r.prefixErr } @@ -1865,6 +1889,9 @@ func (r *recordingChunkManager) RemoveWithPrefix(ctx context.Context, prefix str func (r *recordingChunkManager) Remove(ctx context.Context, key string) error { r.mu.Lock() r.removeCalls = append(r.removeCalls, key) + if r.removeErr == nil { + delete(r.objects, key) + } r.mu.Unlock() return r.removeErr } @@ -1895,6 +1922,50 @@ func TestCleanupExploreTempForJob_Success(t *testing.T) { assert.Equal(t, []string{"__explore_temp__/coord_42"}, removes) } +func TestCleanupExploreTempForJob_PreservesRemoteProducerNamespace(t *testing.T) { + ctx := context.Background() + cm := &recordingChunkManager{rootPath: "files"} + mgr := newManagerWithChunkManager(t, cm) + // ExploreFilesReturnManifestPath writes baseDir directly through loon's + // bucket-root filesystem. Use that producer's path builder, with a nonempty + // configured root, to verify cleanup deletes the objects actually written. + manifestKey := path.Join(exploreTempDirForAttempt(42, 1), "_metadata/manifest-1.avro") + otherJobKey := path.Join(exploreTempDirForAttempt(420, 1), "_metadata/manifest-1.avro") + prefixedKey := path.Join(cm.RootPath(), manifestKey) + for _, key := range []string{manifestKey, otherJobKey, prefixedKey} { + require.NoError(t, cm.Write(ctx, key, []byte("manifest"))) + } + + mgr.cleanupExploreTempForJob(42) + + prefixes, removes := cm.snapshot() + assert.Equal(t, []string{"__explore_temp__/coord_42/"}, prefixes) + assert.Equal(t, []string{"__explore_temp__/coord_42"}, removes) + assert.NotContains(t, cm.objects, manifestKey) + assert.Contains(t, cm.objects, otherJobKey) + assert.Contains(t, cm.objects, prefixedKey) +} + +func TestCleanupExploreTempForJob_LocalProducerNamespace(t *testing.T) { + ctx := context.Background() + root := t.TempDir() + cm := storage.NewLocalChunkManager(objectstorage.RootPath(root)) + mgr := newManagerWithChunkManager(t, cm) + manifestPath := path.Join(root, exploreTempDirForAttempt(42, 1), "_metadata/manifest-1.avro") + sentinelPath := path.Join(root, exploreTempDirForAttempt(420, 1), "_metadata/manifest-1.avro") + require.NoError(t, cm.Write(ctx, manifestPath, []byte("manifest"))) + require.NoError(t, cm.Write(ctx, sentinelPath, []byte("keep"))) + + mgr.cleanupExploreTempForJob(42) + + exists, err := cm.Exist(ctx, manifestPath) + require.NoError(t, err) + assert.False(t, exists) + sentinel, err := cm.Read(ctx, sentinelPath) + require.NoError(t, err) + assert.Equal(t, []byte("keep"), sentinel) +} + func TestCleanupExploreTempForJob_NilChunkManager(t *testing.T) { mgr := newManagerWithChunkManager(t, nil) diff --git a/internal/datacoord/garbage_collector.go b/internal/datacoord/garbage_collector.go index 0a9008e65f..12aa86887d 100644 --- a/internal/datacoord/garbage_collector.go +++ b/internal/datacoord/garbage_collector.go @@ -20,6 +20,7 @@ import ( "context" "fmt" "path" + "path/filepath" "strconv" "strings" "sync" @@ -611,6 +612,32 @@ func (gc *garbageCollector) recycleUnusedBinlogFiles(ctx context.Context) { segmentIDFromPath: storage.ParseSegmentIDByBinlog, label: metrics.StatFileLabel, }, + { + prefix: path.Join(gc.option.cli.RootPath(), common.SegmentBm25LogPath), + checker: func(objectInfo *storage.ChunkObjectInfo, segment *SegmentInfo) bool { + if segment == nil { + return false + } + for _, fieldLogs := range segment.GetBm25Statslogs() { + for _, statsLog := range fieldLogs.GetBinlogs() { + key := statsLog.GetLogPath() + if key == "" { + // Catalog binlogs carry IDs, not paths. Match the field too: + // compound stats reuse the same log ID across fields. + key = metautil.BuildBm25LogPath(gc.option.cli.RootPath(), + segment.GetCollectionID(), segment.GetPartitionID(), segment.GetID(), + fieldLogs.GetFieldID(), statsLog.GetLogID()) + } + if key == objectInfo.FilePath { + return true + } + } + } + return false + }, + segmentIDFromPath: storage.ParseSegmentIDByBinlog, + label: common.SegmentBm25LogPath, + }, { prefix: path.Join(gc.option.cli.RootPath(), common.SegmentDeltaLogPath), checker: func(objectInfo *storage.ChunkObjectInfo, segment *SegmentInfo) bool { @@ -1159,6 +1186,32 @@ func (gc *garbageCollector) getAllSegmentIndexesForDroppedSegment(segmentID int6 return gc.meta.indexMeta.GetAllSegmentIndexes(segmentID) } +// v3SegmentGCPrefix accepts canonical and legacy namespaces beneath the root, +// but only for this segment. Remote keys retain their literal spelling. +func (gc *garbageCollector) v3SegmentGCPrefix(basePath string, segment *SegmentInfo) (string, error) { + root := path.Clean(gc.option.cli.RootPath()) + _, local := gc.option.cli.(*storage.LocalChunkManager) + if path.IsAbs(basePath) != local { + return "", merr.WrapErrDataIntegrityMsg("GC V3 segment %d has an invalid storage path %q", segment.GetID(), basePath) + } + basePath = strings.TrimRight(basePath, "/") + if local { + basePath = path.Clean(basePath) + rel, err := filepath.Rel(root, basePath) + if err != nil || rel == "." || !filepath.IsLocal(rel) { + return "", merr.WrapErrDataIntegrityMsg("GC V3 segment %d path %q is outside storage root %q", segment.GetID(), basePath, root) + } + } else if root != "." && (basePath == root || !strings.HasPrefix(basePath, root+"/")) { + return "", merr.WrapErrDataIntegrityMsg("GC V3 segment %d path %q is outside storage root %q", segment.GetID(), basePath, root) + } + + if !segmentBaseMatches(basePath, segment.GetCollectionID(), segment.GetPartitionID(), segment.GetID()) { + return "", merr.WrapErrDataIntegrityMsg("GC V3 segment %d manifest base path does not match segment identity", segment.GetID()) + } + // Keep segment 2001 from matching sibling 20010 during prefix deletion. + return basePath + "/", nil +} + func (gc *garbageCollector) removeDroppedSegmentFiles(ctx context.Context, cloned *SegmentInfo, indexFiles map[string]struct{}) error { log := mlog.With(mlog.Int64("segmentID", cloned.GetID())) @@ -1176,7 +1229,11 @@ func (gc *garbageCollector) removeDroppedSegmentFiles(ctx context.Context, clone mlog.Err(err)) return err } - log.Info(ctx, "GC V3 segment start...", + basePath, err = gc.v3SegmentGCPrefix(basePath, cloned) + if err != nil { + return err + } + log.Info(ctx, "GC V3 segment start, removing basePath...", mlog.String("basePath", basePath), mlog.Int("indexFiles", len(indexFiles))) if err := gc.removeObjectFiles(ctx, indexFiles); err != nil { diff --git a/internal/datacoord/garbage_collector_bm25_test.go b/internal/datacoord/garbage_collector_bm25_test.go new file mode 100644 index 0000000000..69d9043c34 --- /dev/null +++ b/internal/datacoord/garbage_collector_bm25_test.go @@ -0,0 +1,163 @@ +// Licensed to the LF AI & Data foundation under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package datacoord + +import ( + "context" + "os" + "path" + "strings" + "testing" + "time" + + "github.com/bytedance/mockey" + "github.com/cockroachdb/errors" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + + "github.com/milvus-io/milvus-proto/go-api/v3/commonpb" + "github.com/milvus-io/milvus/internal/mocks" + "github.com/milvus-io/milvus/internal/storage" + "github.com/milvus-io/milvus/pkg/v3/objectstorage" + "github.com/milvus-io/milvus/pkg/v3/proto/datapb" + "github.com/milvus-io/milvus/pkg/v3/util/conc" +) + +func TestGarbageCollectorBM25OrphanFiles(t *testing.T) { + for _, local := range []bool{true, false} { + name := "remote complete keys" + if local { + name = "local filesystem" + } + t.Run(name, func(t *testing.T) { + ctx := context.Background() + root := "files" + if local { + root = t.TempDir() + } + key := func(suffix string) string { return path.Join(root, "bm25_stats", suffix) } + cases := []struct { + suffix string + keep bool + fresh bool + }{ + {"100/10/200/101/1000", true, false}, // compressed metadata + {"100/10/200/101/1", true, false}, // compound stats + {"100/10/200/102/1", false, false}, // same LogID, different field + {"100/10/200/101/999", false, false}, // superseded stats + {"100/10/200/101/777", true, false}, // explicit LogPath overrides LogID + {"100/10/200/101/1002", false, false}, + {"101/10/200/101/1000", false, false}, // same segment/log IDs, wrong collection + {"100/11/200/101/1000", false, false}, // same segment/log IDs, wrong partition + {"100/10/201/101/1000", false, false}, // absent segment + {"100/10/202/101/1000", true, true}, // fresh orphan + {"100/10/205/101/1000", true, false}, // snapshot-protected absent segment + {"100/10/300/101/999", true, false}, // registered V3 skipped + } + segment := &SegmentInfo{SegmentInfo: &datapb.SegmentInfo{ + ID: 200, CollectionID: 100, PartitionID: 10, + State: commonpb.SegmentState_Flushed, + Bm25Statslogs: []*datapb.FieldBinlog{{FieldID: 101, Binlogs: []*datapb.Binlog{ + {LogID: 1000}, {LogID: 1}, {LogID: 1002, LogPath: key("100/10/200/101/777")}, + }}}, + }} + mt := &meta{segments: &SegmentsInfo{segments: map[int64]*SegmentInfo{ + 200: segment, + 300: {SegmentInfo: &datapb.SegmentInfo{ID: 300, CollectionID: 100, PartitionID: 10, StorageVersion: storage.StorageV3}}, + }}, snapshotMeta: &snapshotMeta{}} + blocked := mockey.Mock((*snapshotMeta).IsSegmentGCBlocked).To( + func(_ *snapshotMeta, _ int64, segmentID int64) bool { return segmentID == 205 }, + ).Build() + defer blocked.UnPatch() + pool := conc.NewPool[struct{}](1) + t.Cleanup(pool.Release) + gc := &garbageCollector{meta: mt, option: GcOption{missingTolerance: time.Hour, removeObjectPool: pool}} + old := time.Now().Add(-2 * time.Hour) + if local { + cm := storage.NewLocalChunkManager(objectstorage.RootPath(root)) + gc.option.cli = cm + for _, tc := range cases { + file := key(tc.suffix) + require.NoError(t, cm.Write(ctx, file, []byte(tc.suffix))) + if !tc.fresh { + require.NoError(t, os.Chtimes(file, old, old)) + } + } + gc.recycleUnusedBinlogFiles(ctx) + for _, tc := range cases { + exists, err := cm.Exist(ctx, key(tc.suffix)) + require.NoError(t, err) + assert.Equal(t, tc.keep, exists, tc.suffix) + } + // GC must not decompress or otherwise rewrite authoritative metadata. + assert.Empty(t, segment.GetBm25Statslogs()[0].GetBinlogs()[0].GetLogPath()) + return + } + + cm := mocks.NewChunkManager(t) + gc.option.cli = cm + cm.EXPECT().RootPath().Return(root) + files := make(map[string]*storage.ChunkObjectInfo) + for _, tc := range cases { + modified := old + if tc.fresh { + modified = time.Now() + } + files[key(tc.suffix)] = &storage.ChunkObjectInfo{FilePath: key(tc.suffix), ModifyTime: modified} + } + cm.EXPECT().WalkWithPrefix(mock.Anything, mock.Anything, true, mock.Anything).RunAndReturn( + func(_ context.Context, prefix string, _ bool, walk storage.ChunkObjectWalkFunc) error { + // Snapshot the listing: the deletion worker may run concurrently. + listed := make([]*storage.ChunkObjectInfo, 0) + for file, info := range files { + if strings.HasPrefix(file, strings.TrimSuffix(prefix, "/")+"/") { + listed = append(listed, info) + } + } + for _, info := range listed { + if !walk(info) { + break + } + } + return nil + }) + retryFile := key("100/10/201/101/1000") + cm.EXPECT().Remove(mock.Anything, retryFile).Return(errors.New("temporary delete failure")).Once() + for _, tc := range cases { + if tc.keep || key(tc.suffix) == retryFile { + continue + } + cm.EXPECT().Remove(mock.Anything, key(tc.suffix)).RunAndReturn(func(_ context.Context, file string) error { + delete(files, file) + return nil + }).Once() + } + gc.recycleUnusedBinlogFiles(ctx) + require.Contains(t, files, retryFile) + cm.EXPECT().Remove(mock.Anything, retryFile).RunAndReturn(func(_ context.Context, file string) error { + delete(files, file) + return nil + }).Once() + gc.recycleUnusedBinlogFiles(ctx) + for _, tc := range cases { + _, exists := files[key(tc.suffix)] + assert.Equal(t, tc.keep, exists, tc.suffix) + } + }) + } +} diff --git a/internal/datacoord/garbage_collector_lob.go b/internal/datacoord/garbage_collector_lob.go index fb40ca1714..b03ba716c2 100644 --- a/internal/datacoord/garbage_collector_lob.go +++ b/internal/datacoord/garbage_collector_lob.go @@ -33,7 +33,6 @@ import ( "github.com/milvus-io/milvus/pkg/v3/proto/indexpb" "github.com/milvus-io/milvus/pkg/v3/util/conc" "github.com/milvus-io/milvus/pkg/v3/util/merr" - "github.com/milvus-io/milvus/pkg/v3/util/paramtable" "github.com/milvus-io/milvus/pkg/v3/util/typeutil" ) @@ -154,14 +153,10 @@ func (gc *garbageCollector) recycleUnusedLOBFiles(ctx context.Context) { logger.Info(ctx, "recycleUnusedLOBFiles done", mlog.Duration("timeCost", time.Since(start))) }() - storageConfig := gc.getStorageConfig() - if storageConfig == nil { - logger.Warn(ctx, "failed to get storage config, skip LOB GC") - return - } - lobCtx := newLOBGCContext(gc) - lobCtx.storageConfig = storageConfig + // Same builder as every other primary-storage caller: under local storage + // the key prefix is localStorage.path, not minio.rootPath. + lobCtx.storageConfig = createStorageConfig() // Step 1: Collect all used LOB files from active segments. // If any manifest read fails, abort the entire GC run to avoid @@ -187,30 +182,6 @@ func (gc *garbageCollector) recycleUnusedLOBFiles(ctx context.Context) { lobCtx.cache.Cleanup() } -// getStorageConfig returns the storage configuration for FFI calls -func (gc *garbageCollector) getStorageConfig() *indexpb.StorageConfig { - params := paramtable.Get() - - return &indexpb.StorageConfig{ - Address: params.MinioCfg.Address.GetValue(), - AccessKeyID: params.MinioCfg.AccessKeyID.GetValue(), - SecretAccessKey: params.MinioCfg.SecretAccessKey.GetValue(), - UseSSL: params.MinioCfg.UseSSL.GetAsBool(), - SslCACert: params.MinioCfg.SslCACert.GetValue(), - BucketName: params.MinioCfg.BucketName.GetValue(), - RootPath: params.MinioCfg.RootPath.GetValue(), - UseIAM: params.MinioCfg.UseIAM.GetAsBool(), - IAMEndpoint: params.MinioCfg.IAMEndpoint.GetValue(), - StorageType: params.CommonCfg.StorageType.GetValue(), - Region: params.MinioCfg.Region.GetValue(), - UseVirtualHost: params.MinioCfg.UseVirtualHost.GetAsBool(), - CloudProvider: params.MinioCfg.CloudProvider.GetValue(), - RequestTimeoutMs: params.MinioCfg.RequestTimeoutMs.GetAsInt64(), - MaxConnections: uint32(params.MinioCfg.MaxConnections.GetAsInt()), - GcpCredentialJSON: params.MinioCfg.GcpCredentialJSON.GetValue(), - } -} - // collectUsedLOBFiles collects all LOB file paths that are referenced by active segments // and by dropped segments that are protected by snapshots. // Returns a set of LOB file paths (relative paths within LOB directory). @@ -290,6 +261,10 @@ func (lobCtx *lobGCContext) scanOrphanLOBFiles(ctx context.Context, usedFiles ty safetyWindow := Params.DataCoordCfg.GCLOBSafetyWindow.GetAsDuration(time.Second) // LOB files are stored at: {root_path}/insert_log/{coll}/{part}/lobs/{field_id}/_data/{file_id}.vx + // TODO: Cover the legacy local namespace when upgrading from 3.0.0/3.0.1 + // with LOB enabled. Files under localStorage.path/minio.rootPath/insert_log + // remain readable in place, but this scan misses their partition-level LOBs, + // so unreferenced LOB files there are not reclaimed. lobBasePath := path.Join(lobCtx.gc.option.cli.RootPath(), common.SegmentInsertLogPath) // Walk through all files under insert_log to find LOB files diff --git a/internal/datacoord/garbage_collector_lob_test.go b/internal/datacoord/garbage_collector_lob_test.go index 26d3085aca..2dca3679c9 100644 --- a/internal/datacoord/garbage_collector_lob_test.go +++ b/internal/datacoord/garbage_collector_lob_test.go @@ -26,7 +26,6 @@ import ( "github.com/milvus-io/milvus/internal/storagev2/packed" "github.com/milvus-io/milvus/pkg/v3/proto/datapb" - "github.com/milvus-io/milvus/pkg/v3/proto/indexpb" "github.com/milvus-io/milvus/pkg/v3/util/typeutil" ) @@ -271,15 +270,30 @@ func TestNewLOBGCContext(t *testing.T) { assert.Equal(t, gc, lobCtx.gc) } -func TestGetStorageConfig(t *testing.T) { - // this test verifies the getStorageConfig function returns a valid config - // based on paramtable values - gc := &garbageCollector{} - config := gc.getStorageConfig() +// LOB GC reads manifests through the primary storage config, so its key prefix +// must be localStorage.path under local storage. Deriving it from minio.rootPath +// would address a namespace that holds no LOB files (#53051). +func TestLOBGCStorageConfigUsesPrimaryStorageRoot(t *testing.T) { + params := Params + localRoot := t.TempDir() + require.NoError(t, params.Save(params.MinioCfg.RootPath.Key, "files")) + require.NoError(t, params.Save(params.LocalStorageCfg.Path.Key, localRoot)) + t.Cleanup(func() { + _ = params.Reset(params.CommonCfg.StorageType.Key) + _ = params.Reset(params.MinioCfg.RootPath.Key) + _ = params.Reset(params.LocalStorageCfg.Path.Key) + }) + require.NoError(t, params.Save(params.CommonCfg.StorageType.Key, "local")) + config := createStorageConfig() require.NotNil(t, config) - // verify it's a valid StorageConfig struct - assert.IsType(t, &indexpb.StorageConfig{}, config) + assert.Equal(t, "local", config.GetStorageType()) + assert.Equal(t, localRoot, config.GetRootPath()) + + require.NoError(t, params.Save(params.CommonCfg.StorageType.Key, "remote")) + config = createStorageConfig() + require.NotNil(t, config) + assert.Equal(t, "files", config.GetRootPath()) } func TestCollectLOBFilesFromSegment(t *testing.T) { diff --git a/internal/datacoord/garbage_collector_test.go b/internal/datacoord/garbage_collector_test.go index e0678bf5d5..220da5ea87 100644 --- a/internal/datacoord/garbage_collector_test.go +++ b/internal/datacoord/garbage_collector_test.go @@ -23,6 +23,7 @@ import ( "math/rand" "os" "path" + "path/filepath" "strconv" "strings" "sync" @@ -3089,7 +3090,7 @@ func TestGarbageCollector_getDroppedSegmentIndexFiles_ManifestUnreadable(t *test func TestGarbageCollector_removeDroppedSegmentFilesV3(t *testing.T) { ctx := context.Background() - basePath := "/tmp/test-gc-v3/insert_log/100/10/2001" + basePath := "root/insert_log/100/10/2001" segment := &SegmentInfo{ SegmentInfo: &datapb.SegmentInfo{ ID: 2001, @@ -3103,6 +3104,7 @@ func TestGarbageCollector_removeDroppedSegmentFilesV3(t *testing.T) { t.Run("success with index files", func(t *testing.T) { cm := mocks.NewChunkManager(t) + cm.EXPECT().RootPath().Return("root").Maybe() cm.EXPECT().RemoveWithPrefix(mock.Anything, basePath+"/").Return(nil).Once() cm.EXPECT().Remove(mock.Anything, "root/index_files/40/1/10/2001/idx-file").Return(nil).Once() gc := newGarbageCollector(nil, nil, GcOption{cli: cm}) @@ -3124,14 +3126,25 @@ func TestGarbageCollector_removeDroppedSegmentFilesV3(t *testing.T) { t.Run("remove base path failed", func(t *testing.T) { cm := mocks.NewChunkManager(t) + cm.EXPECT().RootPath().Return("root").Maybe() cm.EXPECT().RemoveWithPrefix(mock.Anything, basePath+"/").Return(errors.New("remove failed")).Once() gc := newGarbageCollector(nil, nil, GcOption{cli: cm}) assert.Error(t, gc.removeDroppedSegmentFiles(ctx, segment, nil)) }) + t.Run("remote not found semantics unchanged", func(t *testing.T) { + cm := mocks.NewChunkManager(t) + cm.EXPECT().RootPath().Return("root").Maybe() + cm.EXPECT().RemoveWithPrefix(mock.Anything, basePath+"/").Return(os.ErrNotExist).Once() + gc := newGarbageCollector(nil, nil, GcOption{cli: cm}) + + assert.ErrorIs(t, gc.removeDroppedSegmentFiles(ctx, segment, nil), os.ErrNotExist) + }) + t.Run("remove index file failed", func(t *testing.T) { cm := mocks.NewChunkManager(t) + cm.EXPECT().RootPath().Return("root").Maybe() // No RemoveWithPrefix expectation: a failed index-file delete must // leave basePath - and the manifest inside it, the only thing naming // the surviving index files - untouched for the next cycle. @@ -3151,6 +3164,7 @@ func TestGarbageCollector_removeDroppedSegmentFilesV3(t *testing.T) { var mu sync.Mutex var calls []string cm := mocks.NewChunkManager(t) + cm.EXPECT().RootPath().Return("root").Maybe() cm.EXPECT().Remove(mock.Anything, mock.Anything).RunAndReturn(func(_ context.Context, file string) error { mu.Lock() defer mu.Unlock() @@ -4408,7 +4422,7 @@ func Test_parseV3SegmentID(t *testing.T) { func TestGarbageCollector_recycleDroppedSegments_V3(t *testing.T) { ctx := context.Background() - cli := storage.NewLocalChunkManager(objectstorage.RootPath("/tmp/test-gc-v3")) + cli := storage.NewLocalChunkManager(objectstorage.RootPath(t.TempDir())) catalog := &datacoord.Catalog{} smMeta := &snapshotMeta{} @@ -4422,7 +4436,9 @@ func TestGarbageCollector_recycleDroppedSegments_V3(t *testing.T) { channelCPs: newChannelCps(), } - basePath := "/tmp/test-gc-v3/insert_log/100/10/2001" + basePath := path.Join(cli.RootPath(), "insert_log/100/10/2001") + require.NoError(t, os.MkdirAll(basePath, 0o755)) + completeBasePath := basePath + "/" manifestPath := packed.MarshalManifestPath(basePath, 1) // V3 dropped segment with ManifestPath and StorageVersion=3 @@ -4509,7 +4525,7 @@ func TestGarbageCollector_recycleDroppedSegments_V3(t *testing.T) { // V3 segment should use RemoveWithPrefix with basePath assert.True(t, removeWithPrefixCalled, "V3 segment should use RemoveWithPrefix") - assert.Equal(t, basePath+"/", removeWithPrefixArg, "RemoveWithPrefix should name the segment directory") + assert.Equal(t, completeBasePath, removeWithPrefixArg, "RemoveWithPrefix should be called with a complete local path") // V1 segment should use removeObjectFiles assert.True(t, removeObjectFilesCalled, "V1 segment should use removeObjectFiles") @@ -4521,6 +4537,304 @@ func TestGarbageCollector_recycleDroppedSegments_V3(t *testing.T) { assert.Nil(t, m.GetSegment(ctx, 2002)) } +func TestGarbageCollectorRemoveDroppedSegmentFilesRejectsUnsafeLocalManifestRoot(t *testing.T) { + ctx := context.Background() + root := t.TempDir() + cli := storage.NewLocalChunkManager(objectstorage.RootPath(root)) + gc := &garbageCollector{option: GcOption{cli: cli}} + sentinelPath := path.Join(root, "sentinel") + require.NoError(t, cli.Write(ctx, sentinelPath, []byte("keep"))) + + unsafeBasePaths := []string{ + "", + ".", + "/", + "../outside", + root, + "s3://bucket/files/insert_log/100/10/2001", + "insert_log", + "insert_log/100/10/20010", + } + for _, basePath := range unsafeBasePaths { + t.Run(fmt.Sprintf("base_%q", basePath), func(t *testing.T) { + segment := &SegmentInfo{SegmentInfo: &datapb.SegmentInfo{ + ID: 2001, + StorageVersion: storage.StorageV3, + ManifestPath: packed.MarshalManifestPath(basePath, 1), + }} + err := gc.removeDroppedSegmentFiles(ctx, segment, nil) + require.Error(t, err) + content, readErr := cli.Read(ctx, sentinelPath) + require.NoError(t, readErr) + assert.Equal(t, []byte("keep"), content) + }) + } +} + +func TestGarbageCollectorRemoveDroppedSegmentFilesUsesSegmentBoundary(t *testing.T) { + ctx := context.Background() + root := t.TempDir() + cli := storage.NewLocalChunkManager(objectstorage.RootPath(root)) + gc := &garbageCollector{option: GcOption{cli: cli}} + segmentFile := path.Join(root, "files/insert_log/100/10/2001/data") + siblingFile := path.Join(root, "files/insert_log/100/10/20010/data") + require.NoError(t, cli.Write(ctx, segmentFile, []byte("remove"))) + require.NoError(t, cli.Write(ctx, siblingFile, []byte("keep"))) + + segment := &SegmentInfo{SegmentInfo: &datapb.SegmentInfo{ + ID: 2001, + CollectionID: 100, + PartitionID: 10, + StorageVersion: storage.StorageV3, + ManifestPath: packed.MarshalManifestPath(path.Join(root, "files/insert_log/100/10/2001"), 1), + }} + require.NoError(t, gc.removeDroppedSegmentFiles(ctx, segment, nil)) + + segmentExists, err := cli.Exist(ctx, segmentFile) + require.NoError(t, err) + assert.False(t, segmentExists) + siblingContent, err := cli.Read(ctx, siblingFile) + require.NoError(t, err) + assert.Equal(t, []byte("keep"), siblingContent) +} + +func TestGarbageCollectorRemoveDroppedSegmentFilesRejectsForeignLocalRoot(t *testing.T) { + ctx := context.Background() + testRoot := t.TempDir() + root := path.Join(testRoot, "owned") + cli := storage.NewLocalChunkManager(objectstorage.RootPath(root)) + gc := &garbageCollector{option: GcOption{cli: cli}} + for _, otherRoot := range []string{path.Join(testRoot, "other"), root + "-other"} { + t.Run(path.Base(otherRoot), func(t *testing.T) { + basePath := path.Join(otherRoot, "insert_log/100/10/2001") + sentinelPath := path.Join(basePath, "data") + require.NoError(t, cli.Write(ctx, sentinelPath, []byte("keep foreign data"))) + segment := &SegmentInfo{SegmentInfo: &datapb.SegmentInfo{ + ID: 2001, CollectionID: 100, PartitionID: 10, + StorageVersion: storage.StorageV3, + ManifestPath: packed.MarshalManifestPath(basePath, 1), + }} + require.ErrorIs(t, gc.removeDroppedSegmentFiles(ctx, segment, nil), merr.ErrDataIntegrity) + content, err := cli.Read(ctx, sentinelPath) + require.NoError(t, err) + assert.Equal(t, "keep foreign data", string(content)) + }) + } +} + +func TestGarbageCollectorRemoveDroppedSegmentFilesLocalFilesystemRoot(t *testing.T) { + ctx := context.Background() + // Even with a filesystem-wide configured root, only test-owned files are + // created and deleted; the manifest's segment suffix remains mandatory. + testRoot, err := filepath.EvalSymlinks(t.TempDir()) + require.NoError(t, err) + basePath := path.Join(testRoot, "insert_log/100/10/2001") + cli := storage.NewLocalChunkManager(objectstorage.RootPath("/")) + gc := &garbageCollector{option: GcOption{cli: cli}} + filePath := path.Join(basePath, "data") + require.NoError(t, cli.Write(ctx, filePath, []byte("remove"))) + segment := &SegmentInfo{SegmentInfo: &datapb.SegmentInfo{ + ID: 2001, CollectionID: 100, PartitionID: 10, + StorageVersion: storage.StorageV3, + ManifestPath: packed.MarshalManifestPath(basePath, 1), + }} + require.NoError(t, gc.removeDroppedSegmentFiles(ctx, segment, nil)) + assert.NoFileExists(t, filePath) +} + +func TestGarbageCollectorRemoveDroppedSegmentFilesAllowsMissingLocalSuffix(t *testing.T) { + for _, existingSuffix := range []string{"", "insert_log/100/10"} { + t.Run(fmt.Sprintf("existing_%q", existingSuffix), func(t *testing.T) { + ctx := context.Background() + root := t.TempDir() + cli := storage.NewLocalChunkManager(objectstorage.RootPath(root)) + gc := newGarbageCollector(nil, nil, GcOption{cli: cli}) + require.NoError(t, os.MkdirAll(path.Join(root, existingSuffix), 0o755)) + basePath := path.Join(root, "insert_log/100/10/2001") + segment := &SegmentInfo{SegmentInfo: &datapb.SegmentInfo{ + ID: 2001, CollectionID: 100, PartitionID: 10, + StorageVersion: storage.StorageV3, + ManifestPath: packed.MarshalManifestPath(basePath, 1), + }} + indexFile := path.Join(root, "index_files/40/1/10/2001/idx-file") + require.NoError(t, cli.Write(ctx, indexFile, []byte("remove index"))) + segment.ID = 20010 + require.ErrorIs(t, gc.removeDroppedSegmentFiles(ctx, segment, map[string]struct{}{indexFile: {}}), merr.ErrDataIntegrity) + require.FileExists(t, indexFile, "a missing base must not bypass segment identity validation") + segment.ID = 2001 + for range 2 { + // Exercise the real local prefix walker: a missing base returns + // ENOENT unless GC recognizes that data removal is already done. + require.NoError(t, gc.removeDroppedSegmentFiles(ctx, segment, map[string]struct{}{indexFile: {}})) + assert.NoFileExists(t, indexFile) + assert.NoDirExists(t, basePath) + } + }) + } +} + +func TestGarbageCollector_recycleDroppedSegment_MissingLocalV3DataStillRemovesMeta(t *testing.T) { + ctx := context.Background() + m, segment, _, _ := setupDroppedSegmentWithIndexForGC(t) + root := t.TempDir() + cli := storage.NewLocalChunkManager(objectstorage.RootPath(root)) + basePath := path.Join(root, "files/insert_log/100/10/1001") + segment.StorageVersion = storage.StorageV3 + segment.ManifestPath = packed.MarshalManifestPath(basePath, 1) + gc := newGarbageCollector(m, newMockHandler(), GcOption{cli: cli}) + _, indexFiles, blocked := gc.getDroppedSegmentIndexFiles(ctx, segment.ID) + require.Equal(t, gcNotBlocked, blocked) + require.NotEmpty(t, indexFiles) + for file := range indexFiles { + require.NoError(t, cli.Write(ctx, file, []byte("remaining index"))) + } + sibling := path.Join(root, "files/insert_log/100/10/10010/data") + require.NoError(t, cli.Write(ctx, sibling, []byte("keep sibling"))) + require.NoDirExists(t, basePath) + + gc.recycleDroppedSegment(ctx, segment.ID, segment) + + assert.Nil(t, m.GetSegment(ctx, segment.ID)) + assert.Empty(t, m.indexMeta.GetAllSegmentIndexes(segment.ID)) + for file := range indexFiles { + assert.NoFileExists(t, file) + } + contents, err := cli.Read(ctx, sibling) + require.NoError(t, err) + assert.Equal(t, "keep sibling", string(contents)) +} + +func TestGarbageCollectorRemoveDroppedSegmentFilesRejectsInvalidConfiguredRoot(t *testing.T) { + segment := &SegmentInfo{SegmentInfo: &datapb.SegmentInfo{ + ID: 2001, CollectionID: 100, PartitionID: 10, + StorageVersion: storage.StorageV3, + }} + t.Run("relative local root", func(t *testing.T) { + cli := storage.NewLocalChunkManager(objectstorage.RootPath("data")) + gc := &garbageCollector{option: GcOption{cli: cli}} + segment.ManifestPath = packed.MarshalManifestPath(path.Join(t.TempDir(), "insert_log/100/10/2001"), 1) + require.ErrorIs(t, gc.removeDroppedSegmentFiles(context.Background(), segment, nil), merr.ErrDataIntegrity) + }) + for _, root := range []string{"/", "/files", "s3://bucket/files"} { + t.Run(fmt.Sprintf("remote_root_%q", root), func(t *testing.T) { + cli := &gcRootedChunkManager{root: root} + gc := &garbageCollector{option: GcOption{cli: cli}} + segment.ManifestPath = packed.MarshalManifestPath("files/insert_log/100/10/2001", 1) + require.ErrorIs(t, gc.removeDroppedSegmentFiles(context.Background(), segment, nil), merr.ErrDataIntegrity) + assert.False(t, cli.removeCalled) + }) + } +} + +type gcRootedChunkManager struct { + storage.ChunkManager + root string + removeCalled bool + removedPrefix string +} + +func (cm *gcRootedChunkManager) RootPath() string { + return cm.root +} + +func (cm *gcRootedChunkManager) RemoveWithPrefix(_ context.Context, prefix string) error { + cm.removeCalled = true + cm.removedPrefix = prefix + return nil +} + +// A manifest must identify both the configured namespace and this segment; +// matching IDs alone do not authorize deletion in another instance's prefix. +func TestGarbageCollectorRemoveDroppedSegmentFilesRejectsMalformedRemoteManifestBase(t *testing.T) { + unsafeBasePaths := []string{ + "files/insert_log", + "files/insert_log/100/10/20010", + "/files/insert_log/100/10/2001", + "s3://bucket/files/insert_log/100/10/2001", + "../insert_log/100/10/2001", + "other/insert_log/100/10/2001", + "files-other/insert_log/100/10/2001", + "insert_log/100/10/2001", + } + for _, basePath := range unsafeBasePaths { + t.Run(fmt.Sprintf("base_%q", basePath), func(t *testing.T) { + cli := &gcRootedChunkManager{root: "files"} + gc := &garbageCollector{option: GcOption{cli: cli}} + segment := &SegmentInfo{SegmentInfo: &datapb.SegmentInfo{ + ID: 2001, + CollectionID: 100, + PartitionID: 10, + StorageVersion: storage.StorageV3, + ManifestPath: packed.MarshalManifestPath(basePath, 1), + }} + err := gc.removeDroppedSegmentFiles(context.Background(), segment, nil) + require.ErrorIs(t, err, merr.ErrDataIntegrity) + assert.False(t, cli.removeCalled) + }) + } +} + +func TestGarbageCollectorRemoveDroppedSegmentFilesUsesCompleteRemoteKey(t *testing.T) { + for _, tc := range []struct { + name string + root string + base string + }{ + {"configured prefix", "files", "files/insert_log/100/10/2001"}, + {"nested prefix", "tenant/files", "tenant/files/insert_log/100/10/2001"}, + {"layout prefix within root", "files", "files/legacy/insert_log/100/10/2001"}, + {"bucket root complete key", "", "files/insert_log/100/10/2001"}, + {"dot bucket root complete key", ".", "files/insert_log/100/10/2001"}, + {"parent component is part of object key", "files", "files/tmp/../insert_log/100/10/2001"}, + {"dot component is part of object key", "files", "files/./insert_log/100/10/2001"}, + {"repeated separator is part of object key", "files", "files//insert_log/100/10/2001"}, + {"trailing directory separator", "files", "files/insert_log/100/10/2001/"}, + {"parent component at bucket root", "", "../insert_log/100/10/2001"}, + {"parent component within bucket", ".", "tmp/../insert_log/100/10/2001"}, + } { + t.Run(tc.name, func(t *testing.T) { + cli := &gcRootedChunkManager{root: tc.root} + gc := &garbageCollector{option: GcOption{cli: cli}} + segment := &SegmentInfo{SegmentInfo: &datapb.SegmentInfo{ + ID: 2001, + CollectionID: 100, + PartitionID: 10, + StorageVersion: storage.StorageV3, + ManifestPath: packed.MarshalManifestPath(tc.base, 1), + }} + + require.NoError(t, gc.removeDroppedSegmentFiles(context.Background(), segment, nil)) + assert.True(t, cli.removeCalled) + assert.Equal(t, strings.TrimRight(tc.base, "/")+"/", cli.removedPrefix) + }) + } +} + +func TestGarbageCollectorRemoveDroppedSegmentFilesRemoteBucketRoot(t *testing.T) { + for _, root := range []string{"", "."} { + t.Run(fmt.Sprintf("root_%q", root), func(t *testing.T) { + cli := &gcRootedChunkManager{root: root} + gc := &garbageCollector{option: GcOption{cli: cli}} + segment := &SegmentInfo{SegmentInfo: &datapb.SegmentInfo{ + ID: 2001, + CollectionID: 100, + PartitionID: 10, + StorageVersion: storage.StorageV3, + ManifestPath: packed.MarshalManifestPath(path.Join(root, "insert_log/100/10/2001"), 1), + }} + require.NoError(t, gc.removeDroppedSegmentFiles(context.Background(), segment, nil)) + assert.Equal(t, "insert_log/100/10/2001/", cli.removedPrefix) + + for _, unsafeBase := range []string{"", ".", "..", "insert_log/100/10/20010"} { + cli.removeCalled = false + segment.ManifestPath = packed.MarshalManifestPath(unsafeBase, 1) + require.Error(t, gc.removeDroppedSegmentFiles(context.Background(), segment, nil)) + assert.False(t, cli.removeCalled) + } + }) + } +} + func TestGarbageCollector_recycleUnusedBinlogFiles_SkipV3(t *testing.T) { ctx := context.Background() diff --git a/internal/datacoord/manifest_index.go b/internal/datacoord/manifest_index.go index 182bc23fa3..9f3811ce47 100644 --- a/internal/datacoord/manifest_index.go +++ b/internal/datacoord/manifest_index.go @@ -312,8 +312,12 @@ func (m *meta) reloadSegmentIndexesFromManifests(ctx context.Context) error { return nil, nil } } - return nil, merr.Wrapf(err, "recover segment %d indexes from manifest %s", - segment.GetID(), segment.GetManifestPath()) + // Do not let initMeta replay the entire metastore load after the + // per-segment retries are exhausted. A second newMeta attempt would + // repeat every catalog read and can expose a partially initialized + // state; the caller can retry the startup as a whole instead. + return nil, retry.Unrecoverable(merr.Wrapf(err, + "recover segment %d indexes from manifest %s", segment.GetID(), segment.GetManifestPath())) } for _, entry := range entries { if _, ok := manifestIndexFilePathInfoForSegment(rootPath, segment.SegmentInfo, entry); !ok { diff --git a/internal/datacoord/manifest_index_review_test.go b/internal/datacoord/manifest_index_review_test.go index 9ca76b9c8d..e0283328c0 100644 --- a/internal/datacoord/manifest_index_review_test.go +++ b/internal/datacoord/manifest_index_review_test.go @@ -104,12 +104,13 @@ func TestInitMetaDoesNotReplayManifestScan(t *testing.T) { require.NoError(t, m.AddSegment(ctx, NewSegmentInfo(&datapb.SegmentInfo{ CollectionID: 100, PartitionID: 10, ID: id, State: commonpb.SegmentState_Flushed, StorageVersion: storage.StorageV3, ManifestHasIndex: true, - ManifestPath: packed.MarshalManifestPath(metautil.JoinIDPath(id), 1), + ManifestPath: packed.MarshalManifestPath(path.Join("/tmp/test-restart", "insert_log", metautil.JoinIDPath(100, 10, id)), 1), }))) } var mu sync.Mutex reads := make(map[string]int) - bad := packed.MarshalManifestPath("5002", 1) + good := packed.MarshalManifestPath(path.Join("/tmp/test-restart", "insert_log", metautil.JoinIDPath(100, 10, 5001)), 1) + bad := packed.MarshalManifestPath(path.Join("/tmp/test-restart", "insert_log", metautil.JoinIDPath(100, 10, 5002)), 1) reader := mockey.Mock(packed.GetManifestIndexInfos).To(func(pointer string, _ *indexpb.StorageConfig) ([]packed.ManifestIndexInfo, error) { mu.Lock() defer mu.Unlock() @@ -133,7 +134,7 @@ func TestInitMetaDoesNotReplayManifestScan(t *testing.T) { b.EXPECT().ListDatabases(mock.Anything).Run(func(context.Context) { close(collectionsLoaded) }).Return(nil, nil).Maybe() server := &Server{ctx: ctx, kv: kv, broker: b} err := server.initMeta(cm) - require.Equal(t, 1, reads[packed.MarshalManifestPath("5001", 1)]) + require.Equal(t, 1, reads[good]) if mode == "transient" { require.NoError(t, err) <-collectionsLoaded diff --git a/internal/datacoord/meta.go b/internal/datacoord/meta.go index bf32f62a33..10f0386e7d 100644 --- a/internal/datacoord/meta.go +++ b/internal/datacoord/meta.go @@ -413,9 +413,23 @@ func (m *meta) reloadFromKV(ctx context.Context, collectionIDs []int64) error { metrics.DataCoordNumSegments.Reset() numStoredRows := int64(0) numSegments := 0 + local, localStorage := m.chunkManager.(*storage.LocalChunkManager) for _, segments := range collectionSegments { numSegments += len(segments) for _, segment := range segments { + if localStorage && segment.GetStorageVersion() == storage.StorageV3 { + manifestPath, err := normalizeLocalManifestPath(segment.GetManifestPath(), local.RootPath(), + Params.MinioCfg.RootPath.GetValue(), segment.GetCollectionID(), segment.GetPartitionID(), segment.GetID()) + if err != nil { + return err + } + if manifestPath != segment.GetManifestPath() { + // Resolve only the loaded view. Startup does not write the + // catalog; a later ordinary update can persist the new path. + segment = proto.Clone(segment).(*datapb.SegmentInfo) + segment.ManifestPath = manifestPath + } + } // segments from catalog.ListSegments will not have logPath m.segments.SetSegment(segment.ID, NewSegmentInfo(segment)) metrics.DataCoordNumSegments.WithLabelValues(segmentMetricLabelValues(NewSegmentInfo(segment))...).Inc() diff --git a/internal/datacoord/meta_local_manifest.go b/internal/datacoord/meta_local_manifest.go new file mode 100644 index 0000000000..d9e1b96122 --- /dev/null +++ b/internal/datacoord/meta_local_manifest.go @@ -0,0 +1,58 @@ +// Licensed to the LF AI & Data foundation under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package datacoord + +import ( + "path" + "path/filepath" + + "github.com/milvus-io/milvus/internal/storagev2/packed" + "github.com/milvus-io/milvus/pkg/v3/common" + "github.com/milvus-io/milvus/pkg/v3/util/merr" + "github.com/milvus-io/milvus/pkg/v3/util/metautil" +) + +// normalizeLocalManifestPath resolves legacy minio-root-relative manifests to +// root/legacyPrefix/insert_log//

/, preserving their existing location. +// Absolute paths are unchanged. This only updates the loaded view, without IO. +// Callers must restrict it to locally owned segments, not foreign snapshots. +func normalizeLocalManifestPath(manifestPath, root, legacyPrefix string, collectionID, partitionID, segmentID int64) (string, error) { + if manifestPath == "" { + return manifestPath, nil + } + base, version, err := packed.UnmarshalManifestPath(manifestPath) + if err != nil { + return "", merr.WrapErrDataIntegrity(err, "segment %d has an invalid local manifest path %q", segmentID, manifestPath) + } + if path.IsAbs(base) { + return manifestPath, nil + } + if !filepath.IsAbs(root) { + return "", merr.WrapErrParameterInvalidMsg("localStorage.path %q must be an absolute path", root) + } + // Validate the relative prefix once, before joining it to the storage root. + // Empty and dot prefixes both address the root directly. + if !filepath.IsLocal(filepath.FromSlash(path.Clean(legacyPrefix))) { + return "", merr.WrapErrDataIntegrityMsg("invalid legacy local manifest prefix %q", legacyPrefix) + } + expectedBase := path.Join(legacyPrefix, common.SegmentInsertLogPath, metautil.JoinIDPath(collectionID, partitionID, segmentID)) + if path.Clean(base) != expectedBase { + return "", merr.WrapErrDataIntegrityMsg("segment %d legacy local manifest base %q does not match its storage prefix and segment identity", segmentID, base) + } + newBase := filepath.Join(root, filepath.FromSlash(expectedBase)) + return packed.MarshalManifestPath(filepath.ToSlash(newBase), version), nil +} diff --git a/internal/datacoord/meta_local_manifest_integration_test.go b/internal/datacoord/meta_local_manifest_integration_test.go new file mode 100644 index 0000000000..6e5ebf42cc --- /dev/null +++ b/internal/datacoord/meta_local_manifest_integration_test.go @@ -0,0 +1,438 @@ +// Licensed to the LF AI & Data foundation under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package datacoord + +import ( + "context" + "io" + "io/fs" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/apache/arrow/go/v17/arrow" + "github.com/apache/arrow/go/v17/arrow/array" + "github.com/apache/arrow/go/v17/arrow/memory" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/proto" + + "github.com/milvus-io/milvus-proto/go-api/v3/commonpb" + "github.com/milvus-io/milvus-proto/go-api/v3/schemapb" + "github.com/milvus-io/milvus/internal/metastore" + catalogmocks "github.com/milvus-io/milvus/internal/metastore/mocks" + "github.com/milvus-io/milvus/internal/storage" + "github.com/milvus-io/milvus/internal/storage/localmigrate" + "github.com/milvus-io/milvus/internal/storagev2/packed" + "github.com/milvus-io/milvus/pkg/v3/objectstorage" + "github.com/milvus-io/milvus/pkg/v3/proto/datapb" + "github.com/milvus-io/milvus/pkg/v3/proto/indexpb" + "github.com/milvus-io/milvus/pkg/v3/util/conc" + "github.com/milvus-io/milvus/pkg/v3/util/merr" + "github.com/milvus-io/milvus/pkg/v3/util/paramtable" + "github.com/milvus-io/milvus/pkg/v3/util/typeutil" +) + +type legacyLocalPackedFixture struct { + root string + legacyRoot string + physicalManifest string + segment *datapb.SegmentInfo + cm *storage.LocalChunkManager + cfg *indexpb.StorageConfig + pks []int64 + texts []string +} + +// Write a real manifest at the physical location used by the old rooted +// filesystem. Its writer-generated internal references are serialized relative +// to the segment/partition, while the catalog retains the old relative base. +// Compatibility must resolve that base to this original location without copying. +// This does not fabricate a parquet, manifest, stats blob, or LOB marker file. +func newLegacyLocalPackedFixture(t *testing.T) *legacyLocalPackedFixture { + t.Helper() + return newLocalPackedFixture(t, false) +} + +// The displaced variant models complete keys passed to the old double-rooted +// filesystem. Both variants use real writers and unchanged relative manifest +// references; only the physical namespace and catalog base differ. +func newLocalPackedFixture(t *testing.T, displaced bool) *legacyLocalPackedFixture { + t.Helper() + setLocalManifestLegacyPrefix(t, "files") + root := t.TempDir() + params := paramtable.Get() + for _, setting := range []struct{ key, value, original string }{ + {params.CommonCfg.StorageType.Key, "local", params.CommonCfg.StorageType.GetValue()}, + {params.LocalStorageCfg.Path.Key, root, params.LocalStorageCfg.Path.GetValue()}, + } { + require.NoError(t, params.Save(setting.key, setting.value)) + t.Cleanup(func() { require.NoError(t, params.Save(setting.key, setting.original)) }) + } + cfg := &indexpb.StorageConfig{StorageType: "local", RootPath: root} + legacyRoot := filepath.Join(root, "files") + catalogBase := "files/insert_log/1/2/3" + if displaced { + legacyRoot = filepath.Join(root, strings.TrimPrefix(root, string(filepath.Separator))) + catalogBase = filepath.Join(root, "insert_log/1/2/3") + } + base := filepath.Join(legacyRoot, "insert_log/1/2/3") + pks := []int64{11, 22, 33} + texts := []string{strings.Repeat("first lob ", 32), strings.Repeat("second lob ", 32), strings.Repeat("third lob ", 32)} + schema := arrow.NewSchema([]arrow.Field{ + {Name: "100", Type: arrow.PrimitiveTypes.Int64, Metadata: arrow.NewMetadata([]string{packed.ArrowFieldIdMetadataKey}, []string{"100"})}, + {Name: "101", Type: arrow.BinaryTypes.String, Metadata: arrow.NewMetadata([]string{packed.ArrowFieldIdMetadataKey}, []string{"101"})}, + }, nil) + builder := array.NewRecordBuilder(memory.DefaultAllocator, schema) + defer builder.Release() + builder.Field(0).(*array.Int64Builder).AppendValues(pks, nil) + builder.Field(1).(*array.StringBuilder).AppendValues(texts, nil) + record := builder.NewRecord() + defer record.Release() + writer, err := packed.NewFFISegmentWriter(schema, &packed.SegmentWriterConfig{ + SegmentPath: base, + WriterFormat: "parquet", + TextColumns: []packed.TextColumnConfig{{ + FieldID: 101, LobBasePath: filepath.Join(legacyRoot, "insert_log/1/2/lobs/101"), + InlineThreshold: 1, MaxLobFileBytes: 1 << 20, FlushThresholdBytes: 1 << 20, + }}, + }, cfg) + require.NoError(t, err) + closed := false + t.Cleanup(func() { + if !closed { + out, closeErr := writer.Close() + if out != nil { + out.Destroy() + } + require.NoError(t, closeErr) + } + }) + require.NoError(t, writer.Write(record)) + output, err := writer.Close() + closed = true + require.NoError(t, err) + defer output.Destroy() + statsWriter := &storage.StatsWriter{} + require.NoError(t, statsWriter.GenerateByData(100, schemapb.DataType_Int64, &storage.Int64FieldData{Data: pks})) + statsPath := filepath.Join(base, "_stats/bloom_filter.100/42") + require.NoError(t, packed.WriteFile(cfg, statsPath, statsWriter.GetBuffer())) + physicalManifest, err := packed.CommitManifestUpdates(base, packed.ManifestEarliest, cfg, &packed.ManifestUpdates{ + NewFiles: output, + Stats: []packed.StatEntry{{Key: "bloom_filter.100", Files: []string{statsPath}, Metadata: map[string]string{"memory_size": "24"}}}, + }) + require.NoError(t, err) + _, version, err := packed.UnmarshalManifestPath(physicalManifest) + require.NoError(t, err) + return &legacyLocalPackedFixture{ + root: root, legacyRoot: legacyRoot, physicalManifest: physicalManifest, + segment: &datapb.SegmentInfo{ + ID: 3, CollectionID: 1, PartitionID: 2, State: commonpb.SegmentState_Flushed, + StorageVersion: storage.StorageV3, NumOfRows: int64(len(pks)), + ManifestPath: packed.MarshalManifestPath(catalogBase, version), + }, + cm: storage.NewLocalChunkManager(objectstorage.RootPath(root)), cfg: cfg, pks: pks, texts: texts, + } +} + +func (f *legacyLocalPackedFixture) assertNoCanonicalLayout(t *testing.T) { + t.Helper() + _, err := os.Stat(filepath.Join(f.root, "insert_log")) + require.ErrorIs(t, err, os.ErrNotExist, "legacy V3 compatibility must not create a canonical insert_log tree") +} + +// Read the old segment's real data, stats and LOB payload in place. This does +// not exercise compaction between the legacy and canonical storage namespaces. +func (f *legacyLocalPackedFixture) assertReadable(t *testing.T, manifest, expectedRoot string) { + t.Helper() + base, _, err := packed.UnmarshalManifestPath(manifest) + require.NoError(t, err) + fragments, err := packed.ReadFragmentsFromManifest(manifest, f.cfg, []string{"100"}) + require.NoError(t, err) + require.NotEmpty(t, fragments) + for _, fragment := range fragments { + require.True(t, strings.HasPrefix(fragment.FilePath, filepath.Join(base, "_data")+"/"), fragment.FilePath) + } + pkSchema := arrow.NewSchema([]arrow.Field{{Name: "100", Type: arrow.PrimitiveTypes.Int64}}, nil) + reader, err := packed.NewFFIPackedReader(manifest, pkSchema, []string{"100"}, 1<<20, f.cfg, nil, packed.ExternalReaderContext{}) + require.NoError(t, err) + defer func() { require.NoError(t, reader.Close()) }() + var gotPKs []int64 + for { + record, readErr := reader.ReadNext() + if readErr == io.EOF { + break + } + require.NoError(t, readErr) + gotPKs = append(gotPKs, record.Column(0).(*array.Int64).Int64Values()...) + } + require.Equal(t, f.pks, gotPKs) + + statsPaths, err := packed.NewStatsResolver(manifest, f.cfg).BloomFilterPaths(100) + require.NoError(t, err) + require.Equal(t, []string{filepath.Join(base, "_stats/bloom_filter.100/42")}, statsPaths) + statsBytes, err := f.cm.Read(context.Background(), statsPaths[0]) + require.NoError(t, err) + statsReader := &storage.StatsReader{} + statsReader.SetBuffer(statsBytes) + stats, err := statsReader.GetPrimaryKeyStats() + require.NoError(t, err) + assert.Equal(t, int64(100), stats.FieldID) + assert.True(t, stats.MinPk.EQ(storage.NewInt64PrimaryKey(f.pks[0]))) + assert.True(t, stats.MaxPk.EQ(storage.NewInt64PrimaryKey(f.pks[len(f.pks)-1]))) + + lobs, err := packed.GetManifestLobFiles(manifest, f.cfg) + require.NoError(t, err) + require.Len(t, lobs, 1, "the TEXT values must be stored out of line in a real Vortex file") + lob := lobs[0] + require.True(t, strings.HasPrefix(lob.Path, filepath.Join(expectedRoot, "insert_log/1/2/lobs/101/_data")+"/"), lob.Path) + require.Equal(t, int64(101), lob.FieldID) + require.Equal(t, int64(len(f.texts)), lob.TotalRows) + lobSchema := arrow.NewSchema([]arrow.Field{{Name: "text_data", Type: arrow.BinaryTypes.String}}, nil) + lobReader, err := packed.NewFFIPackedReaderWithFragments([]string{"text_data"}, "vortex", []packed.Fragment{{ + FilePath: lob.Path, StartRow: 0, EndRow: lob.TotalRows, RowCount: lob.TotalRows, + }}, lobSchema, []string{"text_data"}, 1<<20, f.cfg, nil, packed.ExternalReaderContext{}) + require.NoError(t, err) + defer func() { require.NoError(t, lobReader.Close()) }() + var gotTexts []string + for { + record, readErr := lobReader.ReadNext() + if readErr == io.EOF { + break + } + require.NoError(t, readErr) + column := record.Column(0).(*array.String) + for i := 0; i < column.Len(); i++ { + // ReadNext releases the preceding Arrow batch. Copy strings while + // its backing buffer is still alive, including before the EOF read. + gotTexts = append(gotTexts, strings.Clone(column.Value(i))) + } + } + require.Equal(t, f.texts, gotTexts) +} + +// Keep every fixture read beneath an opened root. The names returned by +// WalkDir are root-relative; they are never reopened as unchecked absolute +// filesystem paths. +func readLocalPackedFixtureFiles(t *testing.T, directory string) map[string][]byte { + t.Helper() + root, err := os.OpenRoot(directory) + require.NoError(t, err) + defer func() { require.NoError(t, root.Close()) }() + rootFS := root.FS() + files := make(map[string][]byte) + require.NoError(t, fs.WalkDir(rootFS, ".", func(name string, entry fs.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if entry.IsDir() { + return nil + } + contents, err := fs.ReadFile(rootFS, name) + if err == nil { + files[name] = contents + } + return err + })) + return files +} + +func TestReloadLocalManifestPreservesPackedReferences(t *testing.T) { + f := newLegacyLocalPackedFixture(t) + t.Chdir(f.root) + f.assertReadable(t, f.physicalManifest, f.legacyRoot) + f.assertNoCanonicalLayout(t) + originalFiles := readLocalPackedFixtureFiles(t, f.legacyRoot) + catalog := catalogmocks.NewDataCoordCatalog(t) + catalog.EXPECT().ListSegments(mock.Anything, int64(1)).Return([]*datapb.SegmentInfo{f.segment}, nil).Twice() + catalog.EXPECT().ListChannelCheckpoint(mock.Anything).Return(nil, nil).Twice() + mt := newLocalManifestTestMeta(catalog, f.cm) + originalSegment := proto.Clone(f.segment) + for i := 0; i < 2; i++ { + // The R/M namespace remains live compatibility data, not a rename source + // candidate, even when startup also checks for the double-root layout. + report, err := localmigrate.Migrate(t.Context(), f.root, localmigrate.Options{LegacyPrefix: "files"}) + require.NoError(t, err) + require.Zero(t, report.Renamed) + require.NoError(t, mt.reloadFromKV(context.Background(), []int64{1})) + loaded := mt.segments.GetSegment(3) + require.Equal(t, f.physicalManifest, loaded.GetManifestPath()) + f.assertReadable(t, loaded.GetManifestPath(), f.legacyRoot) + require.True(t, proto.Equal(originalSegment, f.segment)) + assertLocalManifestCatalogReadOnly(t, catalog) + f.assertNoCanonicalLayout(t) + } + // Neither metadata reload nor real reads rewrite the original manifest, + // data, stats or partition-level LOB files under root/minio.rootPath. + require.Equal(t, originalFiles, readLocalPackedFixtureFiles(t, f.legacyRoot)) + f.assertReadable(t, f.physicalManifest, f.legacyRoot) +} + +func TestMigrateDoubleRootPackedReferencesAndGCRetry(t *testing.T) { + f := newLocalPackedFixture(t, true) + t.Chdir(f.root) + f.assertNoCanonicalLayout(t) + f.assertReadable(t, f.physicalManifest, f.legacyRoot) + sources := readLocalPackedFixtureFiles(t, f.legacyRoot) + // CWD is the canonical root, leaving only the double-root rename source. + opts := localmigrate.Options{LegacyPrefix: "files"} + report, err := localmigrate.Migrate(t.Context(), f.root, opts) + require.NoError(t, err) + // The displaced insert_log directory is moved atomically as one unit; no + // file copy or backup tree is created. + require.Equal(t, 1, report.Renamed) + f.assertReadable(t, f.segment.GetManifestPath(), f.root) + canonicalRoot, err := os.OpenRoot(f.root) + require.NoError(t, err) + defer func() { require.NoError(t, canonicalRoot.Close()) }() + for name, original := range sources { + require.NoFileExists(t, filepath.Join(f.legacyRoot, name)) + contents, err := fs.ReadFile(canonicalRoot.FS(), name) + require.NoError(t, err) + require.Equal(t, original, contents, name) + } + + catalog := catalogmocks.NewDataCoordCatalog(t) + catalog.EXPECT().ListSegments(mock.Anything, int64(1)).Return([]*datapb.SegmentInfo{f.segment}, nil).Twice() + catalog.EXPECT().ListChannelCheckpoint(mock.Anything).Return(nil, nil).Twice() + mt := newLocalManifestTestMeta(catalog, f.cm) + for range 2 { + report, err = localmigrate.Migrate(t.Context(), f.root, opts) + require.NoError(t, err) + require.Zero(t, report.Renamed) + require.NoError(t, mt.reloadFromKV(context.Background(), []int64{1})) + require.Equal(t, f.segment.GetManifestPath(), mt.segments.GetSegment(3).GetManifestPath()) + assertLocalManifestCatalogReadOnly(t, catalog) + f.assertReadable(t, mt.segments.GetSegment(3).GetManifestPath(), f.root) + } + + // Segment GC and partition-level LOB GC may reclaim the canonical files. + removePool := conc.NewPool[struct{}](1) + t.Cleanup(removePool.Release) + gc := &garbageCollector{option: GcOption{cli: f.cm, removeObjectPool: removePool}} + dropped := proto.Clone(f.segment).(*datapb.SegmentInfo) + dropped.State = commonpb.SegmentState_Dropped + for range 2 { + require.NoError(t, gc.removeDroppedSegmentFiles(context.Background(), &SegmentInfo{SegmentInfo: dropped}, nil)) + } + canonicalBase, _, err := packed.UnmarshalManifestPath(f.segment.GetManifestPath()) + require.NoError(t, err) + require.NoFileExists(t, filepath.Join(canonicalBase, "_stats/bloom_filter.100/42")) + params := paramtable.Get() + safetyWindow := params.DataCoordCfg.GCLOBSafetyWindow.GetValue() + require.NoError(t, params.Save(params.DataCoordCfg.GCLOBSafetyWindow.Key, "0")) + t.Cleanup(func() { require.NoError(t, params.Save(params.DataCoordCfg.GCLOBSafetyWindow.Key, safetyWindow)) }) + lobGC := newLOBGCContext(gc) + orphans := lobGC.scanOrphanLOBFiles(t.Context(), typeutil.NewSet[string]()) + require.Len(t, orphans, 1, "the canonical orphan LOB is offered to the deletion callback") + require.True(t, strings.HasPrefix(orphans[0].FilePath, filepath.Join(f.root, "insert_log/1/2/lobs")+"/")) + lobGC.removeOrphanLOBFiles(t.Context(), orphans) + require.Empty(t, lobGC.scanOrphanLOBFiles(t.Context(), typeutil.NewSet[string]())) + // A fresh startup never treats completed renames as sources. GC must not be + // undone by restoring old immutable manifests, stats, data or LOB files. + for range 2 { + report, err = localmigrate.Migrate(t.Context(), f.root, opts) + require.NoError(t, err) + require.Zero(t, report.Renamed) + for name := range sources { + _, err := canonicalRoot.Stat(name) + require.ErrorIs(t, err, fs.ErrNotExist, name) + require.NoFileExists(t, filepath.Join(f.legacyRoot, name)) + } + } +} + +func TestReloadLocalManifestCommitSegmentManifestCAS(t *testing.T) { + for _, mutationType := range []ManifestMutationType{ManifestMutationNoop, ManifestMutationCommitUpdates} { + name := "noop CAS" + if mutationType == ManifestMutationCommitUpdates { + name = "structured mutation" + } + t.Run(name, func(t *testing.T) { + f := newLegacyLocalPackedFixture(t) + f.assertNoCanonicalLayout(t) + stored := proto.Clone(f.segment).(*datapb.SegmentInfo) + catalog := catalogmocks.NewDataCoordCatalog(t) + catalog.EXPECT().ListSegments(mock.Anything, int64(1)).RunAndReturn(func(context.Context, int64) ([]*datapb.SegmentInfo, error) { + return []*datapb.SegmentInfo{stored}, nil + }).Twice() + catalog.EXPECT().ListChannelCheckpoint(mock.Anything).Return(nil, nil).Twice() + mt := newLocalManifestTestMeta(catalog, f.cm) + ctx := context.Background() + require.NoError(t, mt.reloadFromKV(ctx, []int64{1})) + assertLocalManifestCatalogReadOnly(t, catalog) + base, version, err := packed.UnmarshalManifestPath(f.physicalManifest) + require.NoError(t, err) + prepared := packed.MarshalManifestPath(base, version+1) + // CAS compares the authoritative normalized in-memory pointer, not the + // still-relative catalog value. Old workers and old versions are stale. + for _, expected := range []string{f.segment.GetManifestPath(), packed.MarshalManifestPath(base, version-1)} { + err := mt.CommitSegmentManifest(ctx, SegmentManifestCommit{ + SegmentID: 3, ExpectedManifest: expected, + Mutation: ManifestMutation{Type: ManifestMutationNoop, ManifestPath: prepared}, + }) + require.ErrorIs(t, err, errSegmentManifestStale) + require.ErrorIs(t, err, merr.ErrServiceUnavailable) + } + catalog.AssertNotCalled(t, "Update", mock.Anything, mock.Anything) + require.Equal(t, f.segment.GetManifestPath(), stored.GetManifestPath()) + updates := &packed.ManifestUpdates{Stats: []packed.StatEntry{{ + Key: "bloom_filter.100", Files: []string{filepath.Join(base, "_stats/bloom_filter.100/42")}, + Metadata: map[string]string{"memory_size": "48"}, + }}} + commit := SegmentManifestCommit{ + SegmentID: 3, StorageConfig: f.cfg, + Mutation: ManifestMutation{Type: mutationType, Updates: updates}, + } + if mutationType == ManifestMutationNoop { + actualPrepared, err := packed.CommitManifestUpdates(base, version, f.cfg, updates) + require.NoError(t, err) + require.Equal(t, prepared, actualPrepared) + commit.ExpectedManifest = f.physicalManifest + commit.Mutation.ManifestPath = actualPrepared + } + catalog.EXPECT().Update(mock.Anything, mock.Anything).RunAndReturn(func(_ context.Context, actions ...metastore.UpdateAction) error { + require.Equal(t, f.segment.GetManifestPath(), stored.GetManifestPath(), "no startup write may precede the ordinary commit") + require.Equal(t, f.physicalManifest, mt.segments.GetSegment(3).GetManifestPath(), "publish memory only after catalog success") + require.Len(t, actions, 1) + entry, ok := actions[0].Entry.(metastore.SegmentEntry) + require.True(t, ok) + require.Equal(t, prepared, entry.Segment.GetManifestPath()) + stored = proto.Clone(entry.Segment).(*datapb.SegmentInfo) + return nil + }).Once() + require.NoError(t, mt.CommitSegmentManifest(ctx, commit)) + require.Equal(t, prepared, mt.segments.GetSegment(3).GetManifestPath()) + f.assertReadable(t, prepared, f.legacyRoot) + f.assertNoCanonicalLayout(t) + // A result prepared on the previously normalized pointer must not be + // accepted after this publication, even with a higher prepared version. + err = mt.CommitSegmentManifest(ctx, SegmentManifestCommit{ + SegmentID: 3, ExpectedManifest: f.physicalManifest, + Mutation: ManifestMutation{Type: ManifestMutationNoop, ManifestPath: packed.MarshalManifestPath(base, version+2)}, + }) + require.ErrorIs(t, err, errSegmentManifestStale) + require.Equal(t, prepared, stored.GetManifestPath()) + catalog.AssertNumberOfCalls(t, "Update", 1) + require.NoError(t, mt.reloadFromKV(ctx, []int64{1})) + require.Equal(t, prepared, mt.segments.GetSegment(3).GetManifestPath()) + f.assertNoCanonicalLayout(t) + }) + } +} diff --git a/internal/datacoord/meta_local_manifest_test.go b/internal/datacoord/meta_local_manifest_test.go new file mode 100644 index 0000000000..c3776e4185 --- /dev/null +++ b/internal/datacoord/meta_local_manifest_test.go @@ -0,0 +1,306 @@ +// Licensed to the LF AI & Data foundation under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package datacoord + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/proto" + + "github.com/milvus-io/milvus-proto/go-api/v3/commonpb" + "github.com/milvus-io/milvus/internal/metastore" + catalogmocks "github.com/milvus-io/milvus/internal/metastore/mocks" + "github.com/milvus-io/milvus/internal/mocks" + "github.com/milvus-io/milvus/internal/storage" + "github.com/milvus-io/milvus/internal/storagev2/packed" + "github.com/milvus-io/milvus/pkg/v3/objectstorage" + "github.com/milvus-io/milvus/pkg/v3/proto/datapb" + "github.com/milvus-io/milvus/pkg/v3/util/merr" + "github.com/milvus-io/milvus/pkg/v3/util/paramtable" +) + +func TestNormalizeLocalManifestPath(t *testing.T) { + const root = "/var/lib/milvus/data" + const suffix = "insert_log/1/2/3" + const version = int64(17) + for _, tc := range []struct { + name string + base string + prefix string + }{ + {"legacy prefix", "files/" + suffix, "files"}, + {"nested prefix", "legacy/files/" + suffix, "legacy/files"}, + {"empty prefix", suffix, ""}, + {"bucket root prefix", suffix, "."}, + {"empty component", "files//" + suffix, "files"}, + {"trailing separator", "files/" + suffix + "/", "files"}, + {"dot component", "files/./" + suffix, "files"}, + {"parent component within prefix", "files/tmp/../" + suffix, "files"}, + {"dot prefix component", "files/" + suffix, "./files"}, + {"empty prefix component", "legacy/files/" + suffix, "legacy//files"}, + } { + t.Run(tc.name, func(t *testing.T) { + got, err := normalizeLocalManifestPath(packed.MarshalManifestPath(tc.base, version), root, tc.prefix, 1, 2, 3) + require.NoError(t, err) + assert.Equal(t, packed.MarshalManifestPath(filepath.Join(root, tc.base), version), got) + // A persisted absolute path needs no further transformation. + again, err := normalizeLocalManifestPath(got, root, tc.prefix, 1, 2, 3) + require.NoError(t, err) + assert.Equal(t, got, again) + }) + } + + t.Run("absolute manifest preserved byte for byte", func(t *testing.T) { + manifest := ` { "ver": 23, "base_path": "/var/lib/milvus/data/insert_log/1/2/3" } ` + got, err := normalizeLocalManifestPath(manifest, root, "files", 1, 2, 3) + require.NoError(t, err) + assert.Equal(t, manifest, got) + }) + t.Run("no manifest yet", func(t *testing.T) { + got, err := normalizeLocalManifestPath("", root, "files", 1, 2, 3) + require.NoError(t, err) + assert.Empty(t, got) + }) + t.Run("filesystem root", func(t *testing.T) { + got, err := normalizeLocalManifestPath(packed.MarshalManifestPath("files/"+suffix, version), "/", "files", 1, 2, 3) + require.NoError(t, err) + assert.Equal(t, packed.MarshalManifestPath("/files/"+suffix, version), got) + }) + t.Run("relative storage root rejected", func(t *testing.T) { + got, err := normalizeLocalManifestPath(packed.MarshalManifestPath("files/"+suffix, version), "data", "files", 1, 2, 3) + require.ErrorIs(t, err, merr.ErrParameterInvalid) + assert.Empty(t, got) + }) + + for _, tc := range []struct { + name string + base string + prefix string + }{ + {"empty base", "", "files"}, + {"dot base", ".", "files"}, + {"parent base", "..", "files"}, + {"parent component", "files/../" + suffix, "files"}, + {"URI base", "s3://files/" + suffix, "files"}, + {"missing configured prefix", suffix, "files"}, + {"wrong prefix", "other/" + suffix, "files"}, + {"wrong collection", "files/insert_log/9/2/3", "files"}, + {"wrong partition", "files/insert_log/1/9/3", "files"}, + {"wrong segment", "files/insert_log/1/2/9", "files"}, + {"noncanonical ID", "files/insert_log/1/2/03", "files"}, + {"unexpected suffix", "files/" + suffix + "/_manifest", "files"}, + {"absolute prefix", "files/" + suffix, "/files"}, + {"URI prefix", "files/" + suffix, "s3://files"}, + {"parent prefix", "files/" + suffix, "../files"}, + {"outside storage root", "../files/" + suffix, "../files"}, + } { + t.Run(tc.name, func(t *testing.T) { + got, err := normalizeLocalManifestPath(packed.MarshalManifestPath(tc.base, version), root, tc.prefix, 1, 2, 3) + require.ErrorIs(t, err, merr.ErrDataIntegrity) + assert.Empty(t, got) + }) + } + t.Run("malformed JSON", func(t *testing.T) { + got, err := normalizeLocalManifestPath("not-json", root, "files", 1, 2, 3) + require.ErrorIs(t, err, merr.ErrDataIntegrity) + assert.Empty(t, got) + }) +} + +func setLocalManifestLegacyPrefix(t *testing.T, prefix string) { + t.Helper() + paramtable.Init() + params := paramtable.Get() + key := params.MinioCfg.RootPath.Key + original := params.MinioCfg.RootPath.GetValue() + require.NoError(t, params.Save(key, prefix)) + t.Cleanup(func() { require.NoError(t, params.Save(key, original)) }) +} + +func newLocalManifestTestMeta(catalog *catalogmocks.DataCoordCatalog, cm storage.ChunkManager) *meta { + return &meta{ + ctx: context.Background(), + catalog: catalog, + chunkManager: cm, + segments: NewSegmentsInfo(), + channelCPs: newChannelCps(), + } +} + +func assertLocalManifestCatalogReadOnly(t *testing.T, catalog *catalogmocks.DataCoordCatalog) { + t.Helper() + catalog.AssertNotCalled(t, "AlterSegments", mock.Anything, mock.Anything) + for _, call := range catalog.Calls { + assert.Contains(t, []string{"ListSegments", "ListChannelCheckpoint"}, call.Method, + "loading local manifests must not call a catalog mutation") + } +} + +func TestReloadLocalManifestPathsReadOnlyAndLazyPersistence(t *testing.T) { + setLocalManifestLegacyPrefix(t, "files") + ctx := context.Background() + root := t.TempDir() + cm := storage.NewLocalChunkManager(objectstorage.RootPath(root)) + legacyBase := "files/insert_log/1/2/3" + legacyFile := filepath.Join(root, legacyBase, "_data", "test-marker.parquet") + require.NoError(t, os.MkdirAll(filepath.Dir(legacyFile), 0o755)) + require.NoError(t, os.WriteFile(legacyFile, []byte("legacy-segment-data"), 0o600)) + // No layout migration: the marker exists only under the legacy prefix. + require.NoDirExists(t, filepath.Join(root, "insert_log")) + + raw := &datapb.SegmentInfo{ + ID: 3, + CollectionID: 1, + PartitionID: 2, + State: commonpb.SegmentState_Flushed, + StorageVersion: storage.StorageV3, + ManifestPath: packed.MarshalManifestPath(legacyBase, 17), + } + original := proto.Clone(raw).(*datapb.SegmentInfo) + stored := raw + catalog := catalogmocks.NewDataCoordCatalog(t) + catalog.EXPECT().ListSegments(mock.Anything, int64(1)).RunAndReturn(func(context.Context, int64) ([]*datapb.SegmentInfo, error) { + return []*datapb.SegmentInfo{stored}, nil + }).Times(3) + catalog.EXPECT().ListChannelCheckpoint(mock.Anything).Return(nil, nil).Times(3) + mt := newLocalManifestTestMeta(catalog, cm) + expected := packed.MarshalManifestPath(filepath.ToSlash(filepath.Join(root, legacyBase)), 17) + + // The same legacy catalog value is resolved on every reload without writes. + // Stats is deliberately nil: NewSegmentInfo fills it, so the compatibility + // layer must clone before constructing the in-memory segment as well. + for i := 0; i < 2; i++ { + require.NoError(t, mt.reloadFromKV(ctx, []int64{1})) + loaded := mt.segments.GetSegment(3) + require.NotNil(t, loaded) + assert.Equal(t, expected, loaded.GetManifestPath()) + assert.NotSame(t, raw, loaded.SegmentInfo) + assert.True(t, proto.Equal(original, raw), "reload must not mutate the catalog-owned proto") + assertLocalManifestCatalogReadOnly(t, catalog) + + base, version, err := packed.UnmarshalManifestPath(loaded.GetManifestPath()) + require.NoError(t, err) + assert.Equal(t, int64(17), version) + contents, err := cm.Read(ctx, filepath.Join(base, "_data", "test-marker.parquet")) + require.NoError(t, err) + assert.Equal(t, "legacy-segment-data", string(contents)) + assert.NoDirExists(t, filepath.Join(root, "insert_log"), "reload must not copy legacy files into the new layout") + } + + // A failed ordinary write does not undo the resolved in-memory view or + // commit the attempted state change. + catalog.EXPECT().AlterSegments(mock.Anything, mock.Anything).Return(merr.ErrIoTooManyRequests).Once() + require.ErrorIs(t, mt.UpdateSegmentsInfo(ctx, UpdateStatusOperator(3, commonpb.SegmentState_Dropped)), merr.ErrIoTooManyRequests) + assert.Equal(t, expected, mt.segments.GetSegment(3).GetManifestPath()) + assert.Equal(t, commonpb.SegmentState_Flushed, mt.segments.GetSegment(3).GetState()) + assert.True(t, proto.Equal(original, raw)) + assert.Same(t, raw, stored) + + // A successful normal segment update, not startup, persists the resolved path. + catalog.EXPECT().AlterSegments(mock.Anything, mock.Anything).RunAndReturn(func(_ context.Context, segments []*datapb.SegmentInfo, increments ...metastore.BinlogsIncrement) error { + require.Len(t, segments, 1) + assert.Empty(t, increments) + assert.Equal(t, expected, segments[0].GetManifestPath()) + assert.Equal(t, commonpb.SegmentState_Dropped, segments[0].GetState()) + stored = proto.Clone(segments[0]).(*datapb.SegmentInfo) + return nil + }).Once() + require.NoError(t, mt.UpdateSegmentsInfo(ctx, UpdateStatusOperator(3, commonpb.SegmentState_Dropped))) + assert.True(t, proto.Equal(original, raw)) + assert.Equal(t, expected, stored.GetManifestPath()) + require.NoError(t, mt.reloadFromKV(ctx, []int64{1})) + assert.Equal(t, expected, mt.segments.GetSegment(3).GetManifestPath()) + catalog.AssertNumberOfCalls(t, "AlterSegments", 2) + legacyContents, err := os.ReadFile(legacyFile) + require.NoError(t, err) + assert.Equal(t, "legacy-segment-data", string(legacyContents)) + assert.NoDirExists(t, filepath.Join(root, "insert_log")) +} + +func TestReloadLocalManifestPathsRejectInvalid(t *testing.T) { + setLocalManifestLegacyPrefix(t, "files") + for _, tc := range []struct { + name string + manifest string + }{ + {"malformed JSON", "not-json"}, + {"wrong segment", packed.MarshalManifestPath("files/insert_log/1/2/99", 1)}, + {"traversal", packed.MarshalManifestPath("files/../insert_log/1/2/3", 1)}, + {"missing configured prefix", packed.MarshalManifestPath("insert_log/1/2/3", 1)}, + } { + t.Run(tc.name, func(t *testing.T) { + raw := &datapb.SegmentInfo{ + ID: 3, CollectionID: 1, PartitionID: 2, + State: commonpb.SegmentState_Flushed, StorageVersion: storage.StorageV3, + ManifestPath: tc.manifest, + } + original := proto.Clone(raw) + catalog := catalogmocks.NewDataCoordCatalog(t) + catalog.EXPECT().ListSegments(mock.Anything, int64(1)).Return([]*datapb.SegmentInfo{raw}, nil).Once() + catalog.EXPECT().ListChannelCheckpoint(mock.Anything).Return(nil, nil).Maybe() + mt := newLocalManifestTestMeta(catalog, storage.NewLocalChunkManager(objectstorage.RootPath(t.TempDir()))) + require.ErrorIs(t, mt.reloadFromKV(context.Background(), []int64{1}), merr.ErrDataIntegrity) + assert.Nil(t, mt.segments.GetSegment(3), "invalid metadata must never enter the segment cache") + assert.True(t, proto.Equal(original, raw)) + assertLocalManifestCatalogReadOnly(t, catalog) + }) + } +} + +func TestReloadLocalManifestPathsBypass(t *testing.T) { + setLocalManifestLegacyPrefix(t, "files") + for _, tc := range []struct { + name string + remote bool + version int64 + manifest string + }{ + {"remote V3 remains relative", true, storage.StorageV3, packed.MarshalManifestPath("files/insert_log/1/2/3", 7)}, + {"local V2 does not parse manifest", false, storage.StorageV2, "not-a-v3-manifest"}, + {"local V3 absolute unchanged", false, storage.StorageV3, ` { "ver": 9, "base_path": "/var/lib/milvus/data/insert_log/1/2/3" } `}, + {"local V3 no manifest yet", false, storage.StorageV3, ""}, + } { + t.Run(tc.name, func(t *testing.T) { + raw := &datapb.SegmentInfo{ + ID: 3, CollectionID: 1, PartitionID: 2, + State: commonpb.SegmentState_Flushed, StorageVersion: tc.version, + ManifestPath: tc.manifest, Stats: &datapb.Statistics{}, + } + original := proto.Clone(raw) + catalog := catalogmocks.NewDataCoordCatalog(t) + catalog.EXPECT().ListSegments(mock.Anything, int64(1)).Return([]*datapb.SegmentInfo{raw}, nil).Once() + catalog.EXPECT().ListChannelCheckpoint(mock.Anything).Return(nil, nil).Once() + var cm storage.ChunkManager = storage.NewLocalChunkManager(objectstorage.RootPath(t.TempDir())) + if tc.remote { + cm = mocks.NewChunkManager(t) + } + mt := newLocalManifestTestMeta(catalog, cm) + require.NoError(t, mt.reloadFromKV(context.Background(), []int64{1})) + loaded := mt.segments.GetSegment(3) + require.NotNil(t, loaded) + assert.Equal(t, tc.manifest, loaded.GetManifestPath()) + assert.True(t, proto.Equal(original, raw)) + assertLocalManifestCatalogReadOnly(t, catalog) + }) + } +} diff --git a/internal/datacoord/segment_manager.go b/internal/datacoord/segment_manager.go index fc4bbd31f8..0a2b6a0a0b 100644 --- a/internal/datacoord/segment_manager.go +++ b/internal/datacoord/segment_manager.go @@ -19,7 +19,6 @@ package datacoord import ( "context" "fmt" - "path" "sync" "time" @@ -30,13 +29,10 @@ import ( "github.com/milvus-io/milvus/internal/datacoord/allocator" "github.com/milvus-io/milvus/internal/storage" "github.com/milvus-io/milvus/internal/storagev2/packed" - "github.com/milvus-io/milvus/pkg/v3/common" "github.com/milvus-io/milvus/pkg/v3/mlog" "github.com/milvus-io/milvus/pkg/v3/proto/datapb" "github.com/milvus-io/milvus/pkg/v3/util/lock" "github.com/milvus-io/milvus/pkg/v3/util/merr" - "github.com/milvus-io/milvus/pkg/v3/util/metautil" - "github.com/milvus-io/milvus/pkg/v3/util/paramtable" "github.com/milvus-io/milvus/pkg/v3/util/retry" "github.com/milvus-io/milvus/pkg/v3/util/tsoutil" "github.com/milvus-io/milvus/pkg/v3/util/typeutil" @@ -417,8 +413,7 @@ func (s *SegmentManager) openNewSegmentWithGivenSegmentID(ctx context.Context, r var manifestPath string if req.StorageVersion == storage.StorageV3 { - k := metautil.JoinIDPath(req.CollectionID, req.PartitionID, req.SegmentID) - basePath := path.Join(paramtable.Get().MinioCfg.RootPath.GetValue(), common.SegmentInsertLogPath, k) + basePath := storage.SegmentManifestBasePath(createStorageConfig().GetRootPath(), req.CollectionID, req.PartitionID, req.SegmentID) manifestPath = packed.MarshalManifestPath(basePath, packed.ManifestEarliest) } diff --git a/internal/core/src/index/Meta.cpp b/internal/datacoord/segment_path.go similarity index 61% rename from internal/core/src/index/Meta.cpp rename to internal/datacoord/segment_path.go index 96f1ecb9c6..c213f1dd61 100644 --- a/internal/core/src/index/Meta.cpp +++ b/internal/datacoord/segment_path.go @@ -14,8 +14,18 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "index/Meta.h" +package datacoord -namespace milvus::index { -std::string kOverrideRootPathForUT; -} // namespace milvus::index +import ( + "fmt" + "strings" + + "github.com/milvus-io/milvus/pkg/v3/common" +) + +// segmentBaseMatches checks only the segment identity, allowing legacy prefixes. +// Callers own backend-specific normalization and root containment. +func segmentBaseMatches(base string, collectionID, partitionID, segmentID int64) bool { + suffix := fmt.Sprintf("%s/%d/%d/%d", common.SegmentInsertLogPath, collectionID, partitionID, segmentID) + return base == suffix || strings.HasSuffix(base, "/"+suffix) +} diff --git a/internal/datacoord/segment_path_test.go b/internal/datacoord/segment_path_test.go new file mode 100644 index 0000000000..f69e0bf71e --- /dev/null +++ b/internal/datacoord/segment_path_test.go @@ -0,0 +1,44 @@ +// Licensed to the LF AI & Data foundation under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package datacoord + +import "testing" + +func TestSegmentBaseMatches(t *testing.T) { + for _, tc := range []struct { + base string + want bool + }{ + {"insert_log/1/2/3", true}, + {"files/insert_log/1/2/3", true}, + {"/var/lib/milvus/data/files/insert_log/1/2/3", true}, + {"files//insert_log/1/2/3", true}, + {"files/tmp/../insert_log/1/2/3", true}, + {"insert_log/1/2/30", false}, + {"insert_log/1/9/3", false}, + {"insert_log/9/2/3", false}, + {"not_insert_log/1/2/3", false}, + {"insert_log/1/2/3/_stats", false}, + {"", false}, + } { + t.Run(tc.base, func(t *testing.T) { + if got := segmentBaseMatches(tc.base, 1, 2, 3); got != tc.want { + t.Fatalf("segmentBaseMatches(%q) = %v, want %v", tc.base, got, tc.want) + } + }) + } +} diff --git a/internal/datacoord/server.go b/internal/datacoord/server.go index 3316223ab3..82cee51599 100644 --- a/internal/datacoord/server.go +++ b/internal/datacoord/server.go @@ -641,7 +641,6 @@ func (s *Server) initMeta(chunkManager storage.ChunkManager) error { if err != nil { return err } - // Load collection information asynchronously // HINT: please make sure this is the last step in the `reloadEtcdFn` function !!! go func() { diff --git a/internal/datacoord/snapshot_local_manifest_test.go b/internal/datacoord/snapshot_local_manifest_test.go new file mode 100644 index 0000000000..8de8b67de7 --- /dev/null +++ b/internal/datacoord/snapshot_local_manifest_test.go @@ -0,0 +1,197 @@ +// Licensed to the LF AI & Data foundation under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package datacoord + +import ( + "context" + "path" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/milvus-io/milvus-proto/go-api/v3/schemapb" + snapshotstorage "github.com/milvus-io/milvus/internal/snapshotio/storage" + "github.com/milvus-io/milvus/internal/storage" + "github.com/milvus-io/milvus/internal/storagev2/packed" + "github.com/milvus-io/milvus/pkg/v3/objectstorage" + "github.com/milvus-io/milvus/pkg/v3/proto/datapb" + "github.com/milvus-io/milvus/pkg/v3/util/merr" + "github.com/milvus-io/milvus/pkg/v3/util/paramtable" +) + +func newLocalManifestSnapshot(t *testing.T, layout datapb.SnapshotLayout, manifestPath string) (*snapshotMeta, *datapb.SnapshotInfo, map[string][]byte) { + t.Helper() + paramtable.Init() + oldPrefix := Params.MinioCfg.RootPath.GetValue() + require.NoError(t, Params.Save(Params.MinioCfg.RootPath.Key, "files")) + t.Cleanup(func() { require.NoError(t, Params.Save(Params.MinioCfg.RootPath.Key, oldPrefix)) }) + + ctx := context.Background() + cm := storage.NewLocalChunkManager(objectstorage.RootPath(t.TempDir())) + sm := createTestSnapshotMeta(t) + t.Cleanup(sm.loaderCancel) + // A read must not persist to the metadata catalog. + sm.catalog = nil + sm.chunkManager = cm + sm.reader = snapshotstorage.NewSnapshotReader(cm) + sm.writer = snapshotstorage.NewSnapshotWriter(cm) + info := &datapb.SnapshotInfo{ + Id: 10, + CollectionId: 100, + Name: "legacy-local", + State: datapb.SnapshotState_SnapshotStateCommitted, + } + manifestDir, metadataPath := snapshotstorage.GetSnapshotPaths(cm.RootPath(), info.GetCollectionId(), info.GetId()) + info.S3Location = metadataPath + snapshot := &snapshotstorage.SnapshotData{ + SnapshotInfo: info, + Collection: &datapb.CollectionDescription{Schema: &schemapb.CollectionSchema{Name: "local"}}, + Segments: []*datapb.SegmentDescription{{ + SegmentId: 1001, + PartitionId: 1, + StorageVersion: storage.StorageV3, + ManifestPath: manifestPath, + }}, + SegmentIDs: []int64{1001}, + } + _, _, err := sm.writer.SaveToRootWithSize(ctx, snapshot, cm.RootPath(), layout) + require.NoError(t, err) + sm.snapshotID2Info.Insert(info.GetId(), info) + sm.addToSecondaryIndexes(info) + + stored := make(map[string][]byte) + for _, file := range []string{metadataPath, snapshotstorage.GetSegmentManifestPath(manifestDir, 1001)} { + data, err := cm.Read(ctx, file) + require.NoError(t, err) + stored[file] = data + } + return sm, info, stored +} + +func assertLocalSnapshotFilesUnchanged(t *testing.T, sm *snapshotMeta, stored map[string][]byte) { + t.Helper() + for file, original := range stored { + data, err := sm.chunkManager.Read(context.Background(), file) + require.NoError(t, err) + assert.Equal(t, original, data, file) + } +} + +func TestReadSnapshotDataNormalizesLegacyLocalManifestWithoutWrites(t *testing.T) { + ctx := context.Background() + legacy := packed.MarshalManifestPath("files/insert_log/100/1/1001", 7) + sm, info, stored := newLocalManifestSnapshot(t, datapb.SnapshotLayout_SnapshotLayoutReferenced, legacy) + want := packed.MarshalManifestPath(path.Join(sm.chunkManager.RootPath(), "files/insert_log/100/1/1001"), 7) + + var firstFingerprint string + for read := range 2 { + loaded, err := sm.ReadSnapshotData(ctx, info.GetCollectionId(), info.GetName(), true) + require.NoError(t, err) + require.Len(t, loaded.Segments, 1) + assert.Equal(t, want, loaded.Segments[0].GetManifestPath()) + assertLocalSnapshotFilesUnchanged(t, sm, stored) + fingerprint, err := snapshotstorage.SnapshotFingerprint(loaded) + require.NoError(t, err) + if read == 0 { + firstFingerprint = fingerprint + } else { + assert.Equal(t, firstFingerprint, fingerprint) + } + // Neither a prior read nor an in-memory consumer can alter the source. + loaded.Segments[0].ManifestPath = "modified-by-consumer" + } + storedSnapshot, err := sm.reader.ReadSnapshot(ctx, info.GetS3Location(), true) + require.NoError(t, err) + assert.Equal(t, legacy, storedSnapshot.Segments[0].GetManifestPath()) + metadataOnly, err := sm.ReadSnapshotData(ctx, info.GetCollectionId(), info.GetName(), false) + require.NoError(t, err) + assert.Empty(t, metadataOnly.Segments) + assertLocalSnapshotFilesUnchanged(t, sm, stored) +} + +func TestReadSnapshotDataRejectsMismatchedLegacyManifestIdentity(t *testing.T) { + legacy := packed.MarshalManifestPath("files/insert_log/100/1/9999", 7) + sm, info, stored := newLocalManifestSnapshot(t, datapb.SnapshotLayout_SnapshotLayoutReferenced, legacy) + loaded, err := sm.ReadSnapshotData(context.Background(), info.GetCollectionId(), info.GetName(), true) + require.ErrorIs(t, err, merr.ErrDataIntegrity) + assert.Nil(t, loaded) + assertLocalSnapshotFilesUnchanged(t, sm, stored) +} + +func TestReadSnapshotDataPreservesOtherManifestNamespaces(t *testing.T) { + ctx := context.Background() + legacy := packed.MarshalManifestPath("files/insert_log/100/1/1001", 7) + t.Run("self-contained", func(t *testing.T) { + sm, info, stored := newLocalManifestSnapshot(t, datapb.SnapshotLayout_SnapshotLayoutReferenced, legacy) + snapshot, err := sm.reader.ReadSnapshot(ctx, info.GetS3Location(), true) + require.NoError(t, err) + bundleManifest := packed.MarshalManifestPath(path.Join(sm.chunkManager.RootPath(), + snapshotstorage.ExportedSnapshotFilesPath, "insert_log/100/1/1001"), 7) + snapshot.Segments[0].ManifestPath = bundleManifest + _, _, err = sm.writer.SaveToRootWithSize(ctx, snapshot, sm.chunkManager.RootPath(), datapb.SnapshotLayout_SnapshotLayoutSelfContained) + require.NoError(t, err) + for file := range stored { + stored[file], err = sm.chunkManager.Read(ctx, file) + require.NoError(t, err) + } + loaded, err := sm.ReadSnapshotData(ctx, info.GetCollectionId(), info.GetName(), true) + require.NoError(t, err) + assert.Equal(t, bundleManifest, loaded.Segments[0].GetManifestPath()) + assertLocalSnapshotFilesUnchanged(t, sm, stored) + }) + t.Run("self-contained-rejects-outside-bundle", func(t *testing.T) { + sm, info, stored := newLocalManifestSnapshot(t, datapb.SnapshotLayout_SnapshotLayoutSelfContained, legacy) + // A self-contained bundle's paths are governed by the bundle root, + // not by the instance's legacy prefix compatibility rule. + _, directErr := sm.reader.ReadSnapshot(ctx, info.GetS3Location(), true) + require.ErrorIs(t, directErr, merr.ErrDataIntegrity) + loaded, err := sm.ReadSnapshotData(ctx, info.GetCollectionId(), info.GetName(), true) + require.ErrorIs(t, err, merr.ErrDataIntegrity) + assert.Equal(t, directErr.Error(), err.Error()) + assert.Nil(t, loaded) + assertLocalSnapshotFilesUnchanged(t, sm, stored) + }) + t.Run("absolute", func(t *testing.T) { + absolute := packed.MarshalManifestPath("/existing/insert_log/100/1/1001", 9) + sm, info, stored := newLocalManifestSnapshot(t, datapb.SnapshotLayout_SnapshotLayoutReferenced, absolute) + loaded, err := sm.ReadSnapshotData(ctx, info.GetCollectionId(), info.GetName(), true) + require.NoError(t, err) + assert.Equal(t, absolute, loaded.Segments[0].GetManifestPath()) + assertLocalSnapshotFilesUnchanged(t, sm, stored) + }) + t.Run("foreign", func(t *testing.T) { + sm, info, stored := newLocalManifestSnapshot(t, datapb.SnapshotLayout_SnapshotLayoutReferenced, legacy) + loaded, err := sm.ReadExternalSnapshotDataWithChunkManager(ctx, sm.chunkManager, info.GetS3Location(), true) + require.NoError(t, err) + assert.Equal(t, legacy, loaded.Segments[0].GetManifestPath()) + assertLocalSnapshotFilesUnchanged(t, sm, stored) + }) + t.Run("remote", func(t *testing.T) { + sm, info, stored := newLocalManifestSnapshot(t, datapb.SnapshotLayout_SnapshotLayoutReferenced, legacy) + // Keep real snapshot reads on disk while exercising the non-local backend gate. + sm.chunkManager = &snapshotManifestNonLocalChunkManager{ChunkManager: sm.chunkManager} + loaded, err := sm.ReadSnapshotData(ctx, info.GetCollectionId(), info.GetName(), true) + require.NoError(t, err) + assert.Equal(t, legacy, loaded.Segments[0].GetManifestPath()) + assertLocalSnapshotFilesUnchanged(t, sm, stored) + }) +} + +type snapshotManifestNonLocalChunkManager struct { + storage.ChunkManager +} diff --git a/internal/datacoord/snapshot_meta.go b/internal/datacoord/snapshot_meta.go index 83c42f5fb1..fc30dcc9ce 100644 --- a/internal/datacoord/snapshot_meta.go +++ b/internal/datacoord/snapshot_meta.go @@ -939,6 +939,23 @@ func (sm *snapshotMeta) ReadSnapshotData(ctx context.Context, collectionID int64 mlog.Error(context.TODO(), "failed to read snapshot data from S3", mlog.Err(err)) return nil, err } + // Named, referenced snapshots point at this instance's storage. Older + // snapshots retain relative V3 manifest bases whose files remain under the + // legacy local prefix. Resolve only the returned view to their absolute paths; + // the stored snapshot is immutable, and foreign/self-contained layouts use other roots. + if local, ok := sm.chunkManager.(*storage.LocalChunkManager); ok && snapshotData.Layout == datapb.SnapshotLayout_SnapshotLayoutReferenced { + for _, segment := range snapshotData.Segments { + if segment.GetStorageVersion() != storage.StorageV3 || segment.GetManifestPath() == "" { + continue + } + manifestPath, err := normalizeLocalManifestPath(segment.GetManifestPath(), local.RootPath(), + Params.MinioCfg.RootPath.GetValue(), collectionID, segment.GetPartitionId(), segment.GetSegmentId()) + if err != nil { + return nil, merr.Wrapf(err, "normalize local snapshot %d segment %d manifest", snapshotInfo.GetId(), segment.GetSegmentId()) + } + segment.ManifestPath = manifestPath + } + } // Step 3: Merge S3 location from memory into returned data snapshotData.SnapshotInfo.S3Location = snapshotInfo.S3Location diff --git a/internal/datacoord/task_refresh_external_collection_test.go b/internal/datacoord/task_refresh_external_collection_test.go index 082b2759b4..1bf7c01418 100644 --- a/internal/datacoord/task_refresh_external_collection_test.go +++ b/internal/datacoord/task_refresh_external_collection_test.go @@ -913,6 +913,7 @@ func TestRefreshExternalCollectionTask_QueryTaskOnWorker(t *testing.T) { }) t.Run("task_failed_on_worker", func(t *testing.T) { + paramtable.Init() catalog := &stubCatalog{} refreshMeta, err := newExternalCollectionRefreshMeta(context.Background(), catalog) assert.NoError(t, err) @@ -924,6 +925,7 @@ func TestRefreshExternalCollectionTask_QueryTaskOnWorker(t *testing.T) { State: indexpb.JobState_JobStateInProgress, ExternalSource: "s3://bucket/path", ExternalSpec: "iceberg", + TaskIds: []int64{1001, 1002}, } err = refreshMeta.AddJob(job) assert.NoError(t, err) @@ -939,16 +941,26 @@ func TestRefreshExternalCollectionTask_QueryTaskOnWorker(t *testing.T) { } err = refreshMeta.AddTask(protoTask) assert.NoError(t, err) + // A failed task must terminate the job without waiting for siblings. + sibling := proto.Clone(protoTask).(*datapb.ExternalCollectionRefreshTask) + sibling.TaskId = 1002 + assert.NoError(t, refreshMeta.AddTask(sibling)) alloc := &stubAllocator{nextID: 99999} task := newRefreshExternalCollectionTask(protoTask, refreshMeta, nil, alloc) + var failedJobs []int64 + checker := newRefreshChecker(t.Context(), nil, refreshMeta, make(chan struct{}), nil, nil, + func(jobID int64) { failedJobs = append(failedJobs, jobID) }, nil, nil) + task.processFinishedJob = checker.processJobByID cluster := &stubCluster{} - // Mock QueryRefreshExternalCollectionTask to return failed state + // Missing explore manifests surface as a worker failure, even when + // the native reader classifies the underlying error as transient. + failReason := "failed to read explore manifest: File not found: loon FFI transient error" mockQuery := mockey.Mock((*stubCluster).QueryRefreshExternalCollectionTask).Return(&datapb.RefreshExternalCollectionTaskResponse{ State: indexpb.JobState_JobStateFailed, - FailReason: "worker error", + FailReason: failReason, }, nil).Build() defer mockQuery.UnPatch() @@ -957,7 +969,11 @@ func TestRefreshExternalCollectionTask_QueryTaskOnWorker(t *testing.T) { // Task should be marked as failed metaTask := refreshMeta.GetTask(1001) assert.Equal(t, indexpb.JobState_JobStateFailed, metaTask.GetState()) - assert.Contains(t, metaTask.GetFailReason(), "worker error") + assert.Equal(t, failReason, metaTask.GetFailReason()) + assert.Equal(t, indexpb.JobState_JobStateFailed, refreshMeta.GetJob(1).GetState()) + assert.Equal(t, failReason, refreshMeta.GetJob(1).GetFailReason()) + assert.Equal(t, []int64{1}, failedJobs) + assert.Equal(t, indexpb.JobState_JobStateInProgress, refreshMeta.GetTask(1002).GetState()) }) t.Run("task_finished_success", func(t *testing.T) { diff --git a/internal/datacoord/task_stats.go b/internal/datacoord/task_stats.go index 6718b73c2a..ca4a38a314 100644 --- a/internal/datacoord/task_stats.go +++ b/internal/datacoord/task_stats.go @@ -19,6 +19,8 @@ package datacoord import ( "context" "fmt" + "path" + "strings" "time" "github.com/cockroachdb/errors" @@ -353,16 +355,43 @@ func (st *statsTask) cleanupRejectedStatsResultFiles(ctx context.Context, result return } - files, err := collectRejectedStatsResultFiles(result) + expectedSegmentID := st.GetSegmentID() + if st.GetTargetSegmentID() != 0 { + expectedSegmentID = st.GetTargetSegmentID() + } + if result.GetCollectionID() != st.GetCollectionID() || + result.GetPartitionID() != st.GetPartitionID() || + result.GetSegmentID() != expectedSegmentID { + mlog.Warn(ctx, "refuse to cleanup rejected stats files for a mismatched segment", + mlog.FieldTaskID(st.GetTaskID()), + mlog.Int64("resultCollectionID", result.GetCollectionID()), + mlog.Int64("resultPartitionID", result.GetPartitionID()), + mlog.Int64("resultSegmentID", result.GetSegmentID())) + return + } + + basePath, files, err := collectRejectedStatsResultFiles( + result, st.meta.chunkManager.RootPath(), + st.GetCollectionID(), st.GetPartitionID(), expectedSegmentID) if err != nil { mlog.Warn(ctx, "failed to collect rejected stats result files", mlog.FieldTaskID(st.GetTaskID()), mlog.FieldSegmentID(st.GetSegmentID()), mlog.Err(err)) + return } if len(files) == 0 { return } + // Complete keys are absolute filesystem paths locally and bucket-relative + // object keys remotely. Never resolve a local base against the process CWD + // or pass a filesystem path to remote storage. + if _, local := st.meta.chunkManager.(*storage.LocalChunkManager); local != path.IsAbs(basePath) { + mlog.Warn(ctx, "refuse to cleanup rejected stats files with a manifest base for another storage namespace", + mlog.FieldTaskID(st.GetTaskID()), + mlog.String("basePath", basePath)) + return + } if err := st.meta.chunkManager.MultiRemove(ctx, files); err != nil { mlog.Warn(ctx, "failed to cleanup rejected stats result files", mlog.FieldTaskID(st.GetTaskID()), @@ -372,49 +401,131 @@ func (st *statsTask) cleanupRejectedStatsResultFiles(ctx context.Context, result } } -func collectRejectedStatsResultFiles(result *workerpb.StatsResult) ([]string, error) { +func collectRejectedStatsResultFiles( + result *workerpb.StatsResult, + rootPath string, + collectionID, partitionID, segmentID int64, +) (string, []string, error) { files := make([]string, 0) seen := make(map[string]struct{}) - addFile := func(file string) { - if file == "" { - return + addStatsFiles := func(statsBasePath string, statFiles []string, relative bool) error { + local := path.IsAbs(statsBasePath) + statsPrefix := statsBasePath + "/" + if local { + statsPrefix = path.Clean(statsBasePath) + "/" } - if _, ok := seen[file]; ok { - return + for _, file := range statFiles { + if strings.TrimSpace(file) == "" { + continue + } + // JSON reports names relative to the field's stats directory; + // TEXT reports complete keys. Decode before the shared check. + if relative { + if path.IsAbs(file) { + return merr.WrapErrDataIntegrityMsg("rejected JSON stats file must be relative to its stats directory") + } + file = statsBasePath + "/" + file + } + checkedFile := file + if local { + checkedFile = path.Clean(file) + } + // Workers report complete TEXT stats keys, including an absolute + // filesystem path on local storage. Do not guess a prefix for a + // relative filename or a key outside this field's stats directory. + if !strings.HasPrefix(checkedFile, statsPrefix) { + return merr.WrapErrDataIntegrityMsg("rejected stats file escapes its stats directory") + } + // Clean only for the local directory check. Remote object keys must be + // deleted exactly as reported, not rewritten as filesystem paths. + if _, ok := seen[file]; ok { + continue + } + seen[file] = struct{}{} + files = append(files, file) } - seen[file] = struct{}{} - files = append(files, file) + return nil } - - for _, stats := range result.GetTextStatsLogs() { - for _, file := range stats.GetFiles() { - addFile(file) - } - } - - jsonStats := result.GetJsonKeyStatsLogs() - if len(jsonStats) == 0 { - return files, nil - } - manifest := result.GetBaseManifest() if manifest == "" { manifest = result.GetManifest() } - if manifest == "" { - return files, merr.WrapErrServiceInternalMsg("manifest is empty for rejected json stats result") - } - basePath, _, err := packed.UnmarshalManifestPath(manifest) - if err != nil { - return files, err - } - for fieldID, stats := range jsonStats { - statsBasePath := fmt.Sprintf("%s/_stats/json_stats.%d", basePath, fieldID) - for _, file := range metautil.BuildStatsFilePaths(statsBasePath, stats.GetFiles()) { - addFile(file) + basePath := rootPath + if manifest != "" { + var err error + basePath, _, err = packed.UnmarshalManifestPath(manifest) + if err != nil { + return "", nil, err + } + if err := validateStatsManifestBase(basePath, collectionID, partitionID, segmentID); err != nil { + return "", nil, err + } + if result.GetManifest() != "" && result.GetManifest() != manifest { + resultBase, _, err := packed.UnmarshalManifestPath(result.GetManifest()) + if err != nil { + return "", nil, err + } + sameBase := resultBase == basePath + if path.IsAbs(basePath) { + sameBase = path.Clean(resultBase) == path.Clean(basePath) + } + if !sameBase { + return "", nil, merr.WrapErrDataIntegrityMsg("rejected stats result changes manifest base path") + } } } - return files, nil + + // Manifest-less V2 results use build/version directories; V3 uses _stats. + validateLegacyLocation := func(fieldID, buildID, version int64) error { + if buildID <= 0 || version <= 0 { + return merr.WrapErrDataIntegrityMsg( + "rejected stats result for field %d carries no build ID or version to locate its stats directory", fieldID) + } + return nil + } + for fieldID, stats := range result.GetTextStatsLogs() { + statsBase := basePath + fmt.Sprintf("/_stats/text_index.%d", fieldID) + if manifest == "" { + if err := validateLegacyLocation(fieldID, stats.GetBuildID(), stats.GetVersion()); err != nil { + return "", nil, err + } + statsBase = metautil.BuildTextIndexPrefix(rootPath, stats.GetBuildID(), stats.GetVersion(), + collectionID, partitionID, segmentID, fieldID) + } + if err := addStatsFiles(statsBase, stats.GetFiles(), false); err != nil { + return "", nil, err + } + } + for fieldID, stats := range result.GetJsonKeyStatsLogs() { + statsBase := metautil.BuildJSONKeyStatsV3Prefix(basePath, fieldID) + if manifest == "" { + if err := validateLegacyLocation(fieldID, stats.GetBuildID(), stats.GetVersion()); err != nil { + return "", nil, err + } + statsBase = metautil.BuildJSONKeyStatsPrefix(rootPath, + stats.GetJsonKeyStatsDataFormat(), stats.GetBuildID(), stats.GetVersion(), + collectionID, partitionID, segmentID, fieldID) + } + if err := addStatsFiles(statsBase, stats.GetFiles(), true); err != nil { + return "", nil, err + } + } + return basePath, files, nil +} + +func validateStatsManifestBase(basePath string, collectionID, partitionID, segmentID int64) error { + // Manifests carry complete storage keys, not provider URIs. A URI would + // be passed verbatim to ChunkManager cleanup and is not a valid object key. + if strings.Contains(basePath, "://") { + return merr.WrapErrDataIntegrityMsg("stats manifest base must be a storage key") + } + if path.IsAbs(basePath) { + basePath = path.Clean(basePath) + } + if !segmentBaseMatches(basePath, collectionID, partitionID, segmentID) { + return merr.WrapErrDataIntegrityMsg("stats manifest base does not match task segment identity") + } + return nil } func (st *statsTask) DropTaskOnWorker(cluster session.Cluster) { diff --git a/internal/datacoord/task_stats_test.go b/internal/datacoord/task_stats_test.go index 140f90e17b..9d0a9419da 100644 --- a/internal/datacoord/task_stats_test.go +++ b/internal/datacoord/task_stats_test.go @@ -18,12 +18,17 @@ package datacoord import ( "context" + "fmt" + "os" + "path" + "path/filepath" "testing" "time" "github.com/bytedance/mockey" "github.com/cockroachdb/errors" "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" "github.com/milvus-io/milvus-proto/go-api/v3/commonpb" @@ -35,15 +40,498 @@ import ( "github.com/milvus-io/milvus/internal/storage" "github.com/milvus-io/milvus/internal/storagev2/packed" "github.com/milvus-io/milvus/pkg/v3/common" + "github.com/milvus-io/milvus/pkg/v3/objectstorage" "github.com/milvus-io/milvus/pkg/v3/proto/datapb" "github.com/milvus-io/milvus/pkg/v3/proto/indexpb" "github.com/milvus-io/milvus/pkg/v3/proto/workerpb" "github.com/milvus-io/milvus/pkg/v3/taskcommon" "github.com/milvus-io/milvus/pkg/v3/util/lock" "github.com/milvus-io/milvus/pkg/v3/util/merr" + "github.com/milvus-io/milvus/pkg/v3/util/metautil" "github.com/milvus-io/milvus/pkg/v3/util/typeutil" ) +// Manifest bases on local storage are complete filesystem paths; cleanup +// deletes the reported files below /_stats/... directly, without +// joining any root. +func TestCleanupRejectedStatsResultFiles_LocalUsesCompletePaths(t *testing.T) { + root := t.TempDir() + cm := storage.NewLocalChunkManager(objectstorage.RootPath(root)) + basePath := filepath.Join(root, "insert_log/1/2/3") + statsFile := filepath.Join(basePath, "_stats/text_index.100/1178/2/index") + require.NoError(t, cm.Write(context.Background(), statsFile, []byte("stale"))) + + task := &statsTask{ + StatsTask: &indexpb.StatsTask{CollectionID: 1, PartitionID: 2, SegmentID: 3}, + meta: &meta{chunkManager: cm}, + } + task.cleanupRejectedStatsResultFiles(context.Background(), &workerpb.StatsResult{ + CollectionID: 1, + PartitionID: 2, + SegmentID: 3, + BaseManifest: packed.MarshalManifestPath(basePath, 1), + TextStatsLogs: map[int64]*datapb.TextIndexStats{ + 100: {Files: []string{statsFile}}, + }, + }) + + _, err := os.Stat(statsFile) + require.ErrorIs(t, err, os.ErrNotExist) +} + +func TestCleanupRejectedStatsResultFiles_LocalCompleteTextAndRelativeJSON(t *testing.T) { + ctx := context.Background() + root := t.TempDir() + cm := storage.NewLocalChunkManager(objectstorage.RootPath(root)) + basePath := filepath.Join(root, "insert_log/1/2/3") + textFile := filepath.Join(basePath, "_stats/text_index.100/tokenizer.json") + jsonFile := filepath.Join(basePath, "_stats/json_stats.101/shared_key_index/.managed.json_0") + untouchedFile := filepath.Join(basePath, "_stats/json_stats.101/shared_key_index/keep") + for _, file := range []string{textFile, jsonFile, untouchedFile} { + require.NoError(t, cm.Write(ctx, file, []byte("stats"))) + } + + task := &statsTask{ + StatsTask: &indexpb.StatsTask{CollectionID: 1, PartitionID: 2, SegmentID: 3}, + meta: &meta{chunkManager: cm}, + } + task.cleanupRejectedStatsResultFiles(ctx, &workerpb.StatsResult{ + CollectionID: 1, + PartitionID: 2, + SegmentID: 3, + BaseManifest: packed.MarshalManifestPath(basePath, 1), + TextStatsLogs: map[int64]*datapb.TextIndexStats{ + 100: {Files: []string{textFile}}, + }, + JsonKeyStatsLogs: map[int64]*datapb.JsonKeyStats{ + 101: {Files: []string{"shared_key_index/.managed.json_0"}}, + }, + }) + + require.NoFileExists(t, textFile) + require.NoFileExists(t, jsonFile) + content, err := cm.Read(ctx, untouchedFile) + require.NoError(t, err) + require.Equal(t, []byte("stats"), content) +} + +func TestCleanupRejectedStatsResultFiles_LocalRejectsRelativeTextFiles(t *testing.T) { + for _, relativeFile := range []string{"index", "./index", "task/index"} { + t.Run(relativeFile, func(t *testing.T) { + ctx := context.Background() + root := t.TempDir() + cm := storage.NewLocalChunkManager(objectstorage.RootPath(root)) + basePath := filepath.Join(root, "insert_log/1/2/3") + statsDir := filepath.Join(basePath, "_stats/text_index.100") + completeFile := filepath.Join(statsDir, "complete") + guessedFile := filepath.Join(statsDir, relativeFile) + require.NoError(t, cm.Write(ctx, completeFile, []byte("keep-complete"))) + require.NoError(t, cm.Write(ctx, guessedFile, []byte("keep-relative"))) + + task := &statsTask{ + StatsTask: &indexpb.StatsTask{CollectionID: 1, PartitionID: 2, SegmentID: 3}, + meta: &meta{chunkManager: cm}, + } + task.cleanupRejectedStatsResultFiles(ctx, &workerpb.StatsResult{ + CollectionID: 1, + PartitionID: 2, + SegmentID: 3, + BaseManifest: packed.MarshalManifestPath(basePath, 1), + TextStatsLogs: map[int64]*datapb.TextIndexStats{ + 100: {Files: []string{completeFile, relativeFile}}, + }, + }) + + // Reject the whole result, including its otherwise valid entry; no + // relative entry may be guessed into a complete deletion path. + content, err := cm.Read(ctx, completeFile) + require.NoError(t, err) + require.Equal(t, []byte("keep-complete"), content) + content, err = cm.Read(ctx, guessedFile) + require.NoError(t, err) + require.Equal(t, []byte("keep-relative"), content) + }) + } +} + +// A relative manifest base on local storage is a pre-3.0.2 manifest that +// DataCoord has not rewritten; cleanup must neither resolve it against the +// process CWD nor guess a root. +func TestCleanupRejectedStatsResultFiles_LocalRefusesRelativeManifestBase(t *testing.T) { + root := t.TempDir() + cm := storage.NewLocalChunkManager(objectstorage.RootPath(root)) + inRoot := filepath.Join(root, "insert_log/1/2/3/_stats/text_index.100/index") + require.NoError(t, cm.Write(context.Background(), inRoot, []byte("keep"))) + + task := &statsTask{ + StatsTask: &indexpb.StatsTask{CollectionID: 1, PartitionID: 2, SegmentID: 3}, + meta: &meta{chunkManager: cm}, + } + task.cleanupRejectedStatsResultFiles(context.Background(), &workerpb.StatsResult{ + CollectionID: 1, + PartitionID: 2, + SegmentID: 3, + BaseManifest: packed.MarshalManifestPath("insert_log/1/2/3", 1), + TextStatsLogs: map[int64]*datapb.TextIndexStats{ + 100: {Files: []string{"insert_log/1/2/3/_stats/text_index.100/index"}}, + }, + }) + require.FileExists(t, inRoot) +} + +func TestCleanupRejectedStatsResultFiles_LocalRejectsUnsafePaths(t *testing.T) { + ctx := context.Background() + outer := t.TempDir() + root := filepath.Join(outer, "storage") + require.NoError(t, os.MkdirAll(root, 0o755)) + cm := storage.NewLocalChunkManager(objectstorage.RootPath(root)) + task := &statsTask{ + StatsTask: &indexpb.StatsTask{CollectionID: 1, PartitionID: 2, SegmentID: 3}, + meta: &meta{chunkManager: cm}, + } + basePath := filepath.Join(root, "insert_log/1/2/3") + baseManifest := packed.MarshalManifestPath(basePath, 1) + resultForTextPath := func(filePath string) *workerpb.StatsResult { + return &workerpb.StatsResult{ + CollectionID: 1, + PartitionID: 2, + SegmentID: 3, + BaseManifest: baseManifest, + TextStatsLogs: map[int64]*datapb.TextIndexStats{ + 100: {Files: []string{filePath}}, + }, + } + } + + // Traversal and complete paths outside the stats directory are refused. + outsidePath := filepath.Join(outer, "outside") + require.NoError(t, os.WriteFile(outsidePath, []byte("keep"), 0o600)) + task.cleanupRejectedStatsResultFiles(ctx, resultForTextPath("../../../../outside")) + require.FileExists(t, outsidePath) + task.cleanupRejectedStatsResultFiles(ctx, resultForTextPath(outsidePath)) + require.FileExists(t, outsidePath) + + // A file outside /_stats/. is refused even when it is + // below the segment base. + arbitraryPath := filepath.Join(basePath, "not-stats") + require.NoError(t, os.MkdirAll(basePath, 0o755)) + require.NoError(t, os.WriteFile(arbitraryPath, []byte("keep"), 0o600)) + task.cleanupRejectedStatsResultFiles(ctx, resultForTextPath("../../not-stats")) + require.FileExists(t, arbitraryPath) + task.cleanupRejectedStatsResultFiles(ctx, resultForTextPath(arbitraryPath)) + require.FileExists(t, arbitraryPath) + + safePath := filepath.Join(basePath, "_stats/text_index.100/index") + require.NoError(t, os.MkdirAll(filepath.Dir(safePath), 0o755)) + require.NoError(t, os.WriteFile(safePath, []byte("keep"), 0o600)) + for _, unsafePath := range []string{ + "file://" + safePath, + filepath.Dir(safePath), + filepath.Dir(safePath) + "/../../not-stats", + } { + task.cleanupRejectedStatsResultFiles(ctx, resultForTextPath(unsafePath)) + require.FileExists(t, safePath, unsafePath) + } + + // A result for another segment is refused even when it reports a + // complete path in this task's stats directory. + task.cleanupRejectedStatsResultFiles(ctx, &workerpb.StatsResult{ + CollectionID: 1, + PartitionID: 2, + SegmentID: 4, + BaseManifest: packed.MarshalManifestPath(filepath.Join(root, "insert_log/1/2/4"), 1), + TextStatsLogs: map[int64]*datapb.TextIndexStats{ + 100: {Files: []string{safePath}}, + }, + }) + require.FileExists(t, safePath) + + // An unparsable or empty manifest is refused. + task.cleanupRejectedStatsResultFiles(ctx, &workerpb.StatsResult{ + CollectionID: 1, + PartitionID: 2, + SegmentID: 3, + BaseManifest: "not-a-manifest", + TextStatsLogs: map[int64]*datapb.TextIndexStats{ + 100: {Files: []string{safePath}}, + }, + }) + require.FileExists(t, safePath) + task.cleanupRejectedStatsResultFiles(ctx, &workerpb.StatsResult{ + CollectionID: 1, + PartitionID: 2, + SegmentID: 3, + BaseManifest: packed.MarshalManifestPath("", 1), + TextStatsLogs: map[int64]*datapb.TextIndexStats{ + 100: {Files: []string{safePath}}, + }, + }) + require.FileExists(t, safePath) +} + +type rootedStatsChunkManager struct { + storage.ChunkManager + rootPath string + removed []string +} + +func (m *rootedStatsChunkManager) RootPath() string { return m.rootPath } + +func (m *rootedStatsChunkManager) MultiRemove(_ context.Context, filePaths []string) error { + m.removed = append(m.removed, filePaths...) + return nil +} + +// The manifest base is a complete object key; cleanup deletes exactly the keys +// the worker reported under it and does not second-guess them against the +// configured root (a manifest written under an earlier minio.rootPath is still +// authoritative for where its files are). +func TestCleanupRejectedStatsResultFiles_RemoteUsesManifestKeysAsIs(t *testing.T) { + cm := &rootedStatsChunkManager{rootPath: "files"} + task := &statsTask{ + StatsTask: &indexpb.StatsTask{CollectionID: 1, PartitionID: 2, SegmentID: 3}, + meta: &meta{chunkManager: cm}, + } + result := func(basePath string) *workerpb.StatsResult { + return &workerpb.StatsResult{ + CollectionID: 1, + PartitionID: 2, + SegmentID: 3, + BaseManifest: packed.MarshalManifestPath(basePath, 1), + TextStatsLogs: map[int64]*datapb.TextIndexStats{ + 100: {Files: []string{basePath + "/_stats/text_index.100/index"}}, + }, + } + } + + task.cleanupRejectedStatsResultFiles(context.Background(), result("other/insert_log/1/2/3")) + require.Equal(t, []string{"other/insert_log/1/2/3/_stats/text_index.100/index"}, cm.removed) + + cm.removed = nil + task.cleanupRejectedStatsResultFiles(context.Background(), result("files/insert_log/1/2/3")) + require.Equal(t, []string{"files/insert_log/1/2/3/_stats/text_index.100/index"}, cm.removed) + + cm.rootPath = "" + cm.removed = nil + task.cleanupRejectedStatsResultFiles(context.Background(), result("insert_log/1/2/3")) + require.Equal(t, []string{"insert_log/1/2/3/_stats/text_index.100/index"}, cm.removed) + for _, base := range []string{"files/./insert_log/1/2/3", "files//insert_log/1/2/3", "files/tmp/../insert_log/1/2/3"} { + cm.removed = nil + task.cleanupRejectedStatsResultFiles(context.Background(), result(base)) + require.Equal(t, []string{base + "/_stats/text_index.100/index"}, cm.removed) + } + + // Wrong segment identity and URI bases are still refused. + for _, base := range []string{"files/insert_log/1/2/4", "s3://bucket/files/insert_log/1/2/3"} { + cm.removed = nil + task.cleanupRejectedStatsResultFiles(context.Background(), result(base)) + require.Empty(t, cm.removed, base) + } +} + +func TestCleanupRejectedStatsResultFiles_RemoteRejectsIncompleteOrUnsafeTextPaths(t *testing.T) { + for _, basePath := range []string{"files/insert_log/1/2/3", "insert_log/1/2/3"} { + t.Run(basePath, func(t *testing.T) { + statsDir := path.Join(basePath, "_stats/text_index.100") + completeFile := path.Join(statsDir, "index") + for _, file := range []string{ + "index", + "./index", + "task/index", + "../index", + "/" + completeFile, + "s3://bucket/" + completeFile, + path.Join(basePath, "not-stats"), + path.Join(basePath, "_stats/text_index.1000/index"), + statsDir, + path.Join(statsDir, "../text_index.101/index"), + } { + t.Run(file, func(t *testing.T) { + cm := &rootedStatsChunkManager{} + task := &statsTask{ + StatsTask: &indexpb.StatsTask{CollectionID: 1, PartitionID: 2, SegmentID: 3}, + meta: &meta{chunkManager: cm}, + } + result := &workerpb.StatsResult{ + CollectionID: 1, + PartitionID: 2, + SegmentID: 3, + BaseManifest: packed.MarshalManifestPath(basePath, 1), + TextStatsLogs: map[int64]*datapb.TextIndexStats{ + 100: {Files: []string{completeFile, file}}, + }, + } + + _, files, err := collectRejectedStatsResultFiles(result, "files", 1, 2, 3) + require.Error(t, err) + require.Empty(t, files) + task.cleanupRejectedStatsResultFiles(context.Background(), result) + require.Empty(t, cm.removed) + }) + } + }) + } +} + +func TestCleanupRejectedStatsResultFiles_RemoteRejectsAbsoluteManifestBase(t *testing.T) { + cm := &rootedStatsChunkManager{rootPath: "files"} + task := &statsTask{ + StatsTask: &indexpb.StatsTask{CollectionID: 1, PartitionID: 2, SegmentID: 3}, + meta: &meta{chunkManager: cm}, + } + basePath := "/files/insert_log/1/2/3" + task.cleanupRejectedStatsResultFiles(context.Background(), &workerpb.StatsResult{ + CollectionID: 1, + PartitionID: 2, + SegmentID: 3, + BaseManifest: packed.MarshalManifestPath(basePath, 1), + TextStatsLogs: map[int64]*datapb.TextIndexStats{ + 100: {Files: []string{path.Join(basePath, "_stats/text_index.100/index")}}, + }, + }) + require.Empty(t, cm.removed) +} + +func TestCleanupRejectedStatsResultFiles_RejectsInvalidJSONDescriptors(t *testing.T) { + basePath := "files/insert_log/1/2/3" + for _, descriptor := range []string{ + "/tmp/index", + } { + t.Run(descriptor, func(t *testing.T) { + cm := &rootedStatsChunkManager{rootPath: "files"} + task := &statsTask{ + StatsTask: &indexpb.StatsTask{CollectionID: 1, PartitionID: 2, SegmentID: 3}, + meta: &meta{chunkManager: cm}, + } + result := &workerpb.StatsResult{ + CollectionID: 1, + PartitionID: 2, + SegmentID: 3, + BaseManifest: packed.MarshalManifestPath(basePath, 1), + TextStatsLogs: map[int64]*datapb.TextIndexStats{ + 100: {Files: []string{path.Join(basePath, "_stats/text_index.100/index")}}, + }, + JsonKeyStatsLogs: map[int64]*datapb.JsonKeyStats{ + 101: {Files: []string{"shared_key_index/index", descriptor}}, + }, + } + _, files, err := collectRejectedStatsResultFiles(result, "files", 1, 2, 3) + require.Error(t, err) + require.Empty(t, files) + task.cleanupRejectedStatsResultFiles(context.Background(), result) + require.Empty(t, cm.removed) + }) + } +} + +func TestCollectRejectedStatsResultFiles_LocalJSONDirectoryBoundary(t *testing.T) { + basePath := filepath.Join(t.TempDir(), "insert_log/1/2/3") + for _, descriptor := range []string{"../index", "nested/../../index", ".", ".."} { + t.Run(descriptor, func(t *testing.T) { + result := &workerpb.StatsResult{ + BaseManifest: packed.MarshalManifestPath(basePath, 1), + JsonKeyStatsLogs: map[int64]*datapb.JsonKeyStats{ + 101: {Files: []string{descriptor}}, + }, + } + _, files, err := collectRejectedStatsResultFiles(result, "", 1, 2, 3) + require.Error(t, err) + require.Empty(t, files) + }) + } +} + +func TestCollectRejectedStatsResultFiles_ManifestBaseIdentity(t *testing.T) { + for _, root := range []string{"files", t.TempDir()} { + t.Run(root, func(t *testing.T) { + base := root + "/./insert_log/1/2/3" + result := &workerpb.StatsResult{ + BaseManifest: packed.MarshalManifestPath(base, 1), + Manifest: packed.MarshalManifestPath(path.Clean(base), 2), + JsonKeyStatsLogs: map[int64]*datapb.JsonKeyStats{ + 101: {Files: []string{"index"}}, + }, + } + _, files, err := collectRejectedStatsResultFiles(result, root, 1, 2, 3) + if path.IsAbs(root) { + require.NoError(t, err) + require.Equal(t, []string{base + "/_stats/json_stats.101/index"}, files) + } else { + // Equivalent filesystem spellings are distinct remote objects. + require.Error(t, err) + require.Empty(t, files) + } + }) + } +} + +func TestCollectRejectedStatsResultFiles_PreservesFileKeys(t *testing.T) { + for _, root := range []string{"files", "", t.TempDir()} { + for _, manifestLayout := range []bool{false, true} { + t.Run(fmt.Sprintf("root=%s/manifest=%t", root, manifestLayout), func(t *testing.T) { + basePath := path.Join(root, "insert_log/1/2/3") + textDir := metautil.BuildTextIndexPrefix(root, 10, 1, 1, 2, 3, 100) + jsonDir := metautil.BuildJSONKeyStatsPrefix(root, 2, 10, 1, 1, 2, 3, 101) + manifest := "" + if manifestLayout { + manifest = packed.MarshalManifestPath(basePath, 1) + textDir = basePath + "/_stats/text_index.100" + jsonDir = basePath + "/_stats/json_stats.101" + } + names := []string{"./index", "nested//index", "nested/../index", "s3://bucket/index"} + if !path.IsAbs(root) { + // These are literal object names, not filesystem traversal. + names = append(names, "../index", ".", "..") + } + textFiles := make([]string, 0, len(names)) + want := make([]string, 0, 2*len(names)) + for _, name := range names { + textFiles = append(textFiles, textDir+"/"+name) + want = append(want, textDir+"/"+name, jsonDir+"/"+name) + } + result := &workerpb.StatsResult{ + BaseManifest: manifest, + TextStatsLogs: map[int64]*datapb.TextIndexStats{ + 100: {BuildID: 10, Version: 1, Files: append(textFiles, textFiles[0])}, + }, + JsonKeyStatsLogs: map[int64]*datapb.JsonKeyStats{ + 101: {BuildID: 10, Version: 1, JsonKeyStatsDataFormat: 2, Files: names}, + }, + } + _, files, err := collectRejectedStatsResultFiles(result, root, 1, 2, 3) + require.NoError(t, err) + require.ElementsMatch(t, want, files) + require.Equal(t, names, result.GetJsonKeyStatsLogs()[101].GetFiles()) + }) + } + } +} + +func TestCleanupRejectedStatsResultFiles_LocalAcceptsEquivalentPaths(t *testing.T) { + ctx := context.Background() + root := t.TempDir() + cm := storage.NewLocalChunkManager(objectstorage.RootPath(root)) + basePath := filepath.Join(root, "insert_log/1/2/3") + statsDir := basePath + "/_stats/text_index.100" + require.NoError(t, os.MkdirAll(statsDir+"/nested", 0o755)) + task := &statsTask{ + StatsTask: &indexpb.StatsTask{CollectionID: 1, PartitionID: 2, SegmentID: 3}, + meta: &meta{chunkManager: cm}, + } + for _, name := range []string{"./index", "nested//index", "nested/../index"} { + file := statsDir + "/" + name + require.NoError(t, cm.Write(ctx, file, []byte("stale"))) + task.cleanupRejectedStatsResultFiles(ctx, &workerpb.StatsResult{ + CollectionID: 1, + PartitionID: 2, + SegmentID: 3, + BaseManifest: packed.MarshalManifestPath(basePath, 1), + TextStatsLogs: map[int64]*datapb.TextIndexStats{100: {Files: []string{file}}}, + }) + require.NoFileExists(t, file) + } +} + type statsTaskSuite struct { suite.Suite mt *meta @@ -67,6 +555,8 @@ type mockeyChunkManager struct { storage.ChunkManager } +func (m *mockeyChunkManager) RootPath() string { return "" } + func Test_statsTaskSuite(t *testing.T) { suite.Run(t, new(statsTaskSuite)) } @@ -897,7 +1387,7 @@ func (s *statsTaskSuite) TestClassifyStatsManifestCommitError() { func (s *statsTaskSuite) TestCollectRejectedStatsResultFiles() { baseManifest := `{"base_path":"files/insert_log/1/2/1179","ver":2}` s.Run("collect text and json stats files", func() { - files, err := collectRejectedStatsResultFiles(&workerpb.StatsResult{ + basePath, files, err := collectRejectedStatsResultFiles(&workerpb.StatsResult{ BaseManifest: baseManifest, TextStatsLogs: map[int64]*datapb.TextIndexStats{ 101: { @@ -909,9 +1399,10 @@ func (s *statsTaskSuite) TestCollectRejectedStatsResultFiles() { Files: []string{"shared_key_index/.managed.json_0"}, }, }, - }) + }, "files", 1, 2, 1179) s.NoError(err) + s.Equal("files/insert_log/1/2/1179", basePath) s.ElementsMatch([]string{ "files/insert_log/1/2/1179/_stats/text_index.101/tokenizer.json", "files/insert_log/1/2/1179/_stats/json_stats.102/shared_key_index/.managed.json_0", @@ -919,7 +1410,8 @@ func (s *statsTaskSuite) TestCollectRejectedStatsResultFiles() { }) s.Run("deduplicate text stats files without json stats", func() { - files, err := collectRejectedStatsResultFiles(&workerpb.StatsResult{ + basePath, files, err := collectRejectedStatsResultFiles(&workerpb.StatsResult{ + BaseManifest: baseManifest, TextStatsLogs: map[int64]*datapb.TextIndexStats{ 101: { Files: []string{ @@ -929,39 +1421,123 @@ func (s *statsTaskSuite) TestCollectRejectedStatsResultFiles() { }, }, }, - }) + }, "files", 1, 2, 1179) s.NoError(err) + s.Equal("files/insert_log/1/2/1179", basePath) s.Equal([]string{"files/insert_log/1/2/1179/_stats/text_index.101/tokenizer.json"}, files) }) - s.Run("json stats without manifest returns typed error", func() { - files, err := collectRejectedStatsResultFiles(&workerpb.StatsResult{ + // Without a manifest the legacy prefix is rebuilt from the build ID and task + // version, so a result that reports neither does not say where its files + // live. Refuse rather than addressing build 0 / version 0. + s.Run("stats without manifest or build coordinates are refused", func() { + basePath, files, err := collectRejectedStatsResultFiles(&workerpb.StatsResult{ JsonKeyStatsLogs: map[int64]*datapb.JsonKeyStats{ 102: { Files: []string{"shared_key_index/.managed.json_0"}, }, }, - }) + }, "files", 1, 2, 1179) + s.Empty(basePath) s.Empty(files) - s.ErrorIs(err, merr.ErrServiceInternal) - s.Contains(err.Error(), "manifest is empty for rejected json stats result") + s.ErrorIs(err, merr.ErrDataIntegrity) + s.Contains(err.Error(), "carries no build ID or version") }) s.Run("json stats with invalid manifest returns error", func() { - files, err := collectRejectedStatsResultFiles(&workerpb.StatsResult{ + basePath, files, err := collectRejectedStatsResultFiles(&workerpb.StatsResult{ BaseManifest: "invalid", JsonKeyStatsLogs: map[int64]*datapb.JsonKeyStats{ 102: { Files: []string{"shared_key_index/.managed.json_0"}, }, }, - }) + }, "files", 1, 2, 1179) + s.Empty(basePath) s.Empty(files) s.Error(err) }) + + // A StorageV2 stats result has no manifest at all: the worker writes TEXT and + // JSON stats to the traditional prefixes under the storage root. Requiring a + // manifest would skip cleanup and leak those files. + s.Run("legacy text stats without manifest are collected", func() { + basePath, files, err := collectRejectedStatsResultFiles(&workerpb.StatsResult{ + TextStatsLogs: map[int64]*datapb.TextIndexStats{ + 101: { + BuildID: 77, + Version: 3, + Files: []string{ + "files/text_log/77/3/1/2/1179/101/tokenizer.json", + "files/text_log/77/3/1/2/1179/101/meta.json", + }, + }, + }, + }, "files", 1, 2, 1179) + + s.NoError(err) + s.Equal("files", basePath) + s.ElementsMatch([]string{ + "files/text_log/77/3/1/2/1179/101/tokenizer.json", + "files/text_log/77/3/1/2/1179/101/meta.json", + }, files) + }) + + s.Run("legacy json stats without manifest resolve relative names", func() { + basePath, files, err := collectRejectedStatsResultFiles(&workerpb.StatsResult{ + JsonKeyStatsLogs: map[int64]*datapb.JsonKeyStats{ + 102: { + BuildID: 77, + Version: 3, + JsonKeyStatsDataFormat: 2, + Files: []string{"shared_key_index/.managed.json_0"}, + }, + }, + }, "files", 1, 2, 1179) + + s.NoError(err) + s.Equal("files", basePath) + s.Equal([]string{"files/json_stats/2/77/3/1/2/1179/102/shared_key_index/.managed.json_0"}, files) + }) + + // Local storage keys are absolute, and the legacy prefix is rebuilt from the + // storage root rather than guessed against the process working directory. + s.Run("legacy stats under an absolute local root", func() { + basePath, files, err := collectRejectedStatsResultFiles(&workerpb.StatsResult{ + TextStatsLogs: map[int64]*datapb.TextIndexStats{ + 101: { + BuildID: 77, + Version: 3, + Files: []string{"/var/lib/milvus/data/text_log/77/3/1/2/1179/101/tokenizer.json"}, + }, + }, + }, "/var/lib/milvus/data", 1, 2, 1179) + + s.NoError(err) + s.Equal("/var/lib/milvus/data", basePath) + s.Equal([]string{"/var/lib/milvus/data/text_log/77/3/1/2/1179/101/tokenizer.json"}, files) + }) + + // The segment identity still comes from the task, so a file written under + // another segment's legacy prefix is refused rather than deleted. + s.Run("legacy stats outside the task segment prefix are refused", func() { + basePath, files, err := collectRejectedStatsResultFiles(&workerpb.StatsResult{ + TextStatsLogs: map[int64]*datapb.TextIndexStats{ + 101: { + BuildID: 77, + Version: 3, + Files: []string{"files/text_log/77/3/1/2/9999/101/tokenizer.json"}, + }, + }, + }, "files", 1, 2, 1179) + + s.Empty(basePath) + s.Empty(files) + s.ErrorIs(err, merr.ErrDataIntegrity) + }) } func (s *statsTaskSuite) TestQueryTaskOnWorkerDiscardsStaleStatsResult() { diff --git a/internal/datanode/compactor/bump_schema_version_compactor.go b/internal/datanode/compactor/bump_schema_version_compactor.go index 480e68a5c0..b6eb3ae930 100644 --- a/internal/datanode/compactor/bump_schema_version_compactor.go +++ b/internal/datanode/compactor/bump_schema_version_compactor.go @@ -631,12 +631,36 @@ func (t *bumpSchemaVersionCompactionTask) runFullSchemaRewrite(existingFields ma _, ttlFieldPhysicallyPresent := existingFields[ttlFieldID] sourceHasTTLField := ttlFieldID >= common.StartOfUserFieldID && ttlFieldPhysicallyPresent preMaterializeFilter := len(delta) > 0 || t.plan.GetCollectionTtl() > 0 || sourceHasTTLField - reader, _, err := newCompactionSegmentRecordReaderWithFields(t.ctx, segment, t.plan.GetSchema(), t.compactionParams.StorageConfig, existingFields, - storage.WithCollectionID(collectionID), - storage.WithVersion(segment.GetStorageVersion()), - storage.WithDownloader(t.chunkManager.MultiRead), - storage.WithStorageConfig(t.compactionParams.StorageConfig), - ) + + // Prepare LOB (TEXT) handling before opening the source reader: a legacy + // partition namespace needs a TEXT-decoding reader rooted at the source. + if err := t.initLOBCompactionContext(t.ctx); err != nil { + return nil, err + } + textDecodeConfigs, err := t.lobContext.GetSourceTextColumnConfigs(segment.GetManifest()) + if err != nil { + return nil, err + } + var reader storage.RecordReader + if len(textDecodeConfigs) > 0 { + reader, err = storage.NewTextDecodedManifestRecordReader( + t.ctx, + segment.GetManifest(), + compactionReadSchema(t.plan.GetSchema(), existingFields), + textDecodeConfigs, + storage.WithPresentFields(existingFields), + storage.WithCollectionID(collectionID), + storage.WithVersion(segment.GetStorageVersion()), + storage.WithStorageConfig(t.compactionParams.StorageConfig), + ) + } else { + reader, _, err = newCompactionSegmentRecordReaderWithFields(t.ctx, segment, t.plan.GetSchema(), t.compactionParams.StorageConfig, existingFields, + storage.WithCollectionID(collectionID), + storage.WithVersion(segment.GetStorageVersion()), + storage.WithDownloader(t.chunkManager.MultiRead), + storage.WithStorageConfig(t.compactionParams.StorageConfig), + ) + } if err != nil { return nil, err } @@ -648,14 +672,6 @@ func (t *bumpSchemaVersionCompactionTask) runFullSchemaRewrite(existingFields ma } defer materializer.Close() - // Prepare LOB (TEXT) handling: a schema bump keeps existing TEXT LOB data - // unchanged, so REUSE_ALL — the writer preserves the encoded LOB reference - // bytes (WithTextRefsAsBinary) and the source LOB files are merged into the - // output manifest afterwards (applyLOBCompaction). Mirrors sort compaction. - if err := t.initLOBCompactionContext(t.ctx); err != nil { - return nil, err - } - alloc := allocator.NewLocalAllocator(t.plan.GetPreAllocatedLogIDs().GetBegin(), t.plan.GetPreAllocatedLogIDs().GetEnd()) writerOpts := []storage.RwOption{ storage.WithUploader(func(ctx context.Context, kvs map[string][]byte) error { @@ -666,6 +682,19 @@ func (t *bumpSchemaVersionCompactionTask) runFullSchemaRewrite(existingFields ma storage.WithCollectionID(collectionID), storage.WithUseLoonFFI(t.compactionParams.UseLoonFFI), } + if t.lobContext != nil && t.lobContext.ShouldRewriteAnyField() { + lobBasePath := storage.SegmentPartitionBasePath( + t.compactionParams.StorageConfig.GetRootPath(), collectionID, segment.GetPartitionID()) + textColumnConfigs := t.lobContext.GetTextColumnConfigs( + lobBasePath, + t.compactionParams.TextInlineThreshold, + t.compactionParams.TextMaxLobFileBytes, + t.compactionParams.TextFlushThresholdBytes, + ) + if len(textColumnConfigs) > 0 { + writerOpts = append(writerOpts, storage.WithTextColumnConfigs(textColumnConfigs)) + } + } if t.lobContext != nil && t.lobContext.HasReuseAllFields() { // Existing TEXT columns arrive from the reader as encoded binary LOB refs; // tell the writer to write them via the physical binary schema unchanged. @@ -838,11 +867,9 @@ func (t *bumpSchemaVersionCompactionTask) runFullSchemaRewrite(existingFields ma }, nil } -// initLOBCompactionContext prepares REUSE_ALL LOB handling for the (single) input -// segment's TEXT columns. A schema bump is 1->1 and never changes existing TEXT LOB -// data, so all TEXT fields are forced REUSE_ALL (GetForcedStrategy): the source LOB -// files stay in place and only their references are merged into the output manifest. -// No-op when the schema has no TEXT fields. Mirrors sort compaction. +// initLOBCompactionContext prepares LOB handling for the full-rewrite path. A +// schema bump normally reuses references, unless its new output segment has a +// different partition base from the source manifest. func (t *bumpSchemaVersionCompactionTask) initLOBCompactionContext(ctx context.Context) error { textFieldIDs := compaction.GetTEXTFieldIDsFromSchema(t.plan.GetSchema()) if len(textFieldIDs) == 0 { @@ -865,9 +892,20 @@ func (t *bumpSchemaVersionCompactionTask) initLOBCompactionContext(ctx context.C // schema-bump is always 1 source -> 1 output; forced REUSE_ALL. t.lobContext.SetCompactionType(datapb.CompactionType_BumpSchemaVersionCompaction, 1, 1) t.lobContext.ComputeStrategies(textFieldIDs, t.compactionParams.LOBHoleRatioThreshold) - mlog.Info(ctx, "schema bump: initialized LOB compaction context (REUSE_ALL)", + outputPartitionBase := storage.SegmentPartitionBasePath( + t.compactionParams.StorageConfig.GetRootPath(), segment.GetCollectionID(), segment.GetPartitionID()) + partitionBaseMismatch, err := compaction.LOBSourcePartitionBaseMismatch(sourceManifests, outputPartitionBase) + if err != nil { + return err + } + if partitionBaseMismatch { + t.lobContext.ForceRewriteAllAcrossPartitionBases(textFieldIDs) + } + mlog.Info(ctx, "schema bump: initialized LOB compaction context", mlog.Int64("planID", t.GetPlanID()), mlog.FieldSegmentID(segment.GetSegmentID()), + mlog.Bool("partitionBaseMismatch", partitionBaseMismatch), + mlog.String("outputPartitionBase", outputPartitionBase), mlog.Int64s("textFieldIDs", textFieldIDs), ) return nil diff --git a/internal/datanode/compactor/bump_schema_version_compactor_test.go b/internal/datanode/compactor/bump_schema_version_compactor_test.go index 80f5f029f8..3d2f399e1b 100644 --- a/internal/datanode/compactor/bump_schema_version_compactor_test.go +++ b/internal/datanode/compactor/bump_schema_version_compactor_test.go @@ -954,6 +954,9 @@ func (s *BumpSchemaVersionCompactionTaskSuite) buildTextLOBTask(withTextField bo OutputFieldNames: []string{"sparse"}, OutputFieldIds: []int64{102}, }} } + params := compaction.GenParams() + manifest := packed.MarshalManifestPath( + storage.SegmentManifestBasePath(params.StorageConfig.GetRootPath(), 1, 1, 100), 1) plan := &datapb.CompactionPlan{ PlanID: 999, Type: datapb.CompactionType_BumpSchemaVersionCompaction, @@ -961,10 +964,9 @@ func (s *BumpSchemaVersionCompactionTaskSuite) buildTextLOBTask(withTextField bo TotalRows: 3, SegmentBinlogs: []*datapb.CompactionSegmentBinlogs{{ CollectionID: 1, PartitionID: 1, SegmentID: 100, - StorageVersion: storage.StorageV3, Manifest: "manifest", + StorageVersion: storage.StorageV3, Manifest: manifest, }}, } - params := compaction.GenParams() params.StorageVersion = storage.StorageV3 return &bumpSchemaVersionCompactionTask{ ctx: context.Background(), @@ -973,6 +975,23 @@ func (s *BumpSchemaVersionCompactionTaskSuite) buildTextLOBTask(withTextField bo } } +func (s *BumpSchemaVersionCompactionTaskSuite) TestInitLOBCompactionContextRewritesAcrossPartitionBases() { + task := s.buildTextLOBTask(true) + task.plan.SegmentBinlogs[0].Manifest = packed.MarshalManifestPath( + task.compactionParams.StorageConfig.GetRootPath()+"/files/insert_log/1/1/100", 1) + collectPatch := mockey.Mock(compaction.CollectLobFilesFromManifests).Return( + map[int64][]packed.LobFileInfo{100: {{FieldID: 101, TotalRows: 3, ValidRows: 3}}}, nil, + ).Build() + defer collectPatch.UnPatch() + + err := task.initLOBCompactionContext(context.Background()) + s.NoError(err) + s.Require().NotNil(task.lobContext) + s.False(task.lobContext.HasReuseAllFields()) + s.True(task.lobContext.ShouldRewriteAnyField()) + s.Equal(compaction.LOBStrategyRewriteAll, task.lobContext.GetStrategy(101)) +} + // TestInitLOBCompactionContextTextFieldReuseAll: a schema-bump on a segment with a // TEXT field forces REUSE_ALL for that field (never REWRITE_ALL), so its existing // LOB data is carried by reference. diff --git a/internal/datanode/compactor/bump_schema_version_ut_test.go b/internal/datanode/compactor/bump_schema_version_ut_test.go index 6a1243d896..57b25f3573 100644 --- a/internal/datanode/compactor/bump_schema_version_ut_test.go +++ b/internal/datanode/compactor/bump_schema_version_ut_test.go @@ -25,6 +25,7 @@ import ( "os" "path" "path/filepath" + "strconv" "strings" "testing" "time" @@ -117,6 +118,7 @@ type fixtureSpec struct { commitTs uint64 fillTextAnalyzer bool // decorate the base text field with multi_analyzer_params (by_field=lang) textLOBFieldID int64 // >0: write this source TEXT field through the LOB-aware writer + legacyNamespace bool // write the source under localStorage.path/files instead of the canonical root } type fixtureOpt func(*fixtureSpec) @@ -167,6 +169,10 @@ func withTextLOBSource(fieldID int64) fixtureOpt { return func(s *fixtureSpec) { s.textLOBFieldID = fieldID } } +func withLegacySourceNamespace() fixtureOpt { + return func(s *fixtureSpec) { s.legacyNamespace = true } +} + type bumpFixture struct { t *testing.T task *bumpSchemaVersionCompactionTask @@ -270,14 +276,20 @@ func buildBumpFixture(t *testing.T, opts ...fixtureOpt) *bumpFixture { compAlloc := NewCompactionAllocator(segIDAlloc, logIDAlloc) params := compaction.GenParams() params.StorageVersion = storage.StorageV3 + sourceStorageConfig := params.StorageConfig + if spec.legacyNamespace { + sourceStorageConfig = proto.Clone(params.StorageConfig).(*indexpb.StorageConfig) + sourceStorageConfig.RootPath = path.Join(params.StorageConfig.GetRootPath(), "files") + } fixtureWriterOpts := []storage.RwOption{ - storage.WithStorageConfig(params.StorageConfig), + storage.WithStorageConfig(sourceStorageConfig), storage.WithVersion(storage.StorageV3), } if spec.textLOBFieldID > 0 { - lobBasePath := path.Join(params.StorageConfig.GetRootPath(), - common.SegmentInsertLogPath, metautil.JoinIDPath(CollectionID, PartitionID)) + lobBasePath := path.Join( + storage.SegmentPartitionBasePath(sourceStorageConfig.GetRootPath(), CollectionID, PartitionID), + "lobs", strconv.FormatInt(spec.textLOBFieldID, 10)) fixtureWriterOpts = append(fixtureWriterOpts, storage.WithTextColumnConfigs([]packed.TextColumnConfig{{ FieldID: spec.textLOBFieldID, LobBasePath: lobBasePath, @@ -2111,6 +2123,44 @@ func readTextRefs(t *testing.T, fix *bumpFixture, manifest string, textID int64) return refs } +func readLOBPayloads(t *testing.T, fix *bumpFixture, manifest string, textID int64) []string { + lobFiles, err := packed.GetManifestLobFiles(manifest, fix.cfg) + require.NoError(t, err) + var fragments []packed.Fragment + for _, lobFile := range lobFiles { + if lobFile.FieldID != textID { + continue + } + fragments = append(fragments, packed.Fragment{ + FilePath: lobFile.Path, + StartRow: 0, + EndRow: lobFile.TotalRows, + RowCount: lobFile.TotalRows, + }) + } + require.NotEmpty(t, fragments) + schema := arrow.NewSchema([]arrow.Field{{Name: "text_data", Type: arrow.BinaryTypes.String}}, nil) + reader, err := packed.NewFFIPackedReaderWithFragments( + []string{"text_data"}, "vortex", fragments, schema, []string{"text_data"}, + 1<<20, fix.cfg, nil, packed.ExternalReaderContext{}) + require.NoError(t, err) + defer func() { require.NoError(t, reader.Close()) }() + + var values []string + for { + record, readErr := reader.ReadNext() + if readErr == sio.EOF { + break + } + require.NoError(t, readErr) + column := record.Column(0).(*array.String) + for i := 0; i < column.Len(); i++ { + values = append(values, strings.Clone(column.Value(i))) + } + } + return values +} + // [LOB1][S0] additive keeps TEXT LOB data dereferenceable: every row's LOB // reference bytes survive the bump unchanged AND the referenced .vx blob file // stays byte-identical — together this proves content round-trip without @@ -2178,6 +2228,63 @@ func TestBumpUTFullRewriteTextLOBKeptRowsSurvive(t *testing.T) { require.Equal(t, preLob, listLobFiles(t, fix), "LOB blob files must stay byte-identical") } +// [LOB3][S0] a pre-3.0.2 local source can remain under the legacy `files` +// namespace after upgrade. A full schema rewrite creates a canonical output +// segment, so byte-reusing its TEXT references would make the output reader look +// for the legacy UUID under the wrong partition. The compactor must decode the +// source payload and write a new canonical LOB file instead. +func TestBumpUTFullRewriteTextLOBRewritesAcrossPartitionNamespaces(t *testing.T) { + setupBumpUTEnv(t) + const textID = int64(105) + const droppedID = int64(107) + const rows = 6 + fix := buildBumpFixture(t, withRows(rows), + withSourceFields( + &schemapb.FieldSchema{FieldID: textID, Name: "big_text", DataType: schemapb.DataType_Text}, + &schemapb.FieldSchema{FieldID: droppedID, Name: "dropped", DataType: schemapb.DataType_Int64}, + ), + withFillValue(func(i int, ts uint64, v map[int64]any) { + v[textID] = bumpFxLobText(i) + v[droppedID] = int64(i) + }), + withTextLOBSource(textID), + withLegacySourceNamespace(), + withTargetDroppedField(droppedID), + ) + + sourceBase, _, err := packed.UnmarshalManifestPath(fix.sourceManifest) + require.NoError(t, err) + require.Contains(t, sourceBase, "/files/insert_log/") + sourceRefs := readTextRefs(t, fix, fix.sourceManifest, textID) + + segment := runCompact(t, fix) + outputBase, _, err := packed.UnmarshalManifestPath(segment.GetManifest()) + require.NoError(t, err) + require.Equal(t, + storage.SegmentManifestBasePath(fix.cfg.GetRootPath(), CollectionID, PartitionID, segment.GetSegmentID()), + outputBase) + outputRefs := readTextRefs(t, fix, segment.GetManifest(), textID) + require.NotEqual(t, sourceRefs, outputRefs, "cross-namespace rewrite must allocate new LOB references") + + gotPayloads := readLOBPayloads(t, fix, segment.GetManifest(), textID) + require.Len(t, gotPayloads, rows) + for i := 0; i < rows; i++ { + require.Equal(t, bumpFxLobText(i), gotPayloads[i]) + } + for _, lobFile := range mustManifestLobFiles(t, fix, segment.GetManifest()) { + if lobFile.FieldID == textID { + require.NotContains(t, lobFile.Path, "/files/insert_log/") + require.Contains(t, lobFile.Path, "/insert_log/") + } + } +} + +func mustManifestLobFiles(t *testing.T, fix *bumpFixture, manifest string) []packed.LobFileInfo { + lobFiles, err := packed.GetManifestLobFiles(manifest, fix.cfg) + require.NoError(t, err) + return lobFiles +} + // [ST2][S1] full rewrite with deletes + BM25 materialization: the committed // stats ledger must accumulate ONLY the surviving rows — filtered rows leaking // into stats is invisible to every other oracle in the suite. diff --git a/internal/datanode/compactor/clustering_compactor.go b/internal/datanode/compactor/clustering_compactor.go index f4b73157fe..8d24dfa97d 100644 --- a/internal/datanode/compactor/clustering_compactor.go +++ b/internal/datanode/compactor/clustering_compactor.go @@ -584,7 +584,11 @@ func (t *clusteringCompactionTask) mappingSegment( } } - rr, existingFields, err := newCompactionSegmentRecordReader(ctx, segment, t.plan.Schema, t.compactionParams.StorageConfig, + textDecodeConfigs, err := t.lobContext.GetSourceTextColumnConfigs(segment.GetManifest()) + if err != nil { + return err + } + rr, existingFields, err := newTextDecodedCompactionSegmentRecordReader(ctx, segment, t.plan.Schema, t.compactionParams.StorageConfig, textDecodeConfigs, storage.WithDownloader(func(ctx context.Context, paths []string) ([][]byte, error) { return t.binlogIO.Download(ctx, paths) }), @@ -871,7 +875,11 @@ func (t *clusteringCompactionTask) scalarAnalyzeSegment( readSchema.Fields = selectedFields readSchema.StructArrayFields = nil - rr, _, err := newCompactionSegmentRecordReader(ctx, segment, readSchema, t.compactionParams.StorageConfig, + textDecodeConfigs, err := t.lobContext.GetSourceTextColumnConfigs(segment.GetManifest()) + if err != nil { + return nil, err + } + rr, _, err := newTextDecodedCompactionSegmentRecordReader(ctx, segment, readSchema, t.compactionParams.StorageConfig, textDecodeConfigs, storage.WithDownloader(func(ctx context.Context, paths []string) ([][]byte, error) { return t.binlogIO.Download(ctx, paths) }), @@ -1032,9 +1040,10 @@ func (t *clusteringCompactionTask) getWriterOpts() []storage.RwOption { } if t.lobContext != nil && t.lobContext.ShouldRewriteAnyField() { - // LOB base path at partition level: {root}/insert_log/{coll}/{part} - lobBasePath := path.Join(t.compactionParams.StorageConfig.GetRootPath(), - common.SegmentInsertLogPath, metautil.JoinIDPath(t.collectionID, t.partitionID)) + // LOB base path at partition level, sharing the manifest root rule so + // the reader (which reconstructs lobs/ from the manifest base) resolves + // the same files (see #53051). + lobBasePath := storage.SegmentPartitionBasePath(t.compactionParams.StorageConfig.GetRootPath(), t.collectionID, t.partitionID) textColumnConfigs := t.lobContext.GetTextColumnConfigs( lobBasePath, t.compactionParams.TextInlineThreshold, @@ -1088,6 +1097,25 @@ func (t *clusteringCompactionTask) initLOBCompactionContext(ctx context.Context) // compute strategies (will use forced REWRITE_ALL for all TEXT fields) t.lobContext.ComputeStrategies(textFieldIDs, t.compactionParams.LOBHoleRatioThreshold) + sourceManifests := make(map[int64]string) + for _, segment := range t.plan.GetSegmentBinlogs() { + if segment.GetManifest() != "" { + sourceManifests[segment.GetSegmentID()] = segment.GetManifest() + } + } + outputPartitionBase := storage.SegmentPartitionBasePath( + t.compactionParams.StorageConfig.GetRootPath(), t.collectionID, t.partitionID) + partitionBaseMismatch, err := compaction.LOBSourcePartitionBaseMismatch(sourceManifests, outputPartitionBase) + if err != nil { + return err + } + if partitionBaseMismatch { + t.lobContext.ForceRewriteAllAcrossPartitionBases(textFieldIDs) + mlog.Info(ctx, "forcing source-side TEXT LOB decoding across partition namespaces", + mlog.Int64("planID", t.GetPlanID()), + mlog.String("outputPartitionBase", outputPartitionBase), + mlog.Int64s("textFieldIDs", textFieldIDs)) + } // log strategy decisions for fieldID, decision := range t.lobContext.Decisions { diff --git a/internal/datanode/compactor/clustering_compactor_storage_v2_test.go b/internal/datanode/compactor/clustering_compactor_storage_v2_test.go index 2d17da7bce..b00aa6f62d 100644 --- a/internal/datanode/compactor/clustering_compactor_storage_v2_test.go +++ b/internal/datanode/compactor/clustering_compactor_storage_v2_test.go @@ -20,6 +20,7 @@ import ( "context" "fmt" "os" + "path/filepath" "testing" "time" @@ -57,9 +58,10 @@ func (s *ClusteringCompactionTaskStorageV2Suite) SetupTest() { } func (s *ClusteringCompactionTaskStorageV2Suite) TearDownTest() { - os.RemoveAll(paramtable.Get().LocalStorageCfg.Path.GetValue() + "insert_log") - os.RemoveAll(paramtable.Get().LocalStorageCfg.Path.GetValue() + "delta_log") - os.RemoveAll(paramtable.Get().LocalStorageCfg.Path.GetValue() + "stats_log") + localStoragePath := paramtable.Get().LocalStorageCfg.Path.GetValue() + os.RemoveAll(filepath.Join(localStoragePath, "insert_log")) + os.RemoveAll(filepath.Join(localStoragePath, "delta_log")) + os.RemoveAll(filepath.Join(localStoragePath, "stats_log")) s.ClusteringCompactionTaskSuite.TearDownTest() } diff --git a/internal/datanode/compactor/clustering_compactor_storage_v3_test.go b/internal/datanode/compactor/clustering_compactor_storage_v3_test.go index 0fb6633ba3..86c1ce44be 100644 --- a/internal/datanode/compactor/clustering_compactor_storage_v3_test.go +++ b/internal/datanode/compactor/clustering_compactor_storage_v3_test.go @@ -26,6 +26,7 @@ import ( "time" "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" "github.com/milvus-io/milvus-proto/go-api/v3/commonpb" @@ -55,6 +56,36 @@ func TestClusteringCompactionTaskStorageV3Suite(t *testing.T) { suite.Run(t, new(ClusteringCompactionTaskStorageV3Suite)) } +func TestClusteringInitLOBCompactionContextDecodesAcrossPartitionBases(t *testing.T) { + paramtable.Get().Init(paramtable.NewBaseTable()) + params := compaction.GenParams() + task := &clusteringCompactionTask{ + collectionID: 10, + partitionID: 20, + plan: &datapb.CompactionPlan{ + Type: datapb.CompactionType_ClusteringCompaction, + Schema: &schemapb.CollectionSchema{Fields: []*schemapb.FieldSchema{ + {FieldID: 101, Name: "text", DataType: schemapb.DataType_Text}, + }}, + SegmentBinlogs: []*datapb.CompactionSegmentBinlogs{{ + SegmentID: 1, + Manifest: packed.MarshalManifestPath( + params.StorageConfig.GetRootPath()+"/files/insert_log/10/20/1", 1), + }}, + }, + compactionParams: params, + } + + require.NoError(t, task.initLOBCompactionContext(context.Background())) + require.NotNil(t, task.lobContext) + require.True(t, task.lobContext.ShouldRewriteAnyField()) + require.True(t, task.lobContext.DecodeTextFromSource) + sourceConfigs, err := task.lobContext.GetSourceTextColumnConfigs(task.plan.GetSegmentBinlogs()[0].GetManifest()) + require.NoError(t, err) + require.Len(t, sourceConfigs, 1) + require.Contains(t, sourceConfigs[0].LobBasePath, "/files/insert_log/10/20/lobs/101") +} + type ClusteringCompactionTaskStorageV3Suite struct { ClusteringCompactionTaskSuite } @@ -190,7 +221,7 @@ func (s *ClusteringCompactionTaskStorageV3Suite) initStorageV3Segments(rows int, bfs := pkoracle.NewBloomFilterSet() k := metautil.JoinIDPath(CollectionID, PartitionID, segmentID) - basePath := path.Join(common.SegmentInsertLogPath, k) + basePath := path.Join(rootPath, common.SegmentInsertLogPath, k) manifestPath := packed.MarshalManifestPath(basePath, packed.ManifestEarliest) seg := metacache.NewSegmentInfo(&datapb.SegmentInfo{ @@ -395,7 +426,7 @@ func (s *MixCompactionTaskStorageV3Suite) initStorageV3Segments(rows int, segmen bfs := pkoracle.NewBloomFilterSet() k := metautil.JoinIDPath(CollectionID, PartitionID, segmentID) - basePath := path.Join(common.SegmentInsertLogPath, k) + basePath := path.Join(rootPath, common.SegmentInsertLogPath, k) manifestPath := packed.MarshalManifestPath(basePath, packed.ManifestEarliest) seg := metacache.NewSegmentInfo(&datapb.SegmentInfo{ @@ -443,7 +474,7 @@ func (s *MixCompactionTaskStorageV3Suite) initTextLOBStorageV3Segment(rows int, bfs := pkoracle.NewBloomFilterSet() k := metautil.JoinIDPath(CollectionID, PartitionID, segmentID) - basePath := path.Join(common.SegmentInsertLogPath, k) + basePath := path.Join(rootPath, common.SegmentInsertLogPath, k) manifestPath := packed.MarshalManifestPath(basePath, packed.ManifestEarliest) seg := metacache.NewSegmentInfo(&datapb.SegmentInfo{ diff --git a/internal/datanode/compactor/compactor_common.go b/internal/datanode/compactor/compactor_common.go index 8ea67ad929..79c25cd5b3 100644 --- a/internal/datanode/compactor/compactor_common.go +++ b/internal/datanode/compactor/compactor_common.go @@ -235,6 +235,32 @@ func newCompactionSegmentRecordReader(ctx context.Context, segment *datapb.Compa return newCompactionSegmentRecordReaderWithFields(ctx, segment, schema, storageConfig, existingFields, opts...) } +func newTextDecodedCompactionSegmentRecordReader( + ctx context.Context, + segment *datapb.CompactionSegmentBinlogs, + schema *schemapb.CollectionSchema, + storageConfig *indexpb.StorageConfig, + textColumnConfigs []packed.TextColumnConfig, + opts ...storage.RwOption, +) (storage.RecordReader, map[int64]struct{}, error) { + if segment.GetManifest() == "" || len(textColumnConfigs) == 0 { + return newCompactionSegmentRecordReader(ctx, segment, schema, storageConfig, opts...) + } + existingFields, err := compactionSegmentStorageFields(segment, storageConfig) + if err != nil { + return nil, nil, err + } + readSchema := compactionReadSchema(schema, existingFields) + reader, err := storage.NewTextDecodedManifestRecordReader( + ctx, + segment.GetManifest(), + readSchema, + textColumnConfigs, + append(opts, storage.WithPresentFields(existingFields))..., + ) + return reader, existingFields, err +} + func compactionSegmentStorageFields(segment *datapb.CompactionSegmentBinlogs, storageConfig *indexpb.StorageConfig) (map[int64]struct{}, error) { if segment.GetManifest() != "" { return packed.GetManifestFieldIDs(segment.GetManifest(), storageConfig) diff --git a/internal/datanode/compactor/merge_sort.go b/internal/datanode/compactor/merge_sort.go index 06c789e9c2..2cc661a56a 100644 --- a/internal/datanode/compactor/merge_sort.go +++ b/internal/datanode/compactor/merge_sort.go @@ -72,7 +72,11 @@ func mergeSortMultipleSegments(ctx context.Context, segmentFilters := make([]compaction.EntityFilter, len(binlogs)) for i, s := range binlogs { - reader, existingFields, err := newCompactionSegmentRecordReader(ctx, s, plan.GetSchema(), compactionParams.StorageConfig, + textDecodeConfigs, err := lobContext.GetSourceTextColumnConfigs(s.GetManifest()) + if err != nil { + return nil, err + } + reader, existingFields, err := newTextDecodedCompactionSegmentRecordReader(ctx, s, plan.GetSchema(), compactionParams.StorageConfig, textDecodeConfigs, storage.WithCollectionID(collectionID), storage.WithDownloader(binlogIO.Download), storage.WithVersion(s.StorageVersion), diff --git a/internal/datanode/compactor/mix_compactor.go b/internal/datanode/compactor/mix_compactor.go index 687f4ef4f5..0ac4846efd 100644 --- a/internal/datanode/compactor/mix_compactor.go +++ b/internal/datanode/compactor/mix_compactor.go @@ -21,7 +21,6 @@ import ( "fmt" sio "io" "math" - "path" "time" "github.com/apache/arrow/go/v17/arrow/array" @@ -42,7 +41,6 @@ import ( "github.com/milvus-io/milvus/pkg/v3/proto/indexpb" "github.com/milvus-io/milvus/pkg/v3/util/funcutil" "github.com/milvus-io/milvus/pkg/v3/util/merr" - "github.com/milvus-io/milvus/pkg/v3/util/metautil" "github.com/milvus-io/milvus/pkg/v3/util/paramtable" "github.com/milvus-io/milvus/pkg/v3/util/timerecord" "github.com/milvus-io/milvus/pkg/v3/util/typeutil" @@ -174,9 +172,10 @@ func (t *mixCompactionTask) buildWriterOptions(ctx context.Context) []storage.Rw } if t.lobContext != nil && t.lobContext.ShouldRewriteAnyField() { - // LOB base path at partition level: {root}/insert_log/{coll}/{part} - lobBasePath := path.Join(t.compactionParams.StorageConfig.GetRootPath(), - common.SegmentInsertLogPath, metautil.JoinIDPath(t.collectionID, t.partitionID)) + // LOB base path at partition level, sharing the manifest root rule so + // the reader (which reconstructs lobs/ from the manifest base) resolves + // the same files (see #53051). + lobBasePath := storage.SegmentPartitionBasePath(t.compactionParams.StorageConfig.GetRootPath(), t.collectionID, t.partitionID) textColumnConfigs := t.lobContext.GetTextColumnConfigs( lobBasePath, t.compactionParams.TextInlineThreshold, @@ -289,7 +288,11 @@ func (t *mixCompactionTask) writeSegment(ctx context.Context, } entityFilter := compaction.NewEntityFilter(delta, t.plan.GetCollectionTtl(), t.currentTime, seg.GetCommitTimestamp()) - reader, existingFields, err := newCompactionSegmentRecordReader(ctx, seg, t.plan.GetSchema(), t.compactionParams.StorageConfig, + textDecodeConfigs, err := t.lobContext.GetSourceTextColumnConfigs(seg.GetManifest()) + if err != nil { + return 0, 0, err + } + reader, existingFields, err := newTextDecodedCompactionSegmentRecordReader(ctx, seg, t.plan.GetSchema(), t.compactionParams.StorageConfig, textDecodeConfigs, storage.WithCollectionID(t.collectionID), storage.WithDownloader(t.binlogIO.Download), storage.WithVersion(seg.GetStorageVersion()), @@ -776,6 +779,19 @@ func (t *mixCompactionTask) initLOBCompactionContext(ctx context.Context) error t.lobContext.SetCompactionType(datapb.CompactionType_MixCompaction, sourceSegmentCount, targetSegmentCount) t.lobContext.ComputeStrategies(textFieldIDs, t.compactionParams.LOBHoleRatioThreshold) + outputPartitionBase := storage.SegmentPartitionBasePath( + t.compactionParams.StorageConfig.GetRootPath(), t.collectionID, t.partitionID) + partitionBaseMismatch, err := compaction.LOBSourcePartitionBaseMismatch(sourceManifests, outputPartitionBase) + if err != nil { + return err + } + if partitionBaseMismatch { + t.lobContext.ForceRewriteAllAcrossPartitionBases(textFieldIDs) + mlog.Info(ctx, "forcing TEXT LOB rewrite across partition namespaces", + mlog.Int64("planID", t.GetPlanID()), + mlog.String("outputPartitionBase", outputPartitionBase), + mlog.Int64s("textFieldIDs", textFieldIDs)) + } // log strategy decisions for fieldID, decision := range t.lobContext.Decisions { diff --git a/internal/datanode/compactor/mix_compactor_storage_v2_test.go b/internal/datanode/compactor/mix_compactor_storage_v2_test.go index f3e629cf34..1a5e0e67d0 100644 --- a/internal/datanode/compactor/mix_compactor_storage_v2_test.go +++ b/internal/datanode/compactor/mix_compactor_storage_v2_test.go @@ -21,6 +21,7 @@ import ( "fmt" "math" "os" + "path/filepath" "testing" "time" @@ -67,12 +68,13 @@ func (s *MixCompactionTaskStorageV2Suite) SetupTest() { } func (s *MixCompactionTaskStorageV2Suite) TearDownTest() { + localStoragePath := paramtable.Get().LocalStorageCfg.Path.GetValue() + os.RemoveAll(filepath.Join(localStoragePath, "insert_log")) + os.RemoveAll(filepath.Join(localStoragePath, "delta_log")) + os.RemoveAll(filepath.Join(localStoragePath, "stats_log")) paramtable.Get().Reset(paramtable.Get().CommonCfg.StorageType.Key) paramtable.Get().Reset(paramtable.Get().CommonCfg.UseLoonFFI.Key) paramtable.Get().Reset(paramtable.Get().LocalStorageCfg.Path.Key) - os.RemoveAll(paramtable.Get().LocalStorageCfg.Path.GetValue() + "insert_log") - os.RemoveAll(paramtable.Get().LocalStorageCfg.Path.GetValue() + "delta_log") - os.RemoveAll(paramtable.Get().LocalStorageCfg.Path.GetValue() + "stats_log") initcore.CleanArrowFileSystem() } diff --git a/internal/datanode/compactor/mix_compactor_test.go b/internal/datanode/compactor/mix_compactor_test.go index ba3a853763..bcb5c71e9c 100644 --- a/internal/datanode/compactor/mix_compactor_test.go +++ b/internal/datanode/compactor/mix_compactor_test.go @@ -59,7 +59,12 @@ func TestMixCompactionTaskSuite(t *testing.T) { func TestMixInitLOBCompactionContextKeepsReuseAllDecisionWithoutLobFiles(t *testing.T) { paramtable.Get().Init(paramtable.NewBaseTable()) textFieldIDs := []int64{101, 102} + params := compaction.GenParams() + collectionID, partitionID := int64(10), int64(20) + partitionBase := storage.SegmentPartitionBasePath(params.StorageConfig.GetRootPath(), collectionID, partitionID) task := &mixCompactionTask{ + collectionID: collectionID, + partitionID: partitionID, plan: &datapb.CompactionPlan{ Type: datapb.CompactionType_MixCompaction, Schema: &schemapb.CollectionSchema{Fields: []*schemapb.FieldSchema{ @@ -67,11 +72,11 @@ func TestMixInitLOBCompactionContextKeepsReuseAllDecisionWithoutLobFiles(t *test {FieldID: textFieldIDs[1], Name: "text_2", DataType: schemapb.DataType_Text}, }}, SegmentBinlogs: []*datapb.CompactionSegmentBinlogs{ - {SegmentID: 1, Manifest: "manifest-1"}, - {SegmentID: 2, Manifest: "manifest-2"}, + {SegmentID: 1, Manifest: packed.MarshalManifestPath(partitionBase+"/1", 1)}, + {SegmentID: 2, Manifest: packed.MarshalManifestPath(partitionBase+"/2", 1)}, }, }, - compactionParams: compaction.GenParams(), + compactionParams: params, estimatedOutputSegmentCount: 1, } collectPatch := mockey.Mock(compaction.CollectLobFilesFromManifests).Return(map[int64][]packed.LobFileInfo{ @@ -92,6 +97,41 @@ func TestMixInitLOBCompactionContextKeepsReuseAllDecisionWithoutLobFiles(t *test } } +func TestMixInitLOBCompactionContextRewritesAcrossPartitionBases(t *testing.T) { + paramtable.Get().Init(paramtable.NewBaseTable()) + params := compaction.GenParams() + collectionID, partitionID := int64(10), int64(20) + task := &mixCompactionTask{ + collectionID: collectionID, + partitionID: partitionID, + plan: &datapb.CompactionPlan{ + Type: datapb.CompactionType_MixCompaction, + Schema: &schemapb.CollectionSchema{Fields: []*schemapb.FieldSchema{ + {FieldID: 101, Name: "text", DataType: schemapb.DataType_Text}, + }}, + SegmentBinlogs: []*datapb.CompactionSegmentBinlogs{{ + SegmentID: 1, + Manifest: packed.MarshalManifestPath( + params.StorageConfig.GetRootPath()+"/files/insert_log/10/20/1", 1), + }}, + }, + compactionParams: params, + estimatedOutputSegmentCount: 1, + } + collectPatch := mockey.Mock(compaction.CollectLobFilesFromManifests).Return( + map[int64][]packed.LobFileInfo{1: {{FieldID: 101, TotalRows: 1, ValidRows: 1}}}, nil, + ).Build() + defer collectPatch.UnPatch() + + err := task.initLOBCompactionContext(context.Background()) + assert.NoError(t, err) + if assert.NotNil(t, task.lobContext) { + assert.False(t, task.lobContext.HasReuseAllFields()) + assert.True(t, task.lobContext.ShouldRewriteAnyField()) + assert.Equal(t, compaction.LOBStrategyRewriteAll, task.lobContext.GetStrategy(101)) + } +} + func newMixCompactionStorageV1SuiteForDirectTest(t *testing.T) *MixCompactionTaskStorageV1Suite { s := &MixCompactionTaskStorageV1Suite{} s.SetT(t) diff --git a/internal/datanode/compactor/sort_compaction.go b/internal/datanode/compactor/sort_compaction.go index e62a9eff39..9bce4917d7 100644 --- a/internal/datanode/compactor/sort_compaction.go +++ b/internal/datanode/compactor/sort_compaction.go @@ -182,6 +182,19 @@ func (t *sortCompactionTask) sortSegment(ctx context.Context) (*datapb.Compactio storage.WithUseLoonFFI(t.useLoonFFI), storage.WithWriterFormat(t.compactionParams.GetStorageFormat()), } + if t.lobContext != nil && t.lobContext.ShouldRewriteAnyField() { + lobBasePath := storage.SegmentPartitionBasePath( + t.compactionParams.StorageConfig.GetRootPath(), t.collectionID, t.partitionID) + textColumnConfigs := t.lobContext.GetTextColumnConfigs( + lobBasePath, + t.compactionParams.TextInlineThreshold, + t.compactionParams.TextMaxLobFileBytes, + t.compactionParams.TextFlushThresholdBytes, + ) + if len(textColumnConfigs) > 0 { + writerOpts = append(writerOpts, storage.WithTextColumnConfigs(textColumnConfigs)) + } + } if t.lobContext != nil && t.lobContext.HasReuseAllFields() { writerOpts = append(writerOpts, storage.WithTextRefsAsBinary()) } @@ -249,7 +262,12 @@ func (t *sortCompactionTask) sortSegment(ctx context.Context) (*datapb.Compactio } phaseStart = time.Now() - rr, existingFields, err := newCompactionSegmentRecordReader(ctx, t.plan.GetSegmentBinlogs()[0], t.plan.Schema, t.compactionParams.StorageConfig, + textDecodeConfigs, err := t.lobContext.GetSourceTextColumnConfigs(t.manifest) + if err != nil { + srw.Close() + return nil, err + } + rr, existingFields, err := newTextDecodedCompactionSegmentRecordReader(ctx, t.plan.GetSegmentBinlogs()[0], t.plan.Schema, t.compactionParams.StorageConfig, textDecodeConfigs, storage.WithVersion(t.segmentStorageVersion), storage.WithDownloader(t.binlogIO.Download), storage.WithStorageConfig(t.compactionParams.StorageConfig), @@ -563,9 +581,9 @@ func (t *sortCompactionTask) createTextIndex(ctx context.Context, } // initLOBCompactionContext initializes the LOB compaction context for TEXT columns. -// For sort compaction, data is reordered but not redistributed, so TEXT columns -// use REUSE_ALL strategy (LOB references remain valid after reordering). -// The LOB file references need to be copied to the output segment's manifest. +// Sort compaction normally reuses TEXT references because data is reordered but +// not redistributed. If the output uses a different partition base, references +// must instead be rewritten because they do not encode their source namespace. func (t *sortCompactionTask) initLOBCompactionContext(ctx context.Context) error { // check if there are TEXT fields in schema textFieldIDs := compaction.GetTEXTFieldIDsFromSchema(t.plan.GetSchema()) @@ -616,12 +634,28 @@ func (t *sortCompactionTask) initLOBCompactionContext(ctx context.Context) error // compute strategies (will use forced REUSE_ALL for all TEXT fields) t.lobContext.ComputeStrategies(textFieldIDs, t.compactionParams.LOBHoleRatioThreshold) + outputPartitionBase := storage.SegmentPartitionBasePath( + t.compactionParams.StorageConfig.GetRootPath(), t.collectionID, t.partitionID) + partitionBaseMismatch, err := compaction.LOBSourcePartitionBaseMismatch(sourceManifests, outputPartitionBase) + if err != nil { + return err + } + if partitionBaseMismatch { + t.lobContext.ForceRewriteAllAcrossPartitionBases(textFieldIDs) + log.Info(ctx, "forcing TEXT LOB rewrite across partition namespaces", + mlog.String("outputPartitionBase", outputPartitionBase)) + } // log strategy decisions for fieldID, decision := range t.lobContext.Decisions { log.Info(ctx, "LOB compaction strategy decided", mlog.FieldFieldID(fieldID), - mlog.String("strategy", "REUSE_ALL"), + mlog.String("strategy", func() string { + if decision.Strategy == compaction.LOBStrategyRewriteAll { + return "REWRITE_ALL" + } + return "REUSE_ALL" + }()), mlog.Bool("isForced", t.lobContext.IsForced), mlog.Float64("holeRatio", decision.OverallHoleRatio), ) diff --git a/internal/datanode/compactor/sort_compaction_test.go b/internal/datanode/compactor/sort_compaction_test.go index 5e581dc36e..0442859a0d 100644 --- a/internal/datanode/compactor/sort_compaction_test.go +++ b/internal/datanode/compactor/sort_compaction_test.go @@ -50,9 +50,14 @@ func TestSortCompactionTaskSuite(t *testing.T) { func TestSortInitLOBCompactionContextKeepsReuseAllDecisionWithoutLobFiles(t *testing.T) { paramtable.Get().Init(paramtable.NewBaseTable()) textFieldIDs := []int64{101, 102} + params := compaction.GenParams() + collectionID, partitionID := int64(10), int64(20) task := &sortCompactionTask{ - segmentID: 100, - manifest: "manifest-100", + collectionID: collectionID, + partitionID: partitionID, + segmentID: 100, + manifest: packed.MarshalManifestPath( + storage.SegmentManifestBasePath(params.StorageConfig.GetRootPath(), collectionID, partitionID, 100), 1), plan: &datapb.CompactionPlan{ Type: datapb.CompactionType_SortCompaction, Schema: &schemapb.CollectionSchema{Fields: []*schemapb.FieldSchema{ @@ -60,7 +65,7 @@ func TestSortInitLOBCompactionContextKeepsReuseAllDecisionWithoutLobFiles(t *tes {FieldID: textFieldIDs[1], Name: "text_2", DataType: schemapb.DataType_Text}, }}, }, - compactionParams: compaction.GenParams(), + compactionParams: params, } collectPatch := mockey.Mock(compaction.CollectLobFilesFromManifests).Return(map[int64][]packed.LobFileInfo{ 100: {}, @@ -79,6 +84,37 @@ func TestSortInitLOBCompactionContextKeepsReuseAllDecisionWithoutLobFiles(t *tes } } +func TestSortInitLOBCompactionContextRewritesAcrossPartitionBases(t *testing.T) { + paramtable.Get().Init(paramtable.NewBaseTable()) + params := compaction.GenParams() + task := &sortCompactionTask{ + collectionID: 10, + partitionID: 20, + segmentID: 100, + manifest: packed.MarshalManifestPath( + params.StorageConfig.GetRootPath()+"/files/insert_log/10/20/100", 1), + plan: &datapb.CompactionPlan{ + Type: datapb.CompactionType_SortCompaction, + Schema: &schemapb.CollectionSchema{Fields: []*schemapb.FieldSchema{ + {FieldID: 101, Name: "text", DataType: schemapb.DataType_Text}, + }}, + }, + compactionParams: params, + } + collectPatch := mockey.Mock(compaction.CollectLobFilesFromManifests).Return( + map[int64][]packed.LobFileInfo{100: {{FieldID: 101, TotalRows: 1, ValidRows: 1}}}, nil, + ).Build() + defer collectPatch.UnPatch() + + err := task.initLOBCompactionContext(context.Background()) + assert.NoError(t, err) + if assert.NotNil(t, task.lobContext) { + assert.False(t, task.lobContext.HasReuseAllFields()) + assert.True(t, task.lobContext.ShouldRewriteAnyField()) + assert.Equal(t, compaction.LOBStrategyRewriteAll, task.lobContext.GetStrategy(101)) + } +} + type SortCompactionTaskSuite struct { suite.Suite diff --git a/internal/datanode/external/manager_test.go b/internal/datanode/external/manager_test.go index d3ee523ad8..7c37a5f074 100644 --- a/internal/datanode/external/manager_test.go +++ b/internal/datanode/external/manager_test.go @@ -18,6 +18,7 @@ package external import ( "context" + "path/filepath" "sync/atomic" "testing" "time" @@ -26,6 +27,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/milvus-io/milvus-proto/go-api/v3/schemapb" "github.com/milvus-io/milvus/pkg/v3/proto/datapb" "github.com/milvus-io/milvus/pkg/v3/proto/indexpb" ) @@ -189,6 +191,56 @@ func TestExternalCollectionManager_SubmitTask_Failure(t *testing.T) { assert.Equal(t, expectedError.Error(), info.FailReason) } +func TestExternalCollectionManager_MissingExploreManifestFailsTask(t *testing.T) { + for _, name := range []string{"legacy_relative", "absolute"} { + t.Run(name, func(t *testing.T) { + ctx := t.Context() + root := t.TempDir() + manifest := filepath.Join("__explore_temp__", filepath.Base(filepath.Dir(root)), "_metadata", "manifest-1.avro") + if name == "absolute" { + manifest = filepath.Join(root, manifest) + } + req := &datapb.RefreshExternalCollectionTaskRequest{ + TaskID: 1, + CollectionID: 100, + Schema: &schemapb.CollectionSchema{}, + StorageConfig: &indexpb.StorageConfig{StorageType: "local", RootPath: root}, + ExternalSource: filepath.Join(root, "input"), + ExternalSpec: `{"format":"parquet"}`, + ExploreManifestPath: manifest, + PreAllocatedSegmentIds: &datapb.IDRange{Begin: 1000, End: 1010}, + TargetRowsPerSegment: 100, + } + manager := NewExternalCollectionManager(ctx, 1) + defer manager.Close() + err := manager.SubmitTask("test-cluster", req, func(taskCtx context.Context) (*datapb.RefreshExternalCollectionTaskResponse, error) { + task := NewRefreshExternalCollectionTask(taskCtx, req) + if err := task.PreExecute(taskCtx); err != nil { + return nil, err + } + if err := task.Execute(taskCtx); err != nil { + return nil, err + } + return &datapb.RefreshExternalCollectionTaskResponse{State: indexpb.JobState_JobStateFinished}, nil + }) + require.NoError(t, err) + + // Exercise the real manifest reader: even if the native not-found + // error is classified as transient, this task must terminate. + require.Eventually(t, func() bool { + info := manager.Get("test-cluster", req.GetTaskID()) + return info != nil && info.State == indexpb.JobState_JobStateFailed + }, 5*time.Second, 10*time.Millisecond) + info := manager.Get("test-cluster", req.GetTaskID()) + assert.Contains(t, info.FailReason, "failed to fetch fragments") + assert.Contains(t, info.FailReason, "failed to read explore manifest") + assert.Contains(t, info.FailReason, "manifest-1.avro") + assert.Contains(t, info.FailReason, "File not found") + assert.Empty(t, info.UpdatedSegments) + }) + } +} + // Regression for #49225: a panic inside taskFunc (e.g. divide-by-zero from a // malformed external parquet) must be isolated to the task — the manager pool // goroutine must NOT crash the process, and the task must surface as Failed. diff --git a/internal/datanode/external/task_update.go b/internal/datanode/external/task_update.go index e7141524ed..c21357c4fa 100644 --- a/internal/datanode/external/task_update.go +++ b/internal/datanode/external/task_update.go @@ -42,7 +42,6 @@ package external import ( "context" "fmt" - "path" "sort" "strconv" "strings" @@ -62,7 +61,6 @@ import ( "github.com/milvus-io/milvus/pkg/v3/util/conc" "github.com/milvus-io/milvus/pkg/v3/util/externalspec" "github.com/milvus-io/milvus/pkg/v3/util/merr" - "github.com/milvus-io/milvus/pkg/v3/util/metautil" "github.com/milvus-io/milvus/pkg/v3/util/paramtable" "github.com/milvus-io/milvus/pkg/v3/util/timerecord" "github.com/milvus-io/milvus/pkg/v3/util/typeutil" @@ -1061,8 +1059,8 @@ func (t *RefreshExternalCollectionTask) createManifestForSegment( ) (string, error) { // All segments now use final paths with real IDs (no temporary paths needed) // Pre-allocated IDs ensure we can write directly to final locations - basePath := segmentInsertLogBasePath( - t.req.GetStorageConfig(), + basePath := storage.SegmentManifestBasePath( + t.req.GetStorageConfig().GetRootPath(), t.req.GetCollectionID(), t.req.GetPartitionID(), segmentID, @@ -1091,19 +1089,6 @@ func (t *RefreshExternalCollectionTask) createManifestForSegment( return manifestPath, nil } -func segmentInsertLogBasePath( - storageConfig *indexpb.StorageConfig, - collectionID int64, - partitionID int64, - segmentID int64, -) string { - rootPath := "" - if storageConfig != nil { - rootPath = storageConfig.GetRootPath() - } - return path.Join(rootPath, common.SegmentInsertLogPath, metautil.JoinIDPath(collectionID, partitionID, segmentID)) -} - // hasFunctions returns true if the schema defines any functions. func (t *RefreshExternalCollectionTask) hasFunctions() bool { return len(t.req.GetSchema().GetFunctions()) > 0 @@ -1117,8 +1102,8 @@ func (t *RefreshExternalCollectionTask) createManifestWithFunctions( segmentID int64, fragments []packed.Fragment, ) (string, error) { - basePath := segmentInsertLogBasePath( - t.req.GetStorageConfig(), + basePath := storage.SegmentManifestBasePath( + t.req.GetStorageConfig().GetRootPath(), t.req.GetCollectionID(), t.req.GetPartitionID(), segmentID, diff --git a/internal/datanode/importv2/copy_segment_utils.go b/internal/datanode/importv2/copy_segment_utils.go index 4330141e3d..1798b07460 100644 --- a/internal/datanode/importv2/copy_segment_utils.go +++ b/internal/datanode/importv2/copy_segment_utils.go @@ -20,6 +20,7 @@ import ( "context" "net/url" "path" + "slices" "strconv" "strings" "time" @@ -101,6 +102,10 @@ func transformManifestPath( return "", merr.Wrap(err, "failed to unmarshal manifest path") } + // The manifest stores the complete ChunkManager key on every backend (an + // absolute path under localStorage.path for local, a bucket-relative key + // under minio.rootPath for remote), so the transformed physical path is + // the new manifest base as-is. targetBasePath, err := generateTargetPath(basePath, source, target) if err != nil { return "", merr.Wrap(err, "failed to generate target base path") @@ -244,9 +249,14 @@ func collectSegmentFiles( if err != nil { return nil, merr.Wrapf(err, "failed to unmarshal manifest path %q for segment %d", manifestPath, source.GetSegmentId()) } - basePath = snapshotstorage.NormalizeSnapshotObjectPath(basePath) + // The manifest base is the complete ChunkManager key on every backend; + // only foreign snapshot URIs need their object key extracted. + walkBasePath := snapshotstorage.NormalizeSnapshotObjectPath(basePath) + if strings.TrimSpace(walkBasePath) == "" { + return nil, merr.WrapErrDataIntegrityMsg("storage v3 segment %d has an empty manifest base path", source.GetSegmentId()) + } - allFiles, listErr := listAllFiles(ctx, sourceCM, basePath) + allFiles, listErr := listAllFiles(ctx, sourceCM, walkBasePath) if listErr != nil { return nil, merr.Wrapf(listErr, "failed to list files from manifest base path %q for segment %d", basePath, source.GetSegmentId()) } @@ -272,8 +282,8 @@ func collectSegmentFiles( if lobErr != nil { return nil, merr.Wrapf(lobErr, "failed to collect LOB files from manifest for segment %d", source.GetSegmentId()) } else if len(lobFileInfos) > 0 { - // GetManifestLobFiles returns absolute paths (the manifest - // deserializer calls ToAbsolute internally), so use them directly. + // Manifest::ToAbsolutePaths expands entries against the manifest + // base, which is already the complete ChunkManager key. files.LobFiles = lobFileInfosToPaths(lobFileInfos) mlog.Info(ctx, "collected LOB files from segment manifest", mlog.String("manifestPath", manifestPath), @@ -435,23 +445,31 @@ func CopySegmentAndIndexFiles( mlog.Info(ctx, "all files copied successfully", mlog.Int("fileCount", len(mappings))) - // Step 3.5: When manifest is used (StorageV3+), InsertBinlogs were collected from manifest - // (actual file paths under base_path including _data/ and _metadata/), but - // generateSegmentInfoFromSource needs mappings for the protobuf logical paths too. - // Add these "logical-only" mappings AFTER file copying so they don't trigger actual copy operations. + // Step 3.5: When manifest is used (StorageV3+), physical paths were + // collected from the manifest, but generateSegmentInfoFromSource still + // needs mappings for the protobuf paths (which for V3 are summaries whose + // LogPath may be empty or a manifest-internal key). Add those "logical-only" + // mappings after physical copying so they never trigger a copy. if useManifest { - pbInsertPaths := extractFromPb(source.GetInsertBinlogs()) - for _, srcPath := range pbInsertPaths { - if _, exists := mappings[srcPath]; !exists { + logicalPathGroups := [][]string{ + extractFromPb(source.GetInsertBinlogs()), + extractFromPb(source.GetDeltaBinlogs()), + extractFromPb(source.GetStatsBinlogs()), + extractFromPb(source.GetBm25Binlogs()), + } + logicalPathCount := 0 + for _, logicalPaths := range logicalPathGroups { + logicalPathCount += len(logicalPaths) + for _, srcPath := range logicalPaths { dstPath, pathErr := generateTargetPath(srcPath, source, target) if pathErr != nil { - return nil, copiedFiles, merr.Wrapf(pathErr, "failed to generate target path for pb insert binlog %s", srcPath) + return nil, copiedFiles, merr.Wrapf(pathErr, "failed to generate target path for protobuf binlog %s", srcPath) } mappings[srcPath] = dstPath } } mlog.Info(ctx, "added logical insert binlog mappings for manifest segment", - mlog.Int("pbPathCount", len(pbInsertPaths))) + mlog.Int("pbPathCount", logicalPathCount)) } // Step 4: Build index metadata from source @@ -776,19 +794,29 @@ func splitIndexArtifactPaths(filePaths []string) (string, []string, error) { // Parameters: // - srcFieldBinlogs: Source field binlogs with original paths // - mappings: Pre-calculated map of source path -> target path -// - countRows: If true, accumulate total row count from EntriesNum (for insert logs only) -// - isExternalTable: If true, skip path mapping because external table insert -// binlogs carry row metadata without physical log paths +// - opts: see transformFieldBinlogOptions // // Returns: // - []*datapb.FieldBinlog: Transformed binlog list with target paths // - int64: Total row count (sum of EntriesNum from all binlogs if countRows=true, 0 otherwise) // - error: Non-nil if any source path has no mapping (fail-fast on missing mappings) +// +// transformFieldBinlogOptions controls how one binlog family is transformed. +type transformFieldBinlogOptions struct { + // countRows accumulates total row count from EntriesNum (insert logs only). + countRows bool + // isExternalTable skips path mapping: external table insert binlogs carry + // row metadata without physical log paths. + isExternalTable bool + // preservePathless keeps metadata-only summaries that have no physical + // LogPath (StorageV3 insert and delta summaries). + preservePathless bool +} + func transformFieldBinlogs( srcFieldBinlogs []*datapb.FieldBinlog, mappings map[string]string, - countRows bool, - isExternalTable bool, + opts transformFieldBinlogOptions, ) ([]*datapb.FieldBinlog, int64, error) { result := make([]*datapb.FieldBinlog, 0, len(srcFieldBinlogs)) var totalRows int64 @@ -800,20 +828,23 @@ func transformFieldBinlogs( for _, srcBinlog := range srcFieldBinlog.GetBinlogs() { dstBinlog := proto.Clone(srcBinlog).(*datapb.Binlog) - if !isExternalTable { + if !opts.isExternalTable { srcPath := srcBinlog.GetLogPath() if srcPath == "" { - continue + if !opts.preservePathless { + continue + } + } else { + dstPath, ok := mappings[srcPath] + if !ok { + return nil, 0, merr.WrapErrServiceInternalMsg("no mapping found for source path: %s", srcPath) + } + dstBinlog.LogPath = dstPath } - dstPath, ok := mappings[srcPath] - if !ok { - return nil, 0, merr.WrapErrServiceInternalMsg("no mapping found for source path: %s", srcPath) - } - dstBinlog.LogPath = dstPath } dstFieldBinlog.Binlogs = append(dstFieldBinlog.Binlogs, dstBinlog) - if countRows { + if opts.countRows { totalRows += srcBinlog.GetEntriesNum() } } @@ -862,7 +893,13 @@ func generateSegmentInfoFromSource( } // Process insert binlogs (count rows) - binlogs, totalRows, err := transformFieldBinlogs(source.GetInsertBinlogs(), mappings, true, source.GetIsExternalCollection()) + isStorageV3 := source.GetStorageVersion() >= storage.StorageV3 + binlogs, totalRows, err := transformFieldBinlogs(source.GetInsertBinlogs(), mappings, + transformFieldBinlogOptions{ + countRows: true, + isExternalTable: source.GetIsExternalCollection(), + preservePathless: isStorageV3, + }) if err != nil { return nil, merr.Wrap(err, "failed to transform insert binlogs") } @@ -873,21 +910,21 @@ func generateSegmentInfoFromSource( } // Process stats binlogs (no row counting) - statslogs, _, err := transformFieldBinlogs(source.GetStatsBinlogs(), mappings, false, false) + statslogs, _, err := transformFieldBinlogs(source.GetStatsBinlogs(), mappings, transformFieldBinlogOptions{}) if err != nil { return nil, merr.Wrap(err, "failed to transform stats binlogs") } segmentInfo.Statslogs = statslogs // Process delta binlogs (no row counting) - deltalogs, _, err := transformFieldBinlogs(source.GetDeltaBinlogs(), mappings, false, false) + deltalogs, _, err := transformFieldBinlogs(source.GetDeltaBinlogs(), mappings, transformFieldBinlogOptions{preservePathless: isStorageV3}) if err != nil { return nil, merr.Wrap(err, "failed to transform delta binlogs") } segmentInfo.Deltalogs = deltalogs // Process BM25 binlogs (no row counting) - bm25logs, _, err := transformFieldBinlogs(source.GetBm25Binlogs(), mappings, false, false) + bm25logs, _, err := transformFieldBinlogs(source.GetBm25Binlogs(), mappings, transformFieldBinlogOptions{}) if err != nil { return nil, merr.Wrap(err, "failed to transform BM25 binlogs") } @@ -950,14 +987,29 @@ func remapSourceRootPath(sourcePath string, source *datapb.CopySegmentSource, ta } } - targetRootPath := strings.Trim(target.GetTargetRootPath(), "/") - if targetRootPath == "" { - return relativePath, nil + rawTargetRootPath := strings.TrimSpace(target.GetTargetRootPath()) + targetRootIsAbs := path.IsAbs(rawTargetRootPath) + targetRootPath := strings.Trim(rawTargetRootPath, "/") + var targetPath string + switch { + case targetRootPath == "": + targetPath = relativePath + case relativePath == "": + targetPath = targetRootPath + default: + targetPath = path.Join(targetRootPath, relativePath) } - if relativePath == "" { - return targetRootPath, nil + if targetRootIsAbs { + return "/" + targetPath, nil } - return path.Join(targetRootPath, relativePath), nil + return targetPath, nil +} + +// buildCanonicalTargetPath rebuilds the layout under the target root without +// carrying over the source storage prefix. Joining the root separately preserves +// the leading slash for local paths; empty/dot roots address the remote bucket. +func buildCanonicalTargetPath(parts []string, layoutIndex int, target *datapb.CopySegmentTarget) string { + return path.Join(strings.TrimSpace(target.GetTargetRootPath()), path.Join(parts[layoutIndex:]...)) } // generateTargetPath converts source file path to target path by replacing collection/partition/segment IDs @@ -970,7 +1022,7 @@ func generateTargetPath(sourcePath string, source *datapb.CopySegmentSource, tar return "", err } - // Convert IDs to strings for replacement + // Convert IDs to strings for replacement. targetCollectionIDStr := strconv.FormatInt(target.GetCollectionId(), 10) targetPartitionIDStr := strconv.FormatInt(target.GetPartitionId(), 10) targetSegmentIDStr := strconv.FormatInt(target.GetSegmentId(), 10) @@ -978,8 +1030,7 @@ func generateTargetPath(sourcePath string, source *datapb.CopySegmentSource, tar // Split path into parts parts := strings.Split(sourcePath, "/") - // Find the log type index (insert_log, delta_log, stats_log, bm25_stats) - // Path structure: .../log_type/collectionID/partitionID/segmentID/... + // Find the log layout keyword. logTypeIndex := -1 for i, part := range parts { if part == BinlogTypeInsert || part == BinlogTypeDelta || part == BinlogTypeStats || part == BinlogTypeBM25 { @@ -1001,7 +1052,7 @@ func generateTargetPath(sourcePath string, source *datapb.CopySegmentSource, tar parts[logTypeIndex+2] = targetPartitionIDStr parts[logTypeIndex+3] = targetSegmentIDStr - return path.Join(parts...), nil + return buildCanonicalTargetPath(parts, logTypeIndex, target), nil } // generateTargetLOBPath replaces collection and partition IDs in a LOB file path. @@ -1016,13 +1067,7 @@ func generateTargetLOBPath(sourcePath string, source *datapb.CopySegmentSource, parts := strings.Split(sourcePath, "/") - logTypeIndex := -1 - for i, part := range parts { - if part == BinlogTypeInsert { - logTypeIndex = i - break - } - } + logTypeIndex := slices.Index(parts, BinlogTypeInsert) // Path: .../{insert_log}/{coll}/{part}/lobs/... // Need at least logTypeIndex + 2 (coll and part) after insert_log @@ -1033,7 +1078,7 @@ func generateTargetLOBPath(sourcePath string, source *datapb.CopySegmentSource, parts[logTypeIndex+1] = strconv.FormatInt(target.GetCollectionId(), 10) parts[logTypeIndex+2] = strconv.FormatInt(target.GetPartitionId(), 10) - return path.Join(parts...), nil + return buildCanonicalTargetPath(parts, logTypeIndex, target), nil } // buildIndexInfoFromSource builds complete index metadata from source information. @@ -1166,10 +1211,10 @@ func buildIndexInfoFromSource( // File Type Constants // ============================================================================ -// lobFileInfosToPaths extracts absolute file paths from LobFileInfo structs. -// GetManifestLobFiles returns paths that have already been resolved to absolute -// form by the C++ manifest deserializer (Manifest::ToAbsolutePaths), so we use -// them directly without any path concatenation. +// lobFileInfosToPaths extracts manifest-expanded paths from LobFileInfo structs. +// The manifest deserializer (Manifest::ToAbsolutePaths) expands entries against +// the manifest base, which is the complete ChunkManager key on every backend, +// so the paths are used as-is. func lobFileInfosToPaths(infos []packed.LobFileInfo) []string { paths := make([]string, 0, len(infos)) for _, info := range infos { @@ -1238,11 +1283,7 @@ func generateTargetIndexPath( return "", err } - // Split path into parts - parts := strings.Split(sourcePath, "/") - - // Determine keyword and offsets based on index type - var keywordIdx int + // Determine keyword and offsets based on index type. var collectionOffset, partitionOffset, segmentOffset int keyword := indexType @@ -1252,19 +1293,6 @@ func generateTargetIndexPath( keyword = IndexTypeVectorScalarV1 } - // Find the keyword position in the path - keywordIdx = -1 - for i, part := range parts { - if part == keyword { - keywordIdx = i - break - } - } - - if keywordIdx == -1 { - return "", merr.WrapErrServiceInternalMsg("keyword '%s' not found in path: %s", keyword, sourcePath) - } - // Set offsets based on index type // collectionOffset = -1 means collectionID is not present in the path var buildIDOffset int @@ -1298,10 +1326,11 @@ func generateTargetIndexPath( indexType, IndexTypeVectorScalarV0, IndexTypeText, IndexTypeJSONKey, IndexTypeJSONStats) } - // Validate path structure has enough components - if keywordIdx+segmentOffset >= len(parts) { - return "", merr.WrapErrParameterInvalidMsg("invalid %s path structure: %s (expected '%s' with at least %d components after it)", - indexType, sourcePath, indexType, segmentOffset+1) + parts := strings.Split(sourcePath, "/") + keywordIdx := slices.Index(parts, keyword) + if keywordIdx == -1 || keywordIdx+segmentOffset >= len(parts) { + return "", merr.WrapErrParameterInvalidMsg("invalid %s path structure: %s (expected '%s' layout)", + indexType, sourcePath, keyword) } // Replace buildID if a mapping exists in target.NewBuildIds @@ -1323,7 +1352,7 @@ func generateTargetIndexPath( parts[keywordIdx+partitionOffset] = strconv.FormatInt(target.GetPartitionId(), 10) parts[keywordIdx+segmentOffset] = strconv.FormatInt(target.GetSegmentId(), 10) - return path.Join(parts...), nil + return buildCanonicalTargetPath(parts, keywordIdx, target), nil } // ============================================================================ @@ -1385,56 +1414,52 @@ func shortenIndexFilePaths(fullPaths []string) []string { func shortenJSONStatsPath(jsonStats map[int64]*datapb.JsonKeyStats) map[int64]*datapb.JsonKeyStats { for _, stats := range jsonStats { for i, file := range stats.GetFiles() { - stats.Files[i] = shortenSingleJSONStatsPath(file) + stats.Files[i] = shortenJSONStatsFilePath(file, stats) } } return jsonStats } -// shortenSingleJSONStatsPath shortens a single JSON stats file path. -// -// This function extracts the relative path from a full JSON stats file path by: -// 1. Finding "shared_key_index" or "shredding_data" keywords and extracting from that position -// 2. For files directly under fieldID directory (e.g., meta.json), extracting everything after -// the 7 path components following "json_stats" +// shortenJSONStatsFilePath extracts the path after the fieldID component of a +// JSON-stats layout matching stats. The caller only passes non-nil stats that +// contain files; nil map entries are skipped by shortenJSONStatsPath. // // Path format: {root}/json_stats/{dataFormat}/{buildID}/{version}/{collID}/{partID}/{segID}/{fieldID}/... -// -// Path examples: -// - Input: "files/json_stats/2/123/1/444/555/666/100/shared_key_index/inverted_index_0" -// Output: "shared_key_index/inverted_index_0" -// - Input: "files/json_stats/2/123/1/444/555/666/100/shredding_data/parquet_data_0" -// Output: "shredding_data/parquet_data_0" -// - Input: "files/json_stats/2/123/1/444/555/666/100/meta.json" -// Output: "meta.json" -// - Input: "shared_key_index/inverted_index_0" (already shortened) -// Output: "shared_key_index/inverted_index_0" (idempotent) -// - Input: "meta.json" (already shortened) -// Output: "meta.json" (idempotent) -// -// Parameters: -// - fullPath: Full or partial JSON stats file path -// -// Returns: -// - Shortened path relative to fieldID directory -func shortenSingleJSONStatsPath(fullPath string) string { - // Find "shared_key_index" in path - if idx := strings.Index(fullPath, jsonStatsSharedIndexPath); idx != -1 { - return fullPath[idx:] - } - // Find "shredding_data" in path - if idx := strings.Index(fullPath, jsonStatsShreddingDataPath); idx != -1 { - return fullPath[idx:] +func shortenJSONStatsFilePath(fullPath string, stats *datapb.JsonKeyStats) string { + parts := strings.Split(fullPath, "/") + if parts[0] == jsonStatsSharedIndexPath || parts[0] == jsonStatsShreddingDataPath { + return path.Join(parts...) } - // Handle files directly under fieldID directory (e.g., meta.json) - // Path format: .../json_stats/{dataFormat}/{build}/{ver}/{coll}/{part}/{seg}/{field}/filename - // json_stats is followed by 7 components, the 8th onwards is the file path - parts := strings.Split(fullPath, "/") + // Find the last structurally valid JSON-stats layout. Root prefixes may + // themselves contain json_stats/shared_key_index, so substring or first-key + // matching is not safe. Also match the layout's data format, build, version, + // and field components against the stats metadata. + anchor := -1 for i, part := range parts { - if part == common.JSONStatsPath && i+8 < len(parts) { - return path.Join(parts[i+8:]...) + if part != common.JSONStatsPath || i+8 >= len(parts) { + continue } + valid := true + for offset := 1; offset <= 7; offset++ { + if _, err := strconv.ParseInt(parts[i+offset], 10, 64); err != nil { + valid = false + break + } + } + if !valid { + continue + } + if parts[i+1] != strconv.FormatInt(stats.GetJsonKeyStatsDataFormat(), 10) || + parts[i+2] != strconv.FormatInt(stats.GetBuildID(), 10) || + parts[i+3] != strconv.FormatInt(stats.GetVersion(), 10) || + parts[i+7] != strconv.FormatInt(stats.GetFieldID(), 10) { + continue + } + anchor = i + } + if anchor >= 0 { + return path.Join(parts[anchor+8:]...) } // If already shortened or no json_stats found, return as-is diff --git a/internal/datanode/importv2/copy_segment_utils_test.go b/internal/datanode/importv2/copy_segment_utils_test.go index 19e6af707e..52abe1f357 100644 --- a/internal/datanode/importv2/copy_segment_utils_test.go +++ b/internal/datanode/importv2/copy_segment_utils_test.go @@ -20,6 +20,7 @@ import ( "context" "fmt" "path" + "path/filepath" "strings" "testing" "time" @@ -35,6 +36,7 @@ import ( "github.com/milvus-io/milvus/internal/storage" "github.com/milvus-io/milvus/internal/storagev2/packed" "github.com/milvus-io/milvus/pkg/v3/common" + "github.com/milvus-io/milvus/pkg/v3/objectstorage" "github.com/milvus-io/milvus/pkg/v3/proto/datapb" "github.com/milvus-io/milvus/pkg/v3/proto/indexpb" "github.com/milvus-io/milvus/pkg/v3/util/paramtable" @@ -120,12 +122,14 @@ func TestGenerateTargetPath(t *testing.T) { SegmentId: 333, } target := &datapb.CopySegmentTarget{ - CollectionId: 444, - PartitionId: 555, - SegmentId: 666, + TargetRootPath: "files", + CollectionId: 444, + PartitionId: 555, + SegmentId: 666, } tests := []struct { + targetRoot string name string sourcePath string wantPath string @@ -161,10 +165,17 @@ func TestGenerateTargetPath(t *testing.T) { wantPath: "files/insert_log/444/555/666/_stats/bm25.100/12345", wantErr: false, }, + { + name: "local complete filesystem path", + targetRoot: "/var/lib/milvus/data", + sourcePath: "/var/lib/milvus/data/insert_log/111/222/333/_data/0", + wantPath: "/var/lib/milvus/data/insert_log/444/555/666/_data/0", + wantErr: false, + }, { name: "external component in regular binlog root is not external table", sourcePath: "files/external/insert_log/111/222/333/100/log1.log", - wantPath: "files/external/insert_log/444/555/666/100/log1.log", + wantPath: "files/insert_log/444/555/666/100/log1.log", wantErr: false, }, { @@ -183,6 +194,10 @@ func TestGenerateTargetPath(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + target.TargetRootPath = tt.targetRoot + if target.TargetRootPath == "" { + target.TargetRootPath = "files" + } gotPath, err := generateTargetPath(tt.sourcePath, source, target) if tt.wantErr { assert.Error(t, err) @@ -204,9 +219,10 @@ func TestGenerateMappingsFromFiles_ExternalTable(t *testing.T) { IsExternalCollection: true, } target := &datapb.CopySegmentTarget{ - CollectionId: 444, - PartitionId: 555, - SegmentId: 666, + TargetRootPath: "files", + CollectionId: 444, + PartitionId: 555, + SegmentId: 666, } files := &SegmentFiles{ InsertBinlogs: []string{"files/insert_log/111/222/333/_metadata/manifest.json"}, @@ -261,6 +277,70 @@ func TestCollectSegmentFiles_UsesSourceStorageConfigForV3(t *testing.T) { assert.Equal(t, []string{"foreign-root/files/insert_log/100/1/lobs/101/_data/0"}, files.LobFiles) } +// On local storage the manifest base, the LOB paths the manifest expands and +// the protobuf paths are all complete filesystem paths; they are walked and +// copied as-is, with no root joined anywhere. +func TestCollectSegmentFiles_LocalV3ManifestKeysAreCompletePaths(t *testing.T) { + root := t.TempDir() + sourceCM := storage.NewLocalChunkManager(objectstorage.RootPath(root)) + sourceCfg := &indexpb.StorageConfig{ + StorageType: "local", + RootPath: root, + } + basePath := path.Join(root, "insert_log/100/1/10") + manifestPath := packed.MarshalManifestPath(basePath, 2) + source := &datapb.CopySegmentSource{ + CollectionId: 100, + PartitionId: 1, + SegmentId: 10, + StorageVersion: storage.StorageV3, + ManifestPath: manifestPath, + DeltaBinlogs: []*datapb.FieldBinlog{{ + Binlogs: []*datapb.Binlog{{LogPath: path.Join(basePath, "_delta/7")}}, + }}, + } + + mList := mockey.Mock(listAllFiles).To( + func(_ context.Context, cm storage.ChunkManager, gotBasePath string) ([]string, error) { + assert.Same(t, sourceCM, cm) + assert.Equal(t, basePath, gotBasePath) + return []string{path.Join(basePath, "_data/0")}, nil + }).Build() + defer mList.UnPatch() + + mLob := mockey.Mock(packed.GetManifestLobFiles).To( + func(gotManifestPath string, cfg *indexpb.StorageConfig) ([]packed.LobFileInfo, error) { + assert.Equal(t, manifestPath, gotManifestPath) + assert.Equal(t, root, cfg.GetRootPath()) + return []packed.LobFileInfo{{ + Path: path.Join(root, "insert_log/100/1/lobs/101/_data/0"), + }}, nil + }).Build() + defer mLob.UnPatch() + + files, err := collectSegmentFiles(context.Background(), sourceCM, sourceCfg, source) + require.NoError(t, err) + assert.Equal(t, []string{path.Join(basePath, "_data/0")}, files.InsertBinlogs) + assert.Equal(t, []string{path.Join(root, "insert_log/100/1/lobs/101/_data/0")}, files.LobFiles) + assert.Equal(t, []string{path.Join(basePath, "_delta/7")}, files.DeltaBinlogs) +} + +func TestCollectSegmentFiles_LocalV3RejectsEmptyManifestBase(t *testing.T) { + root := t.TempDir() + _, err := collectSegmentFiles( + context.Background(), + storage.NewLocalChunkManager(objectstorage.RootPath(root)), + &indexpb.StorageConfig{StorageType: "local", RootPath: root}, + &datapb.CopySegmentSource{ + SegmentId: 10, + StorageVersion: storage.StorageV3, + ManifestPath: packed.MarshalManifestPath("", 1), + }, + ) + require.Error(t, err) + assert.Contains(t, err.Error(), "empty manifest base path") +} + func TestCollectSegmentFiles_NormalizesV3ManifestURIForListing(t *testing.T) { sourceCM := &struct{ storage.ChunkManager }{} basePath := "s3://foreign-source/foreign-root/files/insert_log/100/1/10" @@ -315,7 +395,7 @@ func TestGenerateTargetPath_RemapExternalRoot(t *testing.T) { ) assert.NoError(t, err) - assert.Equal(t, "target-cluster-root/files/insert_log/444/555/666/100/log1.log", gotPath) + assert.Equal(t, "target-cluster-root/insert_log/444/555/666/100/log1.log", gotPath) } func TestGenerateTargetPath_RemapExternalS3Root(t *testing.T) { @@ -339,7 +419,31 @@ func TestGenerateTargetPath_RemapExternalS3Root(t *testing.T) { ) assert.NoError(t, err) - assert.Equal(t, "target-cluster-root/files/insert_log/444/555/666/100/log1.log", gotPath) + assert.Equal(t, "target-cluster-root/insert_log/444/555/666/100/log1.log", gotPath) +} + +func TestGenerateTargetPath_RemapExternalRootToLocalCompletePath(t *testing.T) { + source := &datapb.CopySegmentSource{ + CollectionId: 111, + PartitionId: 222, + SegmentId: 333, + SourceRootPath: "s3://bucket/source-cluster-root", + } + target := &datapb.CopySegmentTarget{ + CollectionId: 444, + PartitionId: 555, + SegmentId: 666, + TargetRootPath: "/var/lib/milvus/data", + } + + gotPath, err := generateTargetPath( + "source-cluster-root/files/insert_log/111/222/333/100/log1.log", + source, + target, + ) + + require.NoError(t, err) + assert.Equal(t, "/var/lib/milvus/data/insert_log/444/555/666/100/log1.log", gotPath) } func TestGenerateTargetPath_RemapEndpointStyleExternalRoot(t *testing.T) { @@ -388,7 +492,7 @@ func TestGenerateTargetPath_RemapEndpointStyleExternalRoot(t *testing.T) { gotPath, err := generateTargetPath(tt.sourcePath, source, target) assert.NoError(t, err) - assert.Equal(t, "target-cluster-root/files/insert_log/444/555/666/100/log1.log", gotPath) + assert.Equal(t, "target-cluster-root/insert_log/444/555/666/100/log1.log", gotPath) }) } } @@ -414,7 +518,7 @@ func TestGenerateTargetPath_RemapExternalS3SourcePath(t *testing.T) { ) assert.NoError(t, err) - assert.Equal(t, "target-cluster-root/files/insert_log/444/555/666/100/log1.log", gotPath) + assert.Equal(t, "target-cluster-root/insert_log/444/555/666/100/log1.log", gotPath) } func TestGenerateTargetPath_RejectsPathOutsideExternalRoot(t *testing.T) { @@ -457,11 +561,16 @@ func TestGenerateTargetPath_RemapExternalBucketRoot(t *testing.T) { { name: "configured target root", targetRoot: "target-cluster-root", - want: "target-cluster-root/files/insert_log/444/555/666/100/log1.log", + want: "target-cluster-root/insert_log/444/555/666/100/log1.log", }, { name: "target bucket root", - want: "files/insert_log/444/555/666/100/log1.log", + want: "insert_log/444/555/666/100/log1.log", + }, + { + name: "dot target bucket root", + targetRoot: ".", + want: "insert_log/444/555/666/100/log1.log", }, } @@ -486,6 +595,20 @@ func TestGenerateTargetPath_RemapExternalBucketRoot(t *testing.T) { } } +func TestGenerateTargetPath_InternalCopyBucketRoot(t *testing.T) { + source := &datapb.CopySegmentSource{CollectionId: 111, PartitionId: 222, SegmentId: 333} + for _, root := range []string{"", "."} { + t.Run(fmt.Sprintf("root_%q", root), func(t *testing.T) { + target := &datapb.CopySegmentTarget{ + CollectionId: 444, PartitionId: 555, SegmentId: 666, TargetRootPath: root, + } + got, err := generateTargetPath("insert_log/111/222/333/_data/0", source, target) + require.NoError(t, err) + assert.Equal(t, "insert_log/444/555/666/_data/0", got) + }) + } +} + func TestGenerateTargetPath_AllowsEquivalentExternalSchemes(t *testing.T) { tests := []struct { name string @@ -522,7 +645,7 @@ func TestGenerateTargetPath_AllowsEquivalentExternalSchemes(t *testing.T) { gotPath, err := generateTargetPath(tt.sourcePath, source, target) require.NoError(t, err) - assert.Equal(t, "target-cluster-root/files/insert_log/444/555/666/100/log1.log", gotPath) + assert.Equal(t, "target-cluster-root/insert_log/444/555/666/100/log1.log", gotPath) }) } } @@ -581,12 +704,14 @@ func TestGenerateTargetLOBPath(t *testing.T) { SegmentId: 333, } target := &datapb.CopySegmentTarget{ - CollectionId: 444, - PartitionId: 555, - SegmentId: 666, + TargetRootPath: "files", + CollectionId: 444, + PartitionId: 555, + SegmentId: 666, } tests := []struct { + targetRoot string name string sourcePath string wantPath string @@ -600,10 +725,18 @@ func TestGenerateTargetLOBPath(t *testing.T) { }, { name: "LOB file with nested field path", + targetRoot: "root/data", sourcePath: "root/data/insert_log/111/222/lobs/200/_data/xyz.vx", wantPath: "root/data/insert_log/444/555/lobs/200/_data/xyz.vx", wantErr: false, }, + { + name: "local complete filesystem path", + targetRoot: "/var/lib/milvus/data", + sourcePath: "/var/lib/milvus/data/insert_log/111/222/lobs/200/_data/xyz.vx", + wantPath: "/var/lib/milvus/data/insert_log/444/555/lobs/200/_data/xyz.vx", + wantErr: false, + }, { name: "invalid path - no insert_log", sourcePath: "files/other/111/222/lobs/100/_data/abc.vx", @@ -620,6 +753,10 @@ func TestGenerateTargetLOBPath(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + target.TargetRootPath = tt.targetRoot + if target.TargetRootPath == "" { + target.TargetRootPath = "files" + } gotPath, err := generateTargetLOBPath(tt.sourcePath, source, target) if tt.wantErr { assert.Error(t, err) @@ -652,7 +789,7 @@ func TestGenerateTargetLOBPath_RemapExternalRoot(t *testing.T) { ) assert.NoError(t, err) - assert.Equal(t, "target-cluster-root/files/insert_log/444/555/lobs/100/_data/abc123.vx", gotPath) + assert.Equal(t, "target-cluster-root/insert_log/444/555/lobs/100/_data/abc123.vx", gotPath) } func TestLobFileInfosToPaths(t *testing.T) { @@ -740,8 +877,8 @@ func TestCollectSegmentFiles_LOBFromManifest(t *testing.T) { assert.NotNil(t, files) // Verify LOB files come from manifest, not from directory listing. - // GetManifestLobFiles returns absolute paths (after ToAbsolutePaths in C++), - // so lobFileInfosToPaths uses them directly. + // The remote manifest base already contains its object root, so the + // manifest-expanded LOB paths are complete remote object keys. assert.Equal(t, 2, len(files.LobFiles)) assert.Equal(t, "root/insert_log/100/200/lobs/300/_data/seg1_file1.vx", files.LobFiles[0]) assert.Equal(t, "root/insert_log/100/200/lobs/300/_data/seg1_file2.vx", files.LobFiles[1]) @@ -758,12 +895,14 @@ func TestGenerateTargetIndexPath(t *testing.T) { SegmentId: 333, } target := &datapb.CopySegmentTarget{ - CollectionId: 444, - PartitionId: 555, - SegmentId: 666, + TargetRootPath: "files", + CollectionId: 444, + PartitionId: 555, + SegmentId: 666, } tests := []struct { + targetRoot string name string sourcePath string indexType string @@ -786,6 +925,14 @@ func TestGenerateTargetIndexPath(t *testing.T) { wantPath: "files/text_log/123/1/444/555/666/100/index_file", wantErr: false, }, + { + name: "local complete filesystem path", + targetRoot: "/var/lib/milvus/data", + sourcePath: "/var/lib/milvus/data/text_log/123/1/111/222/333/100/index_file", + indexType: IndexTypeText, + wantPath: "/var/lib/milvus/data/text_log/123/1/444/555/666/100/index_file", + wantErr: false, + }, { name: "json key index path", sourcePath: "files/json_key_index_log/123/1/111/222/333/100/index_file", @@ -840,6 +987,10 @@ func TestGenerateTargetIndexPath(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + target.TargetRootPath = tt.targetRoot + if target.TargetRootPath == "" { + target.TargetRootPath = "files" + } gotPath, err := generateTargetIndexPath(tt.sourcePath, source, target, tt.indexType, tt.pathVersion) if tt.wantErr { assert.Error(t, err) @@ -861,10 +1012,11 @@ func TestGenerateTargetIndexPath_BuildIDMapping(t *testing.T) { SegmentId: 333, } target := &datapb.CopySegmentTarget{ - CollectionId: 444, - PartitionId: 555, - SegmentId: 666, - NewBuildIds: map[int64]int64{1001: 2001}, + TargetRootPath: "files", + CollectionId: 444, + PartitionId: 555, + SegmentId: 666, + NewBuildIds: map[int64]int64{1001: 2001}, } tests := []struct { @@ -932,10 +1084,11 @@ func TestGenerateMappingsFromFiles_VectorScalarUsesSourcePathVersion(t *testing. }, } target := &datapb.CopySegmentTarget{ - CollectionId: 444, - PartitionId: 555, - SegmentId: 666, - NewBuildIds: map[int64]int64{1001: 2001, 1002: 2002}, + TargetRootPath: "files", + CollectionId: 444, + PartitionId: 555, + SegmentId: 666, + NewBuildIds: map[int64]int64{1001: 2001, 1002: 2002}, } files := &SegmentFiles{ VectorScalarIndex: []string{ @@ -973,7 +1126,7 @@ func TestGenerateTargetIndexPath_RemapExternalRoot(t *testing.T) { ) assert.NoError(t, err) - assert.Equal(t, "target-cluster-root/files/text_log/123/1/444/555/666/100/index_file", gotPath) + assert.Equal(t, "target-cluster-root/text_log/123/1/444/555/666/100/index_file", gotPath) } func TestTransformFieldBinlogs(t *testing.T) { @@ -1012,7 +1165,7 @@ func TestTransformFieldBinlogs(t *testing.T) { } t.Run("count rows for insert logs", func(t *testing.T) { - result, totalRows, err := transformFieldBinlogs(srcFieldBinlogs, mappings, true, false) + result, totalRows, err := transformFieldBinlogs(srcFieldBinlogs, mappings, transformFieldBinlogOptions{countRows: true}) assert.NoError(t, err) assert.Equal(t, int64(3000), totalRows) assert.Equal(t, 2, len(result)) @@ -1032,7 +1185,7 @@ func TestTransformFieldBinlogs(t *testing.T) { }) t.Run("no row counting for stats logs", func(t *testing.T) { - result, totalRows, err := transformFieldBinlogs(srcFieldBinlogs, mappings, false, false) + result, totalRows, err := transformFieldBinlogs(srcFieldBinlogs, mappings, transformFieldBinlogOptions{}) assert.NoError(t, err) assert.Equal(t, int64(0), totalRows) assert.Equal(t, 2, len(result)) @@ -1050,7 +1203,7 @@ func TestTransformFieldBinlogs(t *testing.T) { }, }, } - result, _, err := transformFieldBinlogs(srcWithEmpty, mappings, false, false) + result, _, err := transformFieldBinlogs(srcWithEmpty, mappings, transformFieldBinlogOptions{}) assert.NoError(t, err) assert.Equal(t, 0, len(result)) }) @@ -1067,7 +1220,7 @@ func TestTransformFieldBinlogs(t *testing.T) { }, }, } - result, totalRows, err := transformFieldBinlogs(srcWithUnmapped, mappings, true, false) + result, totalRows, err := transformFieldBinlogs(srcWithUnmapped, mappings, transformFieldBinlogOptions{countRows: true}) assert.Error(t, err) assert.Contains(t, err.Error(), "no mapping found") assert.Nil(t, result) @@ -1403,9 +1556,10 @@ func TestCopySegmentAndIndexFiles_ReturnsFileList(t *testing.T) { }, } target := &datapb.CopySegmentTarget{ - CollectionId: 444, - PartitionId: 555, - SegmentId: 666, + TargetRootPath: "files", + CollectionId: 444, + PartitionId: 555, + SegmentId: 666, } result, copiedFiles, err := CopySegmentAndIndexFiles( @@ -1454,9 +1608,10 @@ func TestCopySegmentAndIndexFiles_ReturnsFileList(t *testing.T) { }, } target := &datapb.CopySegmentTarget{ - CollectionId: 444, - PartitionId: 555, - SegmentId: 666, + TargetRootPath: "files", + CollectionId: 444, + PartitionId: 555, + SegmentId: 666, } result, copiedFiles, err := CopySegmentAndIndexFiles( @@ -1484,7 +1639,7 @@ func TestCopySegmentAndIndexFiles_NormalizesProviderSourceURI(t *testing.T) { copier := newCopySegmentCopierMock(t, func(_ context.Context, srcBucket, srcObject, dstBucket, dstObject string) error { assert.Equal(t, "source-bucket", srcBucket) assert.Equal(t, "target-bucket", dstBucket) - assert.Equal(t, "target-root/files/insert_log/444/555/666/1/10001", dstObject) + assert.Equal(t, "target-root/insert_log/444/555/666/1/10001", dstObject) copiedSource = srcObject return nil }) @@ -1581,9 +1736,10 @@ func TestGenerateTargetIndexPath_VectorScalarPreservesIDs(t *testing.T) { SegmentId: 333, } target := &datapb.CopySegmentTarget{ - CollectionId: 444, - PartitionId: 555, - SegmentId: 666, + TargetRootPath: "files", + CollectionId: 444, + PartitionId: 555, + SegmentId: 666, } // Verify buildID (1001) and indexVersion (1) are preserved, NOT replaced @@ -1852,7 +2008,7 @@ func TestBuildIndexInfoFromSource_UnmappedPaths(t *testing.T) { } func TestTransformFieldBinlogs_NilInput(t *testing.T) { - result, totalRows, err := transformFieldBinlogs(nil, map[string]string{}, true, false) + result, totalRows, err := transformFieldBinlogs(nil, map[string]string{}, transformFieldBinlogOptions{countRows: true}) assert.NoError(t, err) assert.Equal(t, int64(0), totalRows) assert.Equal(t, 0, len(result)) @@ -1876,7 +2032,7 @@ func TestTransformFieldBinlogs_MultipleBinlogsPerField(t *testing.T) { }, } - result, totalRows, err := transformFieldBinlogs(srcFieldBinlogs, mappings, true, false) + result, totalRows, err := transformFieldBinlogs(srcFieldBinlogs, mappings, transformFieldBinlogOptions{countRows: true}) assert.NoError(t, err) assert.Equal(t, int64(600), totalRows) assert.Equal(t, 1, len(result)) @@ -1899,7 +2055,7 @@ func TestTransformFieldBinlogs_UnmappedPath(t *testing.T) { }, } - _, _, err := transformFieldBinlogs(srcFieldBinlogs, mappings, true, false) + _, _, err := transformFieldBinlogs(srcFieldBinlogs, mappings, transformFieldBinlogOptions{countRows: true}) assert.Error(t, err) assert.Contains(t, err.Error(), "no mapping found for source path") } @@ -1976,7 +2132,7 @@ func TestShortenIndexFilePaths(t *testing.T) { } } -func TestShortenSingleJsonStatsPath(t *testing.T) { +func TestShortenJsonStatsPath_Files(t *testing.T) { tests := []struct { name string inputPath string @@ -2007,12 +2163,74 @@ func TestShortenSingleJsonStatsPath(t *testing.T) { inputPath: "files/other/path/file.json", expectedPath: "files/other/path/file.json", }, + { + name: "already shortened - meta.json", + inputPath: "meta.json", + expectedPath: "meta.json", + }, + { + name: "full path - meta.json", + inputPath: "files/json_stats/2/123/1/444/555/666/100/meta.json", + expectedPath: "meta.json", + }, + { + name: "full path - nested file", + inputPath: "files/json_stats/2/123/1/444/555/666/100/subdir/file.dat", + expectedPath: "subdir/file.dat", + }, + { + name: "local absolute path", + inputPath: "/var/lib/milvus/data/json_stats/2/123/1/444/555/666/100/meta.json", + expectedPath: "meta.json", + }, + { + name: "reserved names in root", + inputPath: "archive/json_stats/shared_key_index/files/json_stats/2/123/1/444/555/666/100/shared_key_index/index", + expectedPath: "shared_key_index/index", + }, + { + name: "shared key component in root", + inputPath: "archive/shared_key_index/files/json_stats/2/123/1/444/555/666/100/meta.json", + expectedPath: "meta.json", + }, + { + name: "shared key substring is not an anchor", + inputPath: "archive/not_shared_key_index/file", + expectedPath: "archive/not_shared_key_index/file", + }, + { + name: "data format mismatch", + inputPath: "files/json_stats/3/123/1/444/555/666/100/meta.json", + expectedPath: "files/json_stats/3/123/1/444/555/666/100/meta.json", + }, + { + name: "build ID mismatch", + inputPath: "files/json_stats/2/456/1/444/555/666/100/meta.json", + expectedPath: "files/json_stats/2/456/1/444/555/666/100/meta.json", + }, + { + name: "version mismatch", + inputPath: "files/json_stats/2/123/2/444/555/666/100/meta.json", + expectedPath: "files/json_stats/2/123/2/444/555/666/100/meta.json", + }, + { + name: "field ID mismatch", + inputPath: "files/json_stats/2/123/1/444/555/666/200/meta.json", + expectedPath: "files/json_stats/2/123/1/444/555/666/200/meta.json", + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - result := shortenSingleJSONStatsPath(tt.inputPath) - assert.Equal(t, tt.expectedPath, result) + stats := &datapb.JsonKeyStats{ + FieldID: 100, + Version: 1, + BuildID: 123, + JsonKeyStatsDataFormat: 2, + Files: []string{tt.inputPath}, + } + result := shortenJSONStatsPath(map[int64]*datapb.JsonKeyStats{100: stats}) + assert.Equal(t, []string{tt.expectedPath}, result[100].GetFiles()) }) } } @@ -2093,28 +2311,19 @@ func TestShortenJsonStatsPath_MetaJson(t *testing.T) { assert.Equal(t, "meta.json", result[102].Files[0]) } -func TestShortenSingleJsonStatsPath_EdgeCases(t *testing.T) { - t.Run("already_shortened_meta", func(t *testing.T) { - result := shortenSingleJSONStatsPath("meta.json") - assert.Equal(t, "meta.json", result) - }) +func TestShortenJsonStatsPath_NilEntries(t *testing.T) { + assert.Nil(t, shortenJSONStatsPath(nil)) + assert.Empty(t, shortenJSONStatsPath(map[int64]*datapb.JsonKeyStats{})) - t.Run("already_shortened_shared_key", func(t *testing.T) { - result := shortenSingleJSONStatsPath("shared_key_index/inverted_index_0") - assert.Equal(t, "shared_key_index/inverted_index_0", result) - }) - - t.Run("full_path_meta_json", func(t *testing.T) { - fullPath := "files/json_stats/2/123/1/444/555/666/100/meta.json" - result := shortenSingleJSONStatsPath(fullPath) - assert.Equal(t, "meta.json", result) - }) - - t.Run("full_path_nested_file", func(t *testing.T) { - fullPath := "files/json_stats/2/123/1/444/555/666/100/subdir/file.dat" - result := shortenSingleJSONStatsPath(fullPath) - assert.Equal(t, "subdir/file.dat", result) - }) + jsonStats := map[int64]*datapb.JsonKeyStats{ + 100: nil, + 200: {}, + } + result := shortenJSONStatsPath(jsonStats) + require.Len(t, result, 2) + assert.Nil(t, result[100]) + assert.Same(t, jsonStats[200], result[200]) + assert.Nil(t, result[200].GetFiles()) } func TestTransformManifestPath(t *testing.T) { @@ -2129,7 +2338,11 @@ func TestTransformManifestPath(t *testing.T) { 2, ) - source := &datapb.CopySegmentSource{} + source := &datapb.CopySegmentSource{ + CollectionId: 449104612037410004, + PartitionId: 449104621518610066, + SegmentId: 449104621518610065, + } targetManifest, err := transformManifestPath(sourceManifest, source, target) assert.NoError(t, err) @@ -2141,15 +2354,58 @@ func TestTransformManifestPath(t *testing.T) { assert.Contains(t, basePath, "2001") } +// Restoring from an external snapshot into local storage produces a manifest +// base in the target's canonical layout. The source's legacy files prefix +// must not survive below TargetRootPath. +func TestTransformManifestPath_ExternalSourceToLocalProducesCompletePath(t *testing.T) { + target := &datapb.CopySegmentTarget{ + SegmentId: 666, + CollectionId: 444, + PartitionId: 555, + TargetRootPath: "/var/lib/milvus/data", + } + source := &datapb.CopySegmentSource{ + CollectionId: 111, + PartitionId: 222, + SegmentId: 333, + SourceRootPath: "s3://source-bucket/source-root", + } + sourceManifest := packed.MarshalManifestPath( + "source-root/files/insert_log/111/222/333", + 2, + ) + + targetManifest, err := transformManifestPath(sourceManifest, source, target) + require.NoError(t, err) + basePath, version, err := packed.UnmarshalManifestPath(targetManifest) + require.NoError(t, err) + assert.Equal(t, int64(2), version) + assert.Equal(t, "/var/lib/milvus/data/insert_log/444/555/666", basePath) + + physicalPath, err := generateTargetPath( + "source-root/files/insert_log/111/222/333/_data/0", + source, + target, + ) + require.NoError(t, err) + assert.Equal(t, "/var/lib/milvus/data/insert_log/444/555/666/_data/0", physicalPath) +} + func TestTransformManifestPath_ExternalTable(t *testing.T) { target := &datapb.CopySegmentTarget{ - SegmentId: 666, - CollectionId: 444, - PartitionId: 555, + TargetRootPath: "files", + SegmentId: 666, + CollectionId: 444, + PartitionId: 555, } sourceManifest := packed.MarshalManifestPath("files/insert_log/111/222/333", 2) - source := &datapb.CopySegmentSource{IsExternalCollection: true} + source := &datapb.CopySegmentSource{ + CollectionId: 111, + PartitionId: 222, + SegmentId: 333, + IsExternalCollection: true, + } targetManifest, err := transformManifestPath(sourceManifest, source, target) assert.NoError(t, err) @@ -2167,7 +2423,12 @@ func TestTransformManifestPath_LegacyExternalTableLayoutUnsupported(t *testing.T } sourceManifest := packed.MarshalManifestPath("external/111/segments/333", 2) - source := &datapb.CopySegmentSource{IsExternalCollection: true} + source := &datapb.CopySegmentSource{ + CollectionId: 111, + PartitionId: 222, + SegmentId: 333, + IsExternalCollection: true, + } targetManifest, err := transformManifestPath(sourceManifest, source, target) assert.Error(t, err) assert.Empty(t, targetManifest) @@ -2188,7 +2449,7 @@ func TestTransformFieldBinlogs_SkipsPathMappingForExternalTable(t *testing.T) { }, }, } - got, totalRows, err := transformFieldBinlogs(src, nil, true, true) + got, totalRows, err := transformFieldBinlogs(src, nil, transformFieldBinlogOptions{countRows: true, isExternalTable: true}) assert.NoError(t, err) assert.Equal(t, int64(123), totalRows) assert.Len(t, got, 1) @@ -2200,7 +2461,7 @@ func TestTransformFieldBinlogs_SkipsPathMappingForExternalTable(t *testing.T) { assert.Equal(t, int64(456), got[0].GetBinlogs()[0].GetMemorySize()) } -func TestTransformFieldBinlogs_SkipsEmptyPathForInternalTable(t *testing.T) { +func TestTransformFieldBinlogs_DropsPathlessSummaryWhenNotPreserved(t *testing.T) { src := []*datapb.FieldBinlog{ { FieldID: 100, @@ -2210,12 +2471,38 @@ func TestTransformFieldBinlogs_SkipsEmptyPathForInternalTable(t *testing.T) { }, } - got, totalRows, err := transformFieldBinlogs(src, nil, false, false) + got, totalRows, err := transformFieldBinlogs(src, nil, transformFieldBinlogOptions{}) assert.NoError(t, err) assert.Zero(t, totalRows) assert.Empty(t, got) } +func TestTransformFieldBinlogs_PreservesPathlessSummary(t *testing.T) { + src := []*datapb.FieldBinlog{ + { + FieldID: 100, + Binlogs: []*datapb.Binlog{ + { + LogID: 10, + EntriesNum: 123, + MemorySize: 456, + TimestampFrom: 1000, + TimestampTo: 2000, + }, + }, + }, + } + + got, totalRows, err := transformFieldBinlogs(src, nil, transformFieldBinlogOptions{countRows: true, preservePathless: true}) + require.NoError(t, err) + require.Len(t, got, 1) + require.Len(t, got[0].GetBinlogs(), 1) + assert.Equal(t, int64(123), totalRows) + assert.Empty(t, got[0].GetBinlogs()[0].GetLogPath()) + assert.Equal(t, src[0].GetBinlogs()[0], got[0].GetBinlogs()[0]) + assert.NotSame(t, src[0].GetBinlogs()[0], got[0].GetBinlogs()[0]) +} + func TestGenerateSegmentInfoFromSource_PreservesExternalTableBinlogMetadata(t *testing.T) { source := &datapb.CopySegmentSource{ CollectionId: 111, @@ -2261,6 +2548,64 @@ func TestGenerateSegmentInfoFromSource_V3UsesDeclaredRowCount(t *testing.T) { assert.Equal(t, int64(4321), segmentInfo.GetImportedRows()) } +func TestGenerateSegmentInfoFromSource_V3PreservesPathlessSummaries(t *testing.T) { + source := &datapb.CopySegmentSource{ + StorageVersion: storage.StorageV3, + NumOfRows: 4321, + InsertBinlogs: []*datapb.FieldBinlog{ + { + FieldID: 10, + ChildFields: []int64{10}, + Format: "parquet", + Binlogs: []*datapb.Binlog{ + { + LogID: 1, + EntriesNum: 123, + MemorySize: 456, + TimestampFrom: 100, + TimestampTo: 200, + }, + }, + }, + }, + DeltaBinlogs: []*datapb.FieldBinlog{ + { + FieldID: 10, + Binlogs: []*datapb.Binlog{ + { + LogID: 2, + EntriesNum: 3, + MemorySize: 128, + TimestampFrom: 300, + TimestampTo: 400, + }, + }, + }, + }, + } + + segmentInfo, err := generateSegmentInfoFromSource( + source, + &datapb.CopySegmentTarget{SegmentId: 666}, + nil, + ) + require.NoError(t, err) + assert.Equal(t, int64(4321), segmentInfo.GetImportedRows()) + require.Len(t, segmentInfo.GetBinlogs(), 1) + require.Len(t, segmentInfo.GetDeltalogs(), 1) + assert.Equal(t, source.GetInsertBinlogs()[0], segmentInfo.GetBinlogs()[0]) + assert.Equal(t, source.GetDeltaBinlogs()[0], segmentInfo.GetDeltalogs()[0]) + + stats := storage.BuildStatsFromFieldBinlogs(segmentInfo.GetBinlogs(), nil, nil, segmentInfo.GetDeltalogs()) + assert.EqualValues(t, 456, stats.GetInsertBinlogSize()) + assert.EqualValues(t, 1, stats.GetInsertBinlogCount()) + assert.EqualValues(t, 128, stats.GetDeltaBinlogSize()) + assert.EqualValues(t, 3, stats.GetDeleteNumRows()) + assert.EqualValues(t, 1, stats.GetDeltaBinlogCount()) + assert.EqualValues(t, 300, stats.GetDeltaTimestampFrom()) + assert.EqualValues(t, 400, stats.GetDeltaTimestampTo()) +} + func TestCopySegmentAndIndexFiles_StorageV2CopiesManifest(t *testing.T) { sourceManifest := "files/insert_log/111/222/333/_metadata/manifest.json" sourceBinlog := "files/insert_log/111/222/333/100/1" @@ -2279,9 +2624,10 @@ func TestCopySegmentAndIndexFiles_StorageV2CopiesManifest(t *testing.T) { }}, } target := &datapb.CopySegmentTarget{ - CollectionId: 444, - PartitionId: 555, - SegmentId: 666, + TargetRootPath: "files", + CollectionId: 444, + PartitionId: 555, + SegmentId: 666, } copied := make(map[string]string) copier := newCopySegmentCopierMock(t, func(_ context.Context, _, src, _, dst string) error { @@ -2332,9 +2678,10 @@ func TestCopySegmentAndIndexFiles_ExternalTable(t *testing.T) { }, } target := &datapb.CopySegmentTarget{ - CollectionId: 444, - PartitionId: 555, - SegmentId: 666, + TargetRootPath: "files", + CollectionId: 444, + PartitionId: 555, + SegmentId: 666, } mList := mockey.Mock(listAllFiles).Return([]string{ @@ -2660,9 +3007,10 @@ func TestGenerateMappingsFromFiles(t *testing.T) { SegmentId: 333, } target := &datapb.CopySegmentTarget{ - CollectionId: 444, - PartitionId: 555, - SegmentId: 666, + TargetRootPath: "files", + CollectionId: 444, + PartitionId: 555, + SegmentId: 666, } t.Run("generate mappings for all file types", func(t *testing.T) { @@ -2753,9 +3101,10 @@ func TestCopySegmentAndIndexFiles_WithManifest(t *testing.T) { } target := &datapb.CopySegmentTarget{ - CollectionId: 444, - PartitionId: 555, - SegmentId: 666, + TargetRootPath: "files", + CollectionId: 444, + PartitionId: 555, + SegmentId: 666, } t.Run("successful copy with manifest", func(t *testing.T) { @@ -2902,7 +3251,7 @@ func TestCopySegmentAndIndexFiles_WithManifest(t *testing.T) { assert.Error(t, err) assert.Nil(t, result) - assert.Contains(t, err.Error(), "failed to generate target path for pb insert binlog") + assert.Contains(t, err.Error(), "failed to generate target path for protobuf binlog") }) } @@ -3051,6 +3400,241 @@ func TestListAllFiles(t *testing.T) { }) } +func TestCopySegmentAndIndexFiles_LocalStorageV3UsesCompletePaths(t *testing.T) { + mockNoManifestLobFiles(t) + + ctx := context.Background() + root := filepath.Join(t.TempDir(), "local", "root") + cm := storage.NewLocalChunkManager(objectstorage.RootPath(root)) + sourceKey := "insert_log/111/222/333/100/10001" + sourcePath := path.Join(root, sourceKey) + targetPath := path.Join(root, "insert_log/444/555/666/100/10001") + sourceDeltaKey := "insert_log/111/222/333/_delta/10002" + sourceDeltaPath := path.Join(root, sourceDeltaKey) + targetDeltaPath := path.Join(root, "insert_log/444/555/666/_delta/10002") + sourcePathlessDeltaKey := "insert_log/111/222/333/_delta/10003" + sourcePathlessDeltaPath := path.Join(root, sourcePathlessDeltaKey) + targetPathlessDeltaPath := path.Join(root, "insert_log/444/555/666/_delta/10003") + require.NoError(t, cm.Write(ctx, sourcePath, []byte("segment-data"))) + require.NoError(t, cm.Write(ctx, sourceDeltaPath, []byte("delta-data"))) + require.NoError(t, cm.Write(ctx, sourcePathlessDeltaPath, []byte("pathless-delta-data"))) + + source := &datapb.CopySegmentSource{ + CollectionId: 111, + PartitionId: 222, + SegmentId: 333, + StorageVersion: storage.StorageV3, + ManifestHasIndex: proto.Bool(false), + ManifestPath: packed.MarshalManifestPath(path.Join(root, "insert_log/111/222/333"), 1), + InsertBinlogs: []*datapb.FieldBinlog{{ + FieldID: 100, + Binlogs: []*datapb.Binlog{{ + LogPath: sourcePath, + EntriesNum: 10, + }}, + }}, + DeltaBinlogs: []*datapb.FieldBinlog{{ + Binlogs: []*datapb.Binlog{ + { + // Protobuf paths are the same complete keys as the physical + // files; step 3.5 maps them to the complete target path. + LogPath: sourceDeltaPath, + EntriesNum: 1, + }, + { + // V3 persists the physical delta in the manifest and keeps only + // this pathless summary in the protobuf metadata. + LogID: 10003, + EntriesNum: 2, + MemorySize: 128, + TimestampFrom: 300, + TimestampTo: 400, + }, + }, + }}, + } + target := &datapb.CopySegmentTarget{ + CollectionId: 444, + PartitionId: 555, + SegmentId: 666, + TargetRootPath: root, + } + storageConfig := &indexpb.StorageConfig{StorageType: "local", RootPath: root} + copySources := make(map[string]string) + copier := newCopySegmentCopierMock(t, func(ctx context.Context, _, src, _, dst string) error { + copySources[src] = dst + assert.True(t, path.IsAbs(src)) + assert.True(t, path.IsAbs(dst)) + return cm.Copy(ctx, src, dst) + }) + + result, copiedFiles, err := CopySegmentAndIndexFiles( + ctx, + cm, + storageConfig, + storageConfig, + copier, + "", + "", + source, + target, + nil, + ) + + require.NoError(t, err) + assert.Equal(t, map[string]string{ + sourcePath: targetPath, + sourceDeltaPath: targetDeltaPath, + sourcePathlessDeltaPath: targetPathlessDeltaPath, + }, copySources) + assert.ElementsMatch(t, []string{targetPath, targetDeltaPath, targetPathlessDeltaPath}, copiedFiles) + assert.Equal(t, packed.MarshalManifestPath(path.Join(root, "insert_log/444/555/666"), 1), result.GetManifestPath()) + require.Len(t, result.GetDeltalogs(), 1) + require.Len(t, result.GetDeltalogs()[0].GetBinlogs(), 2) + assert.Equal(t, int64(10002), result.GetDeltalogs()[0].GetBinlogs()[0].GetLogID()) + assert.Equal(t, int64(10003), result.GetDeltalogs()[0].GetBinlogs()[1].GetLogID()) + assert.Empty(t, result.GetDeltalogs()[0].GetBinlogs()[1].GetLogPath()) + assert.Equal(t, int64(2), result.GetDeltalogs()[0].GetBinlogs()[1].GetEntriesNum()) + assert.Equal(t, int64(128), result.GetDeltalogs()[0].GetBinlogs()[1].GetMemorySize()) + content, err := cm.Read(ctx, targetPath) + require.NoError(t, err) + assert.Equal(t, []byte("segment-data"), content) + deltaContent, err := cm.Read(ctx, targetDeltaPath) + require.NoError(t, err) + assert.Equal(t, []byte("delta-data"), deltaContent) + pathlessDeltaContent, err := cm.Read(ctx, targetPathlessDeltaPath) + require.NoError(t, err) + assert.Equal(t, []byte("pathless-delta-data"), pathlessDeltaContent) + assert.NoDirExists(t, path.Join(root, root), "local root must not be prepended twice") +} + +func TestCopySegmentAndIndexFiles_ExternalSourceToLocalCanonicalLayout(t *testing.T) { + ctx := context.Background() + root := t.TempDir() + cm := storage.NewLocalChunkManager(objectstorage.RootPath(root)) + const sourcePrefix = "source-root/files/" + const sourceBase = sourcePrefix + "insert_log/111/222/333" + const sourceLob = sourcePrefix + "insert_log/111/222/lobs/100/_data/lob.vx" + const sourceStats = sourcePrefix + "stats_log/111/222/333/100/10002" + const sourceBM25 = sourcePrefix + "bm25_stats/111/222/333/100/10003" + const sourceDelta = sourcePrefix + "delta_log/111/222/333/10004" + const sourceIndex = sourcePrefix + "index_v1/111/222/333/7000/1/index.v3" + mappings := map[string]string{ + sourceBase + "/_data/10001": "insert_log/444/555/666/_data/10001", + sourceBase + "/_manifest/17.avro": "insert_log/444/555/666/_manifest/17.avro", + sourceBase + "/_stats/text_index.100/idx": "insert_log/444/555/666/_stats/text_index.100/idx", + sourceBase + "/_stats/json_stats.101/meta.json": "insert_log/444/555/666/_stats/json_stats.101/meta.json", + sourceLob: "insert_log/444/555/lobs/100/_data/lob.vx", + sourceStats: "stats_log/444/555/666/100/10002", + sourceBM25: "bm25_stats/444/555/666/100/10003", + sourceDelta: "delta_log/444/555/666/10004", + sourceIndex: "index_v1/444/555/666/7777/1/index.v3", + } + // Mock the foreign provider's enumeration, but perform every destination + // write through a real LocalChunkManager. Manifest contents are opaque here; + // this test checks copy destinations, not native manifest deserialization. + mList := mockey.Mock(listAllFiles).To(func(_ context.Context, _ storage.ChunkManager, base string) ([]string, error) { + require.Equal(t, sourceBase, base) + return []string{ + sourceBase + "/_data/10001", sourceBase + "/_manifest/17.avro", + sourceBase + "/_stats/text_index.100/idx", sourceBase + "/_stats/json_stats.101/meta.json", + }, nil + }).Build() + defer mList.UnPatch() + mLob := mockey.Mock(packed.GetManifestLobFiles).Return([]packed.LobFileInfo{{Path: sourceLob, FieldID: 100}}, nil).Build() + defer mLob.UnPatch() + binlog := func(file string) []*datapb.FieldBinlog { + return []*datapb.FieldBinlog{{FieldID: 100, Binlogs: []*datapb.Binlog{{LogPath: file, EntriesNum: 1}}}} + } + source := &datapb.CopySegmentSource{ + CollectionId: 111, PartitionId: 222, SegmentId: 333, + SourceRootPath: "s3://source-bucket/source-root", + StorageVersion: storage.StorageV3, ManifestHasIndex: proto.Bool(false), ManifestPath: packed.MarshalManifestPath(sourceBase, 17), NumOfRows: 1, + StatsBinlogs: binlog(sourceStats), Bm25Binlogs: binlog(sourceBM25), DeltaBinlogs: binlog(sourceDelta), + IndexFiles: []*indexpb.IndexFilePathInfo{{ + BuildID: 7000, IndexFilePaths: []string{sourceIndex}, + IndexStorePathVersion: indexpb.IndexStorePathVersion_INDEX_STORE_PATH_VERSION_COLLECTION_ROOTED, + }}, + } + target := &datapb.CopySegmentTarget{ + CollectionId: 444, PartitionId: 555, SegmentId: 666, + TargetRootPath: root, NewBuildIds: map[int64]int64{7000: 7777}, + } + copied := make(map[string]string) + copier := newCopySegmentCopierMock(t, func(ctx context.Context, srcBucket, src, dstBucket, dst string) error { + assert.Equal(t, "source-bucket", srcBucket) + assert.Empty(t, dstBucket) + suffix, ok := mappings[src] + require.True(t, ok, "unexpected source file %q", src) + assert.Equal(t, path.Join(root, suffix), dst) + copied[src] = dst + return cm.Write(ctx, dst, []byte(src)) + }) + result, copiedFiles, err := CopySegmentAndIndexFiles(ctx, &struct{ storage.ChunkManager }{}, + &indexpb.StorageConfig{StorageType: "remote", RootPath: "source-root", BucketName: "source-bucket"}, + &indexpb.StorageConfig{StorageType: "local", RootPath: root}, + copier, "source-bucket", "", source, target, nil) + require.NoError(t, err) + require.Len(t, copiedFiles, len(mappings)) + require.Len(t, copied, len(mappings)) + assert.Equal(t, packed.MarshalManifestPath(path.Join(root, "insert_log/444/555/666"), 17), result.GetManifestPath()) + for src, suffix := range mappings { + contents, err := cm.Read(ctx, path.Join(root, suffix)) + require.NoError(t, err) + assert.Equal(t, src, string(contents)) + } + assert.NoDirExists(t, path.Join(root, "files"), "a restore must not create the source's legacy namespace") + assert.NoDirExists(t, path.Join(root, "source-root")) +} + +func TestGenerateMappingsFromFiles_RemoteCanonicalTargetRoot(t *testing.T) { + const prefix = "source-root/files/" + const insert = prefix + "insert_log/111/222/333/_data/10001" + const delta = prefix + "delta_log/111/222/333/10002" + const stats = prefix + "stats_log/111/222/333/100/10003" + const bm25 = prefix + "bm25_stats/111/222/333/100/10004" + const lob = prefix + "insert_log/111/222/lobs/100/_data/lob.vx" + const v0 = prefix + "index_files/7000/1/222/333/index.v3" + const v1 = prefix + "index_v1/111/222/333/7000/1/index.v3" + const text = prefix + "text_log/7000/1/111/222/333/100/index" + const jsonKey = prefix + "json_key_index_log/7000/1/111/222/333/100/index" + const jsonStats = prefix + "json_stats/2/7000/1/111/222/333/100/meta.json" + source := &datapb.CopySegmentSource{ + CollectionId: 111, PartitionId: 222, SegmentId: 333, + SourceRootPath: "s3://source-bucket/source-root", + IndexFiles: []*indexpb.IndexFilePathInfo{{ + IndexFilePaths: []string{v1}, + IndexStorePathVersion: indexpb.IndexStorePathVersion_INDEX_STORE_PATH_VERSION_COLLECTION_ROOTED, + }}, + } + files := &SegmentFiles{ + InsertBinlogs: []string{insert}, DeltaBinlogs: []string{delta}, StatsBinlogs: []string{stats}, Bm25Binlogs: []string{bm25}, + LobFiles: []string{lob}, VectorScalarIndex: []string{v0, v1}, TextIndex: []string{text}, JSONKeyIndex: []string{jsonKey}, JSONStats: []string{jsonStats}, + } + for _, root := range []string{"files", "tenant/files", "", "."} { + t.Run(fmt.Sprintf("root_%q", root), func(t *testing.T) { + target := &datapb.CopySegmentTarget{ + CollectionId: 444, PartitionId: 555, SegmentId: 666, + TargetRootPath: root, NewBuildIds: map[int64]int64{7000: 7777}, + } + got, err := generateMappingsFromFiles(files, source, target) + require.NoError(t, err) + assert.Equal(t, map[string]string{ + insert: path.Join(root, "insert_log/444/555/666/_data/10001"), + delta: path.Join(root, "delta_log/444/555/666/10002"), + stats: path.Join(root, "stats_log/444/555/666/100/10003"), + bm25: path.Join(root, "bm25_stats/444/555/666/100/10004"), + lob: path.Join(root, "insert_log/444/555/lobs/100/_data/lob.vx"), + v0: path.Join(root, "index_files/7777/1/555/666/index.v3"), + v1: path.Join(root, "index_v1/444/555/666/7777/1/index.v3"), + text: path.Join(root, "text_log/7777/1/444/555/666/100/index"), + jsonKey: path.Join(root, "json_key_index_log/7777/1/444/555/666/100/index"), + jsonStats: path.Join(root, "json_stats/2/7777/1/444/555/666/100/meta.json"), + }, got) + }) + } +} + // TestCopySegmentAndIndexFiles_V3WithTextAndJsonStats verifies that V3 segments // with text index and JSON key stats succeed during copy. Before the fix, V3 // segments had wrong-format paths in TextIndexFiles/JsonKeyIndexFiles (etcd @@ -3099,10 +3683,11 @@ func TestCopySegmentAndIndexFiles_V3WithTextAndJsonStats(t *testing.T) { } target := &datapb.CopySegmentTarget{ - CollectionId: 444, - PartitionId: 555, - SegmentId: 666, - NewBuildIds: map[int64]int64{7000: 7777, 8000: 9000}, + TargetRootPath: "files", + CollectionId: 444, + PartitionId: 555, + SegmentId: 666, + NewBuildIds: map[int64]int64{7000: 7777, 8000: 9000}, } mList := mockey.Mock(listAllFiles).To(func(_ context.Context, _ storage.ChunkManager, basePath string) ([]string, error) { diff --git a/internal/datanode/importv2/task_copy_segment_test.go b/internal/datanode/importv2/task_copy_segment_test.go index ea4e1e8f19..794a7f8c96 100644 --- a/internal/datanode/importv2/task_copy_segment_test.go +++ b/internal/datanode/importv2/task_copy_segment_test.go @@ -35,6 +35,7 @@ import ( "github.com/milvus-io/milvus/pkg/v3/proto/datapb" "github.com/milvus-io/milvus/pkg/v3/proto/indexpb" "github.com/milvus-io/milvus/pkg/v3/util/conc" + "github.com/milvus-io/milvus/pkg/v3/util/metautil" ) type crossBucketCopyCall struct { @@ -57,7 +58,15 @@ func copySegmentTaskTestDependencies( t.Helper() storageConfig := req.GetStorageConfig() if storageConfig == nil { - storageConfig = &indexpb.StorageConfig{BucketName: "test-bucket"} + storageConfig = &indexpb.StorageConfig{BucketName: "test-bucket", RootPath: "files"} + req.StorageConfig = storageConfig + } + // AssembleCopySegmentRequest supplies the target storage root in production. + // Keep explicit empty/dot storage roots as bucket-root configurations. + for _, target := range req.GetTargets() { + if target.GetTargetRootPath() == "" { + target.TargetRootPath = storageConfig.GetRootPath() + } } var copier storage.CrossBucketCopier if len(copiers) > 0 { @@ -965,7 +974,7 @@ func TestCopySegmentTaskWithIndexFiles(t *testing.T) { FieldID: 100, IndexID: 1001, BuildID: 1002, - IndexFilePaths: []string{"files/index_files/111/222/333/100/1001/1002/index1"}, + IndexFilePaths: metautil.NewIndexPathBuilder("files", indexpb.IndexStorePathVersion_INDEX_STORE_PATH_VERSION_BUILD_ROOTED, 111, 222, 333, 1002, 1).BuildFilePaths([]string{"index1"}), SerializedSize: 5000, }, }, @@ -1167,7 +1176,7 @@ func TestCopySegmentTaskWithIndexFiles(t *testing.T) { FieldID: 100, IndexID: 1001, BuildID: 1002, - IndexFilePaths: []string{"files/index_files/111/222/333/100/1001/1002/index1"}, + IndexFilePaths: metautil.NewIndexPathBuilder("files", indexpb.IndexStorePathVersion_INDEX_STORE_PATH_VERSION_BUILD_ROOTED, 111, 222, 333, 1002, 1).BuildFilePaths([]string{"index1"}), SerializedSize: 5000, }, }, diff --git a/internal/datanode/importv2/util.go b/internal/datanode/importv2/util.go index 7d5d05b07f..77e493e101 100644 --- a/internal/datanode/importv2/util.go +++ b/internal/datanode/importv2/util.go @@ -20,7 +20,6 @@ import ( "context" "fmt" "math/rand" - "path" "strconv" "time" @@ -42,7 +41,6 @@ import ( "github.com/milvus-io/milvus/pkg/v3/proto/datapb" "github.com/milvus-io/milvus/pkg/v3/proto/indexpb" "github.com/milvus-io/milvus/pkg/v3/util/merr" - "github.com/milvus-io/milvus/pkg/v3/util/metautil" "github.com/milvus-io/milvus/pkg/v3/util/paramtable" "github.com/milvus-io/milvus/pkg/v3/util/retry" "github.com/milvus-io/milvus/pkg/v3/util/typeutil" @@ -76,8 +74,7 @@ func NewSyncTask(ctx context.Context, } // init first manifest path if useLoonFFI { - k := metautil.JoinIDPath(collectionID, partitionID, segmentID) - basePath := path.Join(storageConfig.GetRootPath(), common.SegmentInsertLogPath, k) + basePath := storage.SegmentManifestBasePath(storageConfig.GetRootPath(), collectionID, partitionID, segmentID) // ManifestEarliest for first write segment.ManifestPath = packed.MarshalManifestPath(basePath, packed.ManifestEarliest) } diff --git a/internal/flushcommon/syncmgr/growing_source.go b/internal/flushcommon/syncmgr/growing_source.go index e8a9c24748..07b242a8ec 100644 --- a/internal/flushcommon/syncmgr/growing_source.go +++ b/internal/flushcommon/syncmgr/growing_source.go @@ -42,7 +42,6 @@ import ( "github.com/milvus-io/milvus/pkg/v3/proto/datapb" "github.com/milvus-io/milvus/pkg/v3/proto/indexpb" "github.com/milvus-io/milvus/pkg/v3/util/merr" - "github.com/milvus-io/milvus/pkg/v3/util/metautil" "github.com/milvus-io/milvus/pkg/v3/util/paramtable" "github.com/milvus-io/milvus/pkg/v3/util/retry" "github.com/milvus-io/milvus/pkg/v3/util/timerecord" @@ -873,10 +872,24 @@ func (t *GrowingSourceSyncTask) buildFlushConfig(segment *metacache.SegmentInfo, return nil, merr.WrapErrDataIntegrityMsg("growing source flush requires StorageV3 segment, segmentID=%d storageVersion=%d", t.segmentID, segment.GetStorageVersion()) } - segmentBasePath := path.Join(t.chunkManager.RootPath(), common.SegmentInsertLogPath, - metautil.JoinIDPath(t.collectionID, t.partitionID, t.segmentID)) - partitionBasePath := path.Join(t.chunkManager.RootPath(), common.SegmentInsertLogPath, - metautil.JoinIDPath(t.collectionID, t.partitionID)) + // Same key rule as ordinary flush and the initial manifest: the chunk + // manager's root is the storage prefix on every backend. + rootPath := t.chunkManager.RootPath() + segmentBasePath := storage.SegmentManifestBasePath(rootPath, t.collectionID, t.partitionID, t.segmentID) + partitionBasePath := storage.SegmentPartitionBasePath(rootPath, t.collectionID, t.partitionID) + manifestBasePath, readVersion, err := growingSourceManifest(segment.ManifestPath(), columnGroups) + if err != nil { + return nil, err + } + if manifestBasePath != "" { + // A base and a read version identify one manifest together, so they must + // come from the same place on every backend: changing only the base would + // reopen that version somewhere it does not exist. A recovered local + // segment can still live under the legacy minio prefix, which is why the + // stored base wins over the recomputed one. + segmentBasePath = manifestBasePath + partitionBasePath = path.Dir(manifestBasePath) + } allowedFieldIDs, allowedFieldSet := allowedFieldsFromColumnGroups(columnGroups) var textFieldIDs []int64 @@ -915,10 +928,6 @@ func (t *GrowingSourceSyncTask) buildFlushConfig(segment *metacache.SegmentInfo, if err != nil { return nil, err } - readVersion, err := growingSourceReadVersion(segment.ManifestPath(), columnGroups) - if err != nil { - return nil, err - } schemaBasedFormats := strings.Join(storagecommon.ColumnGroupFormats(columnGroups, writerFormat), ",") return &GrowingFlushConfig{ @@ -1018,24 +1027,24 @@ func (t *GrowingSourceSyncTask) fillPrimaryKeyStatsConfig(ctx context.Context, s return nil } -func growingSourceReadVersion(manifestPath string, columnGroups []storagecommon.ColumnGroup) (int64, error) { +func growingSourceManifest(manifestPath string, columnGroups []storagecommon.ColumnGroup) (string, int64, error) { if manifestPath == "" { - return packed.ManifestEarliest, nil + return "", packed.ManifestEarliest, nil } - _, version, err := packedManifestVersion(manifestPath) + basePath, version, err := packedManifestVersion(manifestPath) if err != nil { - return 0, err + return "", 0, err } if version == packed.ManifestEarliest { - return version, nil + return basePath, version, nil } for _, columnGroup := range columnGroups { if columnGroup.Format == "" { - return 0, merr.WrapErrDataIntegrityMsg("column group %d fields %v missing format for existing manifest %s", + return "", 0, merr.WrapErrDataIntegrityMsg("column group %d fields %v missing format for existing manifest %s", columnGroup.GroupID, columnGroup.Fields, manifestPath) } } - return version, nil + return basePath, version, nil } func allowedFieldsFromColumnGroups(columnGroups []storagecommon.ColumnGroup) ([]int64, map[int64]struct{}) { diff --git a/internal/flushcommon/syncmgr/growing_source_test.go b/internal/flushcommon/syncmgr/growing_source_test.go index 1bed0f8d25..099e500674 100644 --- a/internal/flushcommon/syncmgr/growing_source_test.go +++ b/internal/flushcommon/syncmgr/growing_source_test.go @@ -2,6 +2,7 @@ package syncmgr import ( "context" + "path" "testing" "github.com/cockroachdb/errors" @@ -20,7 +21,9 @@ import ( "github.com/milvus-io/milvus/internal/storagecommon" "github.com/milvus-io/milvus/internal/storagev2/packed" "github.com/milvus-io/milvus/pkg/v3/metrics" + "github.com/milvus-io/milvus/pkg/v3/objectstorage" "github.com/milvus-io/milvus/pkg/v3/proto/datapb" + "github.com/milvus-io/milvus/pkg/v3/proto/indexpb" "github.com/milvus-io/milvus/pkg/v3/util/merr" "github.com/milvus-io/milvus/pkg/v3/util/paramtable" ) @@ -277,6 +280,127 @@ func TestGrowingSourceSyncTaskBuildFlushConfigStartsFromEarliestManifest(t *test require.EqualValues(t, packed.ManifestEarliest, config.ReadVersion) } +func TestGrowingSourceFlushManifestNamespace(t *testing.T) { + params := paramtable.Get() + params.Init(paramtable.NewBaseTable()) + // New segments use the canonical storage prefix on every backend. Existing + // local segments also retain legacy namespaces recorded in their manifests. + for _, tc := range []struct { + name, root, legacyPrefix string + local bool + }{ + {"local", t.TempDir(), "", true}, + {"legacy local", t.TempDir(), "files", true}, + {"remote", "files", "", false}, + {"bucket root", "", "", false}, + {"dot bucket root", ".", "", false}, + } { + t.Run(tc.name, func(t *testing.T) { + for _, existing := range []bool{false, true} { + name := "new segment" + if existing { + name = "existing segment" + } + t.Run(name, func(t *testing.T) { + segmentBase := path.Join(tc.root, "insert_log/3/2/1") + manifestPath := "" + version := packed.ManifestEarliest + if existing { + segmentBase = path.Join(tc.root, tc.legacyPrefix, "insert_log/3/2/1") + version = 7 + manifestPath = packed.MarshalManifestPath(segmentBase, version) + } + partitionBase := path.Dir(segmentBase) + if tc.local { + require.True(t, path.IsAbs(segmentBase)) + } + var cm storage.ChunkManager + if tc.local { + cm = storage.NewLocalChunkManager(objectstorage.RootPath(tc.root)) + } else { + mockCM := mock_storage.NewMockChunkManager(t) + mockCM.EXPECT().RootPath().Return(tc.root) + cm = mockCM + } + segment := metacache.NewSegmentInfo(&datapb.SegmentInfo{ + ID: 1, PartitionID: 2, StorageVersion: storage.StorageV3, + ManifestPath: manifestPath, + }, pkoracle.NewBloomFilterSet(), nil, nil) + task := NewGrowingSourceSyncTask().WithCollectionID(3).WithPartitionID(2). + WithSegmentID(1).WithChunkManager(cm).WithSchema(&schemapb.CollectionSchema{ + Fields: []*schemapb.FieldSchema{{FieldID: 101, DataType: schemapb.DataType_Text}}, + }) + config, err := task.buildFlushConfig(segment, nil) + require.NoError(t, err) + require.Equal(t, segmentBase, config.SegmentBasePath) + require.Equal(t, partitionBase, config.PartitionBasePath) + require.Equal(t, []string{partitionBase + "/lobs/101"}, config.TextLobPaths) + require.EqualValues(t, version, config.ReadVersion) + }) + } + }) + } +} + +func TestGrowingSourceFlushResumesLegacyLocalManifest(t *testing.T) { + params := paramtable.Get() + params.Init(paramtable.NewBaseTable()) + root := t.TempDir() + for _, setting := range []struct{ key, value, previous string }{ + {params.CommonCfg.StorageType.Key, "local", params.CommonCfg.StorageType.GetValue()}, + {params.LocalStorageCfg.Path.Key, root, params.LocalStorageCfg.Path.GetValue()}, + } { + require.NoError(t, params.Save(setting.key, setting.value)) + t.Cleanup(func() { require.NoError(t, params.Save(setting.key, setting.previous)) }) + } + storageConfig := &indexpb.StorageConfig{StorageType: "local", RootPath: root} + legacyBase := path.Join(root, "files/insert_log/3/2/1") + statPath := path.Join(legacyBase, "_stats/bloom_filter.100/1") + require.NoError(t, packed.WriteFile(storageConfig, statPath, []byte("original stats"))) + manifest := packed.MarshalManifestPath(legacyBase, packed.ManifestEarliest) + for version := int64(0); version < 7; version++ { + var err error + manifest, err = packed.CommitManifestUpdates(legacyBase, version, storageConfig, &packed.ManifestUpdates{ + Stats: []packed.StatEntry{{Key: "bloom_filter.100", Files: []string{statPath}}}, + }) + require.NoError(t, err) + } + segment := metacache.NewSegmentInfo(&datapb.SegmentInfo{ + ID: 1, CollectionID: 3, PartitionID: 2, StorageVersion: storage.StorageV3, ManifestPath: manifest, + }, pkoracle.NewBloomFilterSet(), nil, nil) + task := NewGrowingSourceSyncTask().WithCollectionID(3).WithPartitionID(2).WithSegmentID(1). + WithChunkManager(storage.NewLocalChunkManager(objectstorage.RootPath(root))). + WithSchema(&schemapb.CollectionSchema{Fields: []*schemapb.FieldSchema{{FieldID: 101, DataType: schemapb.DataType_Text}}}) + config, err := task.buildFlushConfig(segment, nil) + require.NoError(t, err) + require.Equal(t, legacyBase, config.SegmentBasePath) + require.EqualValues(t, 7, config.ReadVersion) + require.Equal(t, path.Dir(legacyBase), config.PartitionBasePath) + require.Equal(t, []string{path.Join(path.Dir(legacyBase), "lobs/101")}, config.TextLobPaths) + + // Reopen the actual loon manifest, then append through the same base/version + // that growing flush will use. A canonical base paired with legacy version 7 + // cannot even reopen the old manifest, let alone preserve its existing stats. + stats, err := packed.GetManifestStats(packed.MarshalManifestPath(config.SegmentBasePath, config.ReadVersion), storageConfig) + require.NoError(t, err) + require.Equal(t, []string{statPath}, stats["bloom_filter.100"].Paths) + newStatPath := path.Join(config.SegmentBasePath, "_stats/bm25.102/2") + require.NoError(t, packed.WriteFile(storageConfig, newStatPath, []byte("new stats"))) + updatedManifest, err := packed.CommitManifestUpdates(config.SegmentBasePath, config.ReadVersion, storageConfig, &packed.ManifestUpdates{ + Stats: []packed.StatEntry{{Key: "bm25.102", Files: []string{newStatPath}}}, + }) + require.NoError(t, err) + require.Equal(t, packed.MarshalManifestPath(legacyBase, 8), updatedManifest) + stats, err = packed.GetManifestStats(updatedManifest, storageConfig) + require.NoError(t, err) + require.Len(t, stats, 2) + require.Equal(t, []string{statPath}, stats["bloom_filter.100"].Paths) + require.Equal(t, []string{newStatPath}, stats["bm25.102"].Paths) + content, err := packed.ReadFile(storageConfig, statPath) + require.NoError(t, err) + require.Equal(t, "original stats", string(content)) +} + func TestGrowingSourceSyncTaskBuildFlushConfigBM25AllocatorError(t *testing.T) { schema := &schemapb.CollectionSchema{ Name: "bm25", diff --git a/internal/flushcommon/syncmgr/meta_writer_test.go b/internal/flushcommon/syncmgr/meta_writer_test.go index 816abc6c27..ed7e4b8017 100644 --- a/internal/flushcommon/syncmgr/meta_writer_test.go +++ b/internal/flushcommon/syncmgr/meta_writer_test.go @@ -287,9 +287,9 @@ func (s *MetaWriterSuite) TestGrowingSourceSyncShipsStats() { ctx := context.Background() cfg := &indexpb.StorageConfig{StorageType: "local", RootPath: s.T().TempDir()} - basePath := "files/growing_stats/seg1" - bloomPath := path.Join(cfg.RootPath, basePath, "_stats/bloom_filter.100/1") - bm25Path := path.Join(cfg.RootPath, basePath, "_stats/bm25.102/1") + basePath := path.Join(cfg.RootPath, "growing_stats/seg1") + bloomPath := path.Join(basePath, "_stats/bloom_filter.100/1") + bm25Path := path.Join(basePath, "_stats/bm25.102/1") s.Require().NoError(packed.WriteFile(cfg, bloomPath, []byte("bloom"))) s.Require().NoError(packed.WriteFile(cfg, bm25Path, []byte("bm25"))) manifestPath, err := packed.CommitManifestUpdates(basePath, packed.ManifestEarliest, cfg, &packed.ManifestUpdates{ diff --git a/internal/flushcommon/syncmgr/pack_writer_v3_test.go b/internal/flushcommon/syncmgr/pack_writer_v3_test.go index ea4d10f8f5..d9dc72beb8 100644 --- a/internal/flushcommon/syncmgr/pack_writer_v3_test.go +++ b/internal/flushcommon/syncmgr/pack_writer_v3_test.go @@ -145,7 +145,7 @@ func (s *PackWriterV3Suite) TestPackWriterV3_Write() { bfs := pkoracle.NewBloomFilterSet() k := metautil.JoinIDPath(collectionID, partitionID, segmentID) - basePath := path.Join(common.SegmentInsertLogPath, k) + basePath := path.Join(s.rootPath, common.SegmentInsertLogPath, k) manifestPath := packed.MarshalManifestPath(basePath, packed.ManifestEarliest) seg := metacache.NewSegmentInfo(&datapb.SegmentInfo{ @@ -197,7 +197,7 @@ func (s *PackWriterV3Suite) TestPackWriterV3_UsesManifestFormatAfterConfigSwitch bfs := pkoracle.NewBloomFilterSet() k := metautil.JoinIDPath(collectionID, partitionID, segmentID) - basePath := path.Join(common.SegmentInsertLogPath, k) + basePath := path.Join(s.rootPath, common.SegmentInsertLogPath, k) manifestPath := packed.MarshalManifestPath(basePath, packed.ManifestEarliest) seg := metacache.NewSegmentInfo(&datapb.SegmentInfo{ @@ -266,7 +266,7 @@ func (s *PackWriterV3Suite) TestResolveInsertWriterFormatsUsesColumnGroupFormats } bw := NewBulkPackWriterV3(nil, s.schema, s.cm, s.logIDAlloc, packed.DefaultWriteBufferSize, 0, s.storageConfig, columnGroups, - packed.MarshalManifestPath("files/missing-manifest-segment", 42)) + packed.MarshalManifestPath(path.Join(s.rootPath, "missing-manifest-segment"), 42)) bw.initialManifestPath = bw.manifestPath writerFormat, schemaBasedFormats, err := bw.resolveInsertWriterFormats() @@ -287,7 +287,7 @@ func (s *PackWriterV3Suite) TestResolveInsertWriterFormatsRequiresFormatsForExis } bw := NewBulkPackWriterV3(nil, s.schema, s.cm, s.logIDAlloc, packed.DefaultWriteBufferSize, 0, s.storageConfig, columnGroups, - packed.MarshalManifestPath("files/existing-manifest-segment", 42)) + packed.MarshalManifestPath(path.Join(s.rootPath, "existing-manifest-segment"), 42)) bw.initialManifestPath = bw.manifestPath _, _, err := bw.resolveInsertWriterFormats() @@ -303,7 +303,7 @@ func (s *PackWriterV3Suite) TestWriteEmptyInsertData() { channelName := fmt.Sprintf("by-dev-rootcoord-dml_0_%dv0", collectionID) k := metautil.JoinIDPath(collectionID, partitionID, segmentID) - basePath := path.Join(common.SegmentInsertLogPath, k) + basePath := path.Join(s.rootPath, common.SegmentInsertLogPath, k) manifestPath := packed.MarshalManifestPath(basePath, packed.ManifestEarliest) bfs := pkoracle.NewBloomFilterSet() @@ -333,7 +333,7 @@ func (s *PackWriterV3Suite) TestNoPkField() { segmentID := int64(789) k := metautil.JoinIDPath(collectionID, partitionID, segmentID) - basePath := path.Join(common.SegmentInsertLogPath, k) + basePath := path.Join(s.rootPath, common.SegmentInsertLogPath, k) manifestPath := packed.MarshalManifestPath(basePath, packed.ManifestEarliest) channelName := fmt.Sprintf("by-dev-rootcoord-dml_0_%dv0", collectionID) @@ -361,7 +361,7 @@ func (s *PackWriterV3Suite) TestWriteInsertDataError() { channelName := fmt.Sprintf("by-dev-rootcoord-dml_0_%dv0", collectionID) k := metautil.JoinIDPath(collectionID, partitionID, segmentID) - basePath := path.Join(common.SegmentInsertLogPath, k) + basePath := path.Join(s.rootPath, common.SegmentInsertLogPath, k) manifestPath := packed.MarshalManifestPath(basePath, packed.ManifestEarliest) mc := metacache.NewMockMetaCache(s.T()) @@ -410,7 +410,7 @@ func (s *PackWriterV3Suite) TestWriteWithDeleteData() { bfs := pkoracle.NewBloomFilterSet() k := metautil.JoinIDPath(collectionID, partitionID, segmentID) - basePath := path.Join(common.SegmentInsertLogPath, k) + basePath := path.Join(s.rootPath, common.SegmentInsertLogPath, k) manifestPath := packed.MarshalManifestPath(basePath, packed.ManifestEarliest) seg := metacache.NewSegmentInfo(&datapb.SegmentInfo{ @@ -459,7 +459,7 @@ func (s *PackWriterV3Suite) TestV3InheritsV2Fields() { segmentID := int64(789) k := metautil.JoinIDPath(collectionID, partitionID, segmentID) - basePath := path.Join(common.SegmentInsertLogPath, k) + basePath := path.Join(s.rootPath, common.SegmentInsertLogPath, k) manifestPath := packed.MarshalManifestPath(basePath, packed.ManifestEarliest) mc := metacache.NewMockMetaCache(s.T()) @@ -523,7 +523,7 @@ func (s *PackWriterV3Suite) TestMultiBatchStatsAccumulation() { bfs := pkoracle.NewBloomFilterSet() k := metautil.JoinIDPath(collectionID, partitionID, segmentID) - basePath := path.Join(common.SegmentInsertLogPath, k) + basePath := path.Join(s.rootPath, common.SegmentInsertLogPath, k) manifestPath := packed.MarshalManifestPath(basePath, packed.ManifestEarliest) seg := metacache.NewSegmentInfo(&datapb.SegmentInfo{ @@ -613,7 +613,7 @@ func (s *PackWriterV3Suite) TestWrite_PropagatesPriorStatsReadError() { bfs := pkoracle.NewBloomFilterSet() k := metautil.JoinIDPath(collectionID, partitionID, segmentID) - basePath := path.Join(common.SegmentInsertLogPath, k) + basePath := path.Join(s.rootPath, common.SegmentInsertLogPath, k) manifestPath := packed.MarshalManifestPath(basePath, packed.ManifestEarliest) seg := metacache.NewSegmentInfo(&datapb.SegmentInfo{ManifestPath: manifestPath}, bfs, nil, metacache.NewEmptySegmentStats()) @@ -692,7 +692,7 @@ func (s *PackWriterV3Suite) TestMultiBatchBM25StatsAccumulation() { bfs := pkoracle.NewBloomFilterSet() k := metautil.JoinIDPath(collectionID, partitionID, segmentID) - basePath := path.Join(common.SegmentInsertLogPath, k) + basePath := path.Join(s.rootPath, common.SegmentInsertLogPath, k) manifestPath := packed.MarshalManifestPath(basePath, packed.ManifestEarliest) seg := metacache.NewSegmentInfo(&datapb.SegmentInfo{ @@ -770,7 +770,7 @@ func (s *PackWriterV3Suite) TestWrite_SingleVersionBumpAcrossSections() { bfs := pkoracle.NewBloomFilterSet() k := metautil.JoinIDPath(collectionID, partitionID, segmentID) - basePath := path.Join(common.SegmentInsertLogPath, k) + basePath := path.Join(s.rootPath, common.SegmentInsertLogPath, k) manifestPath := packed.MarshalManifestPath(basePath, packed.ManifestEarliest) seg := metacache.NewSegmentInfo(&datapb.SegmentInfo{ManifestPath: manifestPath}, bfs, nil, metacache.NewEmptySegmentStats()) @@ -846,7 +846,7 @@ func (s *PackWriterV3Suite) TestWrite_RetryDoesNotLeakVersionBumps() { bfs := pkoracle.NewBloomFilterSet() k := metautil.JoinIDPath(collectionID, partitionID, segmentID) - basePath := path.Join(common.SegmentInsertLogPath, k) + basePath := path.Join(s.rootPath, common.SegmentInsertLogPath, k) manifestPath := packed.MarshalManifestPath(basePath, packed.ManifestEarliest) seg := metacache.NewSegmentInfo(&datapb.SegmentInfo{ManifestPath: manifestPath}, bfs, nil, metacache.NewEmptySegmentStats()) metacache.UpdateNumOfRows(1000)(seg) diff --git a/internal/flushcommon/syncmgr/task_test.go b/internal/flushcommon/syncmgr/task_test.go index 675858ce8f..899405381b 100644 --- a/internal/flushcommon/syncmgr/task_test.go +++ b/internal/flushcommon/syncmgr/task_test.go @@ -220,7 +220,8 @@ func (s *SyncTaskSuite) createSegment(storageVersion int64) *metacache.SegmentIn // For StorageV3, set up a manifest path if storageVersion == storage.StorageV3 { k := fmt.Sprintf("%d/%d/%d", s.collectionID, s.partitionID, s.segmentID) - basePath := fmt.Sprintf("insert_log/%s", k) + // Complete key under the suite's local storage root ("/tmp"). + basePath := fmt.Sprintf("/tmp/insert_log/%s", k) // Use JSON format for manifest path: {"ver": 0, "base_path": "..."} segInfo.ManifestPath = packed.MarshalManifestPath(basePath, packed.ManifestEarliest) } diff --git a/internal/flushcommon/writebuffer/write_buffer.go b/internal/flushcommon/writebuffer/write_buffer.go index 925463994a..9e52248aa3 100644 --- a/internal/flushcommon/writebuffer/write_buffer.go +++ b/internal/flushcommon/writebuffer/write_buffer.go @@ -3,7 +3,6 @@ package writebuffer import ( "context" "fmt" - "path" "strings" "sync" "time" @@ -23,14 +22,12 @@ import ( "github.com/milvus-io/milvus/internal/storage" "github.com/milvus-io/milvus/internal/storagev2/packed" "github.com/milvus-io/milvus/internal/util/streamingutil" - "github.com/milvus-io/milvus/pkg/v3/common" "github.com/milvus-io/milvus/pkg/v3/metrics" "github.com/milvus-io/milvus/pkg/v3/mlog" "github.com/milvus-io/milvus/pkg/v3/mq/msgstream" "github.com/milvus-io/milvus/pkg/v3/proto/datapb" "github.com/milvus-io/milvus/pkg/v3/util/conc" "github.com/milvus-io/milvus/pkg/v3/util/merr" - "github.com/milvus-io/milvus/pkg/v3/util/metautil" "github.com/milvus-io/milvus/pkg/v3/util/paramtable" "github.com/milvus-io/milvus/pkg/v3/util/retry" "github.com/milvus-io/milvus/pkg/v3/util/tsoutil" @@ -1411,8 +1408,7 @@ func (wb *writeBufferBase) newGrowingSegmentManifestPath(partitionID int64, segm if storageVersion != storage.StorageV3 { return "" } - k := metautil.JoinIDPath(wb.collectionID, partitionID, segmentID) - basePath := path.Join(paramtable.Get().MinioCfg.RootPath.GetValue(), common.SegmentInsertLogPath, k) + basePath := storage.SegmentManifestBasePath(packed.CreateStorageConfig().GetRootPath(), wb.collectionID, partitionID, segmentID) return packed.MarshalManifestPath(basePath, packed.ManifestEarliest) } diff --git a/internal/querynodev2/segments/segment_loader.go b/internal/querynodev2/segments/segment_loader.go index e1522db22e..a084b356bc 100644 --- a/internal/querynodev2/segments/segment_loader.go +++ b/internal/querynodev2/segments/segment_loader.go @@ -43,6 +43,7 @@ import ( "github.com/milvus-io/milvus-proto/go-api/v3/commonpb" "github.com/milvus-io/milvus-proto/go-api/v3/schemapb" + "github.com/milvus-io/milvus/internal/metastore/kv/binlog" "github.com/milvus-io/milvus/internal/querynodev2/pkoracle" "github.com/milvus-io/milvus/internal/storage" "github.com/milvus-io/milvus/internal/storagecommon" @@ -1012,7 +1013,8 @@ func separateLoadInfoV2(loadInfo *querypb.SegmentLoadInfo, schema *schemapb.Coll // For V2 (non-manifest) segments, compute basePaths from metadata. // The resolver returns empty basePaths for V2; we compute them here. - rootPath := paramtable.Get().MinioCfg.RootPath.GetValue() + // Match the writer's primary storage root; local stats do not use MinIO's prefix. + rootPath := binlog.GetRootPath() for fieldID, stats := range textIndexedInfo { if _, ok := textBasePaths[fieldID]; !ok { textBasePaths[fieldID] = metautil.BuildTextIndexPrefix(rootPath, diff --git a/internal/querynodev2/segments/segment_loader_test.go b/internal/querynodev2/segments/segment_loader_test.go index 52ed0dedcf..f83ffaebf6 100644 --- a/internal/querynodev2/segments/segment_loader_test.go +++ b/internal/querynodev2/segments/segment_loader_test.go @@ -27,6 +27,7 @@ import ( "github.com/bytedance/mockey" "github.com/cockroachdb/errors" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" "go.uber.org/atomic" @@ -49,6 +50,39 @@ import ( "github.com/milvus-io/milvus/pkg/v3/util/typeutil" ) +func TestSeparateLoadInfoV2LegacyStatsRoot(t *testing.T) { + paramtable.Init() + params := paramtable.Get() + for _, item := range []*paramtable.ParamItem{¶ms.CommonCfg.StorageType, ¶ms.LocalStorageCfg.Path, ¶ms.MinioCfg.RootPath} { + key, value := item.Key, item.GetValue() + t.Cleanup(func() { require.NoError(t, params.Save(key, value)) }) + } + const localRoot = "/var/lib/milvus/data" + require.NoError(t, params.Save(params.LocalStorageCfg.Path.Key, localRoot)) + require.NoError(t, params.Save(params.MinioCfg.RootPath.Key, "files")) + for _, backend := range []struct{ name, root string }{{"local", localRoot}, {"remote", "files"}} { + t.Run(backend.name, func(t *testing.T) { + require.NoError(t, params.Save(params.CommonCfg.StorageType.Key, backend.name)) + for _, version := range []int64{storage.StorageV1, storage.StorageV2} { + loadInfo := &querypb.SegmentLoadInfo{ + StorageVersion: version, CollectionID: 2, PartitionID: 3, SegmentID: 4, + TextStatsLogs: map[int64]*datapb.TextIndexStats{ + 106: {FieldID: 106, BuildID: 10, Version: 1, Files: []string{"index.v3"}}, + }, + JsonKeyStatsLogs: map[int64]*datapb.JsonKeyStats{ + 107: {FieldID: 107, BuildID: 11, Version: 1, JsonKeyStatsDataFormat: 3, Files: []string{"meta.json", "shared_key_index/0", "shredding_data/0/0"}}, + }, + } + _, _, textStats, _, jsonStats, textBases, jsonBases := separateLoadInfoV2(loadInfo, &schemapb.CollectionSchema{}) + assert.Equal(t, backend.root+"/text_log/10/1/2/3/4/106", textBases[106]) + assert.Equal(t, backend.root+"/json_stats/3/11/1/2/3/4/107", jsonBases[107]) + assert.Equal(t, loadInfo.TextStatsLogs, textStats) + assert.Equal(t, loadInfo.JsonKeyStatsLogs, jsonStats) + } + }) + } +} + type SegmentLoaderSuite struct { suite.Suite loader Loader diff --git a/internal/snapshotio/storage/snapshot_data_test.go b/internal/snapshotio/storage/snapshot_data_test.go index 7be0049f1f..9c5e0d38e6 100644 --- a/internal/snapshotio/storage/snapshot_data_test.go +++ b/internal/snapshotio/storage/snapshot_data_test.go @@ -1525,6 +1525,10 @@ func TestListSnapshotDataFiles_StorageV3IncludesManifestRootObjectsAndLobs(t *te basePath := path.Join(tempDir, "files/insert_log/100/20/1001") require.NoError(t, cm.Write(ctx, path.Join(basePath, "manifest"), []byte("manifest"))) require.NoError(t, cm.Write(ctx, path.Join(basePath, "_data/cg0.parquet"), []byte("data"))) + deltaPath := path.Join(basePath, "_delta/7") + require.NoError(t, cm.Write(ctx, deltaPath, []byte("delta"))) + pathlessDeltaPath := path.Join(basePath, "_delta/8") + require.NoError(t, cm.Write(ctx, pathlessDeltaPath, []byte("pathless delta"))) siblingPath := basePath + "0/manifest" require.NoError(t, cm.Write(ctx, siblingPath, []byte("sibling"))) @@ -1552,7 +1556,15 @@ func TestListSnapshotDataFiles_StorageV3IncludesManifestRootObjectsAndLobs(t *te }, } segment.Statslogs = nil - segment.Deltalogs = nil + // StorageV3 delta metadata: one entry with a complete key and one + // compressed summary placeholder without a path (the manifest owns the + // physical file, which the manifest walk lists). + segment.Deltalogs = []*datapb.FieldBinlog{{ + Binlogs: []*datapb.Binlog{ + {LogPath: deltaPath}, + {LogID: 8}, + }, + }} segment.Bm25Statslogs = nil segment.IndexFiles = nil @@ -1560,12 +1572,20 @@ func TestListSnapshotDataFiles_StorageV3IncludesManifestRootObjectsAndLobs(t *te BucketName: "foreign-bucket", RootPath: "foreign-root", } + getLobCalls := 0 mockGetLobFiles := mockey.Mock(packed.GetManifestLobFiles).To( func(gotManifestPath string, gotStorageConfig *indexpb.StorageConfig) ([]packed.LobFileInfo, error) { assert.Equal(t, segment.GetManifestPath(), gotManifestPath) - assert.Same(t, foreignStorageConfig, gotStorageConfig) + // Listing passes the caller's storage config through; the rewrite + // below calls again with its own config. + if getLobCalls == 0 { + assert.Same(t, foreignStorageConfig, gotStorageConfig) + } + getLobCalls++ + // Manifest-expanded LOB paths are complete keys, so on local + // storage they are absolute like the manifest base. return []packed.LobFileInfo{ - {Path: "files/insert_log/100/20/lobs/30/_data/lob.vx", FieldID: 30}, + {Path: path.Join(tempDir, "files/insert_log/100/20/lobs/30/_data/lob.vx"), FieldID: 30}, }, nil }).Build() defer mockGetLobFiles.UnPatch() @@ -1581,11 +1601,32 @@ func TestListSnapshotDataFiles_StorageV3IncludesManifestRootObjectsAndLobs(t *te assert.Equal(t, snapshotstorage.SnapshotFileTypeStorageV3ManifestRoot, byPath[basePath].Type) assert.Equal(t, snapshotstorage.SnapshotFileTypeStorageV3ManifestObject, byPath[path.Join(basePath, "manifest")].Type) assert.Equal(t, snapshotstorage.SnapshotFileTypeStorageV3ManifestObject, byPath[path.Join(basePath, "_data/cg0.parquet")].Type) - assert.Equal(t, snapshotstorage.SnapshotFileTypeStorageV3LOBFile, byPath["files/insert_log/100/20/lobs/30/_data/lob.vx"].Type) + assert.Equal(t, snapshotstorage.SnapshotFileTypeStorageV3LOBFile, byPath[path.Join(tempDir, "files/insert_log/100/20/lobs/30/_data/lob.vx")].Type) assert.NotContains(t, byPath, "files/text_index/100/12/1001/7002/posting") assert.NotContains(t, byPath, "files/json_index/100/13/1001/7003/key") assert.NotContains(t, byPath, siblingPath) - assert.Len(t, byPath, 4) + assert.Contains(t, byPath, deltaPath) + assert.Contains(t, byPath, pathlessDeltaPath) + assert.Len(t, byPath, 6) + + // Rewriting keeps the pathless summary pathless instead of failing on a + // missing mapping for "". + mappings := make(map[string]string, len(refs)*2) + for _, ref := range refs { + destination := snapshotstorage.ExportedSnapshotPath(cm, ref.NormalizedPath, "bundle") + mappings[ref.Path] = destination + mappings[ref.NormalizedPath] = destination + } + rewritten, err := snapshotstorage.RewriteSnapshotWithMapping( + snapshot, + mappings, + "bundle", + "s3://bucket/bundle/snapshots/100/metadata/1.json", + ) + require.NoError(t, err) + assert.Equal(t, mappings[deltaPath], rewritten.Segments[0].GetDeltalogs()[0].GetBinlogs()[0].GetLogPath()) + assert.Empty(t, rewritten.Segments[0].GetDeltalogs()[0].GetBinlogs()[1].GetLogPath()) + assert.Equal(t, int64(8), rewritten.Segments[0].GetDeltalogs()[0].GetBinlogs()[1].GetLogID()) } func validateSnapshotDataFiles( diff --git a/internal/storage/binlog_record_writer.go b/internal/storage/binlog_record_writer.go index e23ea58425..8abd130ea8 100644 --- a/internal/storage/binlog_record_writer.go +++ b/internal/storage/binlog_record_writer.go @@ -59,6 +59,32 @@ type BinlogRecordWriter interface { Schema() *schemapb.CollectionSchema } +// SegmentManifestBasePath returns the loon manifest base path for a segment's +// insert log: /insert_log///. +// +// The rule is the same for every backend: a key is the storage prefix +// (minio.rootPath for remote, localStorage.path for local) followed by the +// layout, and the manifest stores exactly that key. The loon filesystem is +// rooted at the namespace root (the bucket, or "/" for local; see +// storagev2.LoonFSRootPath), so the manifest path, the ChunkManager key and the +// physical location are one string. Every primary-storage V3 writer (flush, +// compaction, import) must derive the path through this helper; building it +// from minio.rootPath under local storage was #53052. +func SegmentManifestBasePath(rootPath string, collectionID, partitionID, segmentID UniqueID) string { + k := metautil.JoinIDPath(collectionID, partitionID, segmentID) + return path.Join(rootPath, common.SegmentInsertLogPath, k) +} + +// SegmentPartitionBasePath returns the partition-level base path used for TEXT +// LOB files: /insert_log//. +// It shares the rule of SegmentManifestBasePath so LOB references written by +// mix/clustering compaction resolve to the same files the reader reconstructs +// from the manifest base. +func SegmentPartitionBasePath(rootPath string, collectionID, partitionID UniqueID) string { + k := metautil.JoinIDPath(collectionID, partitionID) + return path.Join(rootPath, common.SegmentInsertLogPath, k) +} + type packedBinlogRecordWriterBase struct { // attributes collectionID UniqueID @@ -477,8 +503,7 @@ func (pw *PackedManifestRecordWriter) initWriters(r Record) error { writerFormat, schemaBasedFormats := pw.fillV3ColumnGroupFormats() var err error - k := metautil.JoinIDPath(pw.collectionID, pw.partitionID, pw.segmentID) - pw.basePath = path.Join(pw.storageConfig.GetRootPath(), common.SegmentInsertLogPath, k) + pw.basePath = SegmentManifestBasePath(pw.storageConfig.GetRootPath(), pw.collectionID, pw.partitionID, pw.segmentID) pw.writer, err = newPackedRecordBatchWriter(pw.basePath, pw.schema, pw.bufferSize, pw.multiPartUploadSize, pw.columnGroups, pw.storageConfig, pw.storagePluginContext, true, pw.textRefsAsBinary, writerFormat, schemaBasedFormats) if err != nil { return merr.WrapErrStorage(err, "can not new packed record writer") @@ -710,8 +735,7 @@ func (pw *PackedTextManifestRecordWriter) initWriters(r Record) error { writerFormat, schemaBasedFormats := pw.fillV3ColumnGroupFormats() var err error - k := metautil.JoinIDPath(pw.collectionID, pw.partitionID, pw.segmentID) - pw.basePath = path.Join(pw.storageConfig.GetRootPath(), common.SegmentInsertLogPath, k) + pw.basePath = SegmentManifestBasePath(pw.storageConfig.GetRootPath(), pw.collectionID, pw.partitionID, pw.segmentID) pw.writer, err = NewPackedTextBatchWriter(pw.storageConfig.GetBucketName(), pw.basePath, pw.schema, pw.bufferSize, pw.multiPartUploadSize, pw.columnGroups, pw.storageConfig, pw.textColumnConfigs, writerFormat, schemaBasedFormats) if err != nil { return merr.WrapErrStorage(err, "can not new packed text writer") diff --git a/internal/storage/binlog_record_writer_v3_test.go b/internal/storage/binlog_record_writer_v3_test.go index 453f54e390..4d6c4828fc 100644 --- a/internal/storage/binlog_record_writer_v3_test.go +++ b/internal/storage/binlog_record_writer_v3_test.go @@ -355,3 +355,67 @@ func TestPackedTextManifestRecordWriter_FillsV3ColumnGroupFormats(t *testing.T) assert.Equal(t, "parquet", gotColumnGroups[1].Format) assert.Equal(t, gotColumnGroups, w.columnGroups) } + +// TestSegmentManifestBasePath verifies that the V3 manifest base path stays +// consistent with the loon filesystem root for both storage types. For local +// storage the filesystem is rooted at localStorage.path, so the key must stay +// relative to it and minio.rootPath must NOT leak in (see #53051, #53052). For +// remote storage the logical minio.rootPath is a key prefix and must be kept. +func TestSegmentManifestBasePath(t *testing.T) { + // One rule for every backend: /insert_log///. + // The prefix is localStorage.path for local and minio.rootPath for remote; + // the manifest stores the complete key and never a root-relative path. + localCfg := &indexpb.StorageConfig{StorageType: "local", RootPath: "/var/lib/milvus/data"} + assert.Equal(t, "/var/lib/milvus/data/insert_log/1/2/3", + SegmentManifestBasePath(localCfg.GetRootPath(), 1, 2, 3)) + assert.Equal(t, "/var/lib/milvus/data/insert_log/1/2", + SegmentPartitionBasePath(localCfg.GetRootPath(), 1, 2)) + + remoteCfg := &indexpb.StorageConfig{StorageType: "minio", RootPath: "files"} + assert.Equal(t, "files/insert_log/1/2/3", + SegmentManifestBasePath(remoteCfg.GetRootPath(), 1, 2, 3)) + assert.Equal(t, "files/insert_log/1/2", + SegmentPartitionBasePath(remoteCfg.GetRootPath(), 1, 2)) + + // Bucket-root remote layouts keep working: an empty prefix adds nothing. + bucketRootCfg := &indexpb.StorageConfig{StorageType: "remote", RootPath: ""} + assert.Equal(t, "insert_log/1/2/3", + SegmentManifestBasePath(bucketRootCfg.GetRootPath(), 1, 2, 3)) +} + +// TestPackedManifestRecordWriter_LocalStorageBasePath verifies that initWriters +// derives the complete key /insert_log//

/ for local +// storage, exactly as it derives /insert_log/... for remote. +func TestPackedManifestRecordWriter_LocalStorageBasePath(t *testing.T) { + schema := &schemapb.CollectionSchema{Fields: []*schemapb.FieldSchema{ + {FieldID: common.TimeStampField, DataType: schemapb.DataType_Int64}, + {FieldID: common.RowIDField, DataType: schemapb.DataType_Int64, IsPrimaryKey: true}, + }} + columnGroups := []storagecommon.ColumnGroup{ + {GroupID: 0, Columns: []int{0, 1}, Fields: []int64{common.TimeStampField, common.RowIDField}}, + } + + var gotBasePath string + patch := mockey.Mock(newPackedRecordBatchWriter).To( + func(basePath string, _ *schemapb.CollectionSchema, _, _ int64, _ []storagecommon.ColumnGroup, + _ *indexpb.StorageConfig, _ *indexcgopb.StoragePluginContext, _ bool, _ bool, + _ string, _ []string, + ) (*packedRecordBatchWriter, error) { + gotBasePath = basePath + return &packedRecordBatchWriter{}, nil + }).Build() + defer patch.UnPatch() + + root := t.TempDir() + cfg := &indexpb.StorageConfig{StorageType: "local", RootPath: root} + + w, err := newPackedManifestRecordWriter(1, 2, 3, schema, + ChunkedBlobsWriter(func(_ []*Blob) error { return nil }), + allocator.NewLocalAllocator(1, 1<<20), + 1024, 0, 0, columnGroups, cfg, nil, false, "") + require.NoError(t, err) + require.NoError(t, w.initWriters(nil)) + + assert.Equal(t, root+"/insert_log/1/2/3", w.basePath) + assert.Equal(t, root+"/insert_log/1/2/3", gotBasePath) +} diff --git a/internal/storage/local_chunk_manager.go b/internal/storage/local_chunk_manager.go index e3860ed46c..9d86b33ff5 100644 --- a/internal/storage/local_chunk_manager.go +++ b/internal/storage/local_chunk_manager.go @@ -172,11 +172,23 @@ func (lcm *LocalChunkManager) WalkWithPrefix(ctx context.Context, prefix string, if ctx.Err() != nil { return ctx.Err() } + // A missing scan root is an empty listing, including repeated GC. + if filePath == dir && os.IsNotExist(err) { + return nil + } + matches := strings.HasPrefix(filePath, prefix) + // Prune unrelated directories before handling their read errors. + // Keep matches (including insert_logExtra) and ancestors of prefix. + if f != nil && f.IsDir() && filePath != dir && + !matches && + !strings.HasPrefix(prefix, filePath+string(filepath.Separator)) { + return filepath.SkipDir + } if err != nil { return err } - if strings.HasPrefix(filePath, prefix) && !f.IsDir() { + if matches && !f.IsDir() { if !walkFunc(&ChunkObjectInfo{FilePath: filePath, ModifyTime: f.ModTime()}) { return nil } @@ -251,6 +263,16 @@ func (lcm *LocalChunkManager) Size(ctx context.Context, filePath string) (int64, } func (lcm *LocalChunkManager) Remove(ctx context.Context, filePath string) error { + // Keep an empty key as a no-op and never allow this API to remove the + // configured storage root. Some existing local-only cleanup paths remove a + // derived job subdirectory recursively, so retain that behavior below the + // root while making the dangerous root cases explicit. + if filePath == "" { + return nil + } + if lcm.localPath != "" && filepath.Clean(filePath) == filepath.Clean(lcm.localPath) { + return merr.WrapErrParameterInvalidMsg("refuse to remove local storage root") + } err := os.RemoveAll(filePath) return merr.WrapErrIoFailed(filePath, err) } diff --git a/internal/storage/local_chunk_manager_test.go b/internal/storage/local_chunk_manager_test.go index ed54255f22..b32cd23424 100644 --- a/internal/storage/local_chunk_manager_test.go +++ b/internal/storage/local_chunk_manager_test.go @@ -18,6 +18,7 @@ package storage import ( "context" + "os" "path" "path/filepath" "testing" @@ -30,6 +31,122 @@ import ( var localPath string +func TestLocalChunkManagerWalkWithPrefixPrunesUnrelatedDirectories(t *testing.T) { + for _, readOnlyParent := range []bool{false, true} { + name := "unreadable backup" + if readOnlyParent { + name = "read-only backup with unreadable child" + } + t.Run(name, func(t *testing.T) { + root := t.TempDir() + cm := NewLocalChunkManager(objectstorage.RootPath(root)) + key := filepath.Join(root, "insert_log/1/2/3/data") + backup := filepath.Join(root, ".backup") + blocked := backup + if readOnlyParent { + blocked = filepath.Join(backup, "nested") + } + require.NoError(t, cm.Write(t.Context(), key, []byte("live"))) + require.NoError(t, cm.Write(t.Context(), filepath.Join(blocked, "retained"), []byte("backup"))) + t.Cleanup(func() { + require.NoError(t, os.Chmod(backup, 0o700)) + require.NoError(t, os.Chmod(blocked, 0o700)) + }) + if readOnlyParent { + require.NoError(t, os.Chmod(backup, 0o500)) + } + require.NoError(t, os.Chmod(blocked, 0)) + if _, err := os.ReadDir(blocked); err == nil { + t.Skip("filesystem permissions do not prevent this user from reading the directory") + } else { + require.ErrorIs(t, err, os.ErrPermission) + } + var found []string + require.NoError(t, cm.WalkWithPrefix(t.Context(), filepath.Join(root, "insert_log"), true, func(info *ChunkObjectInfo) bool { + found = append(found, info.FilePath) + return true + })) + require.Equal(t, []string{key}, found) + + // Pruning unrelated directories must not hide a real error under + // the requested prefix itself. + err := cm.WalkWithPrefix(t.Context(), blocked+string(filepath.Separator), true, func(*ChunkObjectInfo) bool { return true }) + require.ErrorIs(t, err, os.ErrPermission) + }) + } +} + +func TestLocalChunkManagerWalkWithPrefixKeepsStringPrefixSemantics(t *testing.T) { + root := t.TempDir() + cm := NewLocalChunkManager(objectstorage.RootPath(root)) + keys := []string{"insert_log/1/data", "insert_log/10/data", "insert_logExtra/1/data", "insert_log.file", "unrelated/data"} + for _, key := range keys { + require.NoError(t, cm.Write(t.Context(), filepath.Join(root, key), []byte(key))) + } + for _, test := range []struct { + prefix string + want []string + }{ + {prefix: "insert_log", want: keys[:4]}, + {prefix: "insert_log/", want: keys[:2]}, + {prefix: "insert_log/1", want: keys[:2]}, + {prefix: "insert_log/1/", want: keys[:1]}, + {prefix: "insert_logExtra", want: keys[2:3]}, + {prefix: "missing"}, + } { + t.Run(test.prefix, func(t *testing.T) { + var found []string + // Concatenate rather than Join so a caller's trailing slash survives. + prefix := root + string(filepath.Separator) + filepath.FromSlash(test.prefix) + require.NoError(t, cm.WalkWithPrefix(t.Context(), prefix, true, func(info *ChunkObjectInfo) bool { + rel, err := filepath.Rel(root, info.FilePath) + require.NoError(t, err) + found = append(found, filepath.ToSlash(rel)) + return true + })) + require.ElementsMatch(t, test.want, found) + }) + } + t.Run("relative prefix", func(t *testing.T) { + t.Chdir(root) + var found []string + require.NoError(t, cm.WalkWithPrefix(t.Context(), "insert_log", true, func(info *ChunkObjectInfo) bool { + found = append(found, filepath.ToSlash(info.FilePath)) + return true + })) + require.ElementsMatch(t, keys[:4], found) + }) +} + +func TestLocalChunkManagerEmptyKeyDoesNotDeleteRoot(t *testing.T) { + ctx := context.Background() + root := t.TempDir() + testCM := NewLocalChunkManager(objectstorage.RootPath(root)) + sentinelPath := filepath.Join(root, "sentinel") + removedPath := filepath.Join(root, "remove-me") + require.NoError(t, testCM.Write(ctx, sentinelPath, []byte("keep"))) + require.NoError(t, testCM.Write(ctx, removedPath, []byte("remove"))) + + require.NoError(t, testCM.Remove(ctx, "")) + content, err := testCM.Read(ctx, sentinelPath) + require.NoError(t, err) + assert.Equal(t, []byte("keep"), content) + + err = testCM.Remove(ctx, root) + require.Error(t, err) + content, err = testCM.Read(ctx, sentinelPath) + require.NoError(t, err) + assert.Equal(t, []byte("keep"), content) + + require.NoError(t, testCM.MultiRemove(ctx, []string{"", removedPath})) + content, err = testCM.Read(ctx, sentinelPath) + require.NoError(t, err) + assert.Equal(t, []byte("keep"), content) + exists, err := testCM.Exist(ctx, removedPath) + require.NoError(t, err) + assert.False(t, exists) +} + func TestLocalCM(t *testing.T) { ctx := context.Background() @@ -731,3 +848,45 @@ func readAllChunkWithPrefix(ctx context.Context, manager ChunkManager, prefix st } return paths, contents, nil } + +// An object store returns an empty listing for a prefix that holds nothing, so +// the local manager must not report ENOENT for a directory that was never +// created or was already cleaned. Callers (segment GC) rely on prefix removal +// being idempotent across retries. +func TestLocalChunkManagerWalkMissingPrefixIsEmpty(t *testing.T) { + ctx := context.Background() + root := t.TempDir() + lcm := NewLocalChunkManager(objectstorage.RootPath(root)) + + for _, prefix := range []string{ + path.Join(root, "insert_log", "1", "2", "3"), + path.Join(root, "insert_log", "1", "2", "3") + "/", + path.Join(root, "never", "existed"), + } { + walked := 0 + require.NoError(t, lcm.WalkWithPrefix(ctx, prefix, true, func(*ChunkObjectInfo) bool { + walked++ + return true + })) + assert.Zero(t, walked) + require.NoError(t, lcm.RemoveWithPrefix(ctx, prefix)) + } + + // Removing a prefix twice must succeed both times. + key := path.Join(root, "insert_log", "1", "2", "4", "a.parquet") + require.NoError(t, lcm.Write(ctx, key, []byte("payload"))) + prefix := path.Join(root, "insert_log", "1", "2", "4") + "/" + require.NoError(t, lcm.RemoveWithPrefix(ctx, prefix)) + require.NoError(t, lcm.RemoveWithPrefix(ctx, prefix)) + exist, err := lcm.Exist(ctx, key) + require.NoError(t, err) + assert.False(t, exist) +} + +// An empty prefix matches every object in the bucket, so this API must refuse it +// rather than erase the instance. LocalChunkManager already did; the remote one +// is the backend where the blast radius is the whole bucket. +func TestRemoteChunkManagerRemoveWithPrefixRejectsEmptyPrefix(t *testing.T) { + rcm := &RemoteChunkManager{} + require.Error(t, rcm.RemoveWithPrefix(context.Background(), "")) +} diff --git a/internal/storage/localmigrate/displaced_empty_test.go b/internal/storage/localmigrate/displaced_empty_test.go new file mode 100644 index 0000000000..b57d1df18c --- /dev/null +++ b/internal/storage/localmigrate/displaced_empty_test.go @@ -0,0 +1,112 @@ +// Licensed to the LF AI & Data foundation under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 + +package localmigrate + +import ( + "io/fs" + "os" + "path/filepath" + "testing" + + "github.com/bytedance/mockey" + "github.com/stretchr/testify/require" + "golang.org/x/sys/unix" +) + +func TestMigrateDisplacedEmptySkeleton(t *testing.T) { + for _, layout := range LayoutDirs { + t.Run(layout, func(t *testing.T) { + root := t.TempDir() + t.Chdir(t.TempDir()) + skeleton := filepath.Join(layout, "1/2/3/empty") + require.NoError(t, os.MkdirAll(filepath.Join(displaced(root), skeleton), 0o755)) + canonical := filepath.Join(root, skeleton) + require.NoError(t, os.MkdirAll(canonical, 0o755)) + unrelated := filepath.Join(displaced(root), "unrelated/empty") + require.NoError(t, os.MkdirAll(unrelated, 0o755)) + + report, err := Migrate(t.Context(), root, Options{LegacyPrefix: "files"}) + require.NoError(t, err) + require.Empty(t, report.Candidates) + require.Zero(t, report.Renamed) + require.Zero(t, report.Copied) + require.NoDirExists(t, filepath.Join(displaced(root), layout)) + require.DirExists(t, canonical) + require.DirExists(t, unrelated) + require.DirExists(t, root) + }) + } +} + +func TestMigrateDisplacedEmptyRecoveryAfterLastFile(t *testing.T) { + root := t.TempDir() + t.Chdir(t.TempDir()) + key := "insert_log/100/10/2001/_data/data.parquet" + source := filepath.Join(displaced(root), key) + writeFile(t, source, "complete-data") + // Existing canonical files force a recursive merge instead of a subtree rename. + writeFile(t, filepath.Join(root, "insert_log/100/10/2001/_data/other.parquet"), "canonical-data") + var originalRemove func(*os.Root, string) error + patch := mockey.Mock((*os.Root).Remove).Origin(&originalRemove).To(func(opened *os.Root, name string) error { + if filepath.Join(opened.Name(), name) == filepath.Dir(source) { + return unix.EIO + } + return originalRemove(opened, name) + }).Build() + t.Cleanup(func() { patch.UnPatch() }) + report, err := Migrate(t.Context(), root, Options{LegacyPrefix: "files"}) + require.ErrorIs(t, err, unix.EIO) + require.Equal(t, 1, report.Renamed) + require.NoFileExists(t, source) + require.Equal(t, "complete-data", readFile(t, filepath.Join(root, key))) + patch.UnPatch() + + report, err = Migrate(t.Context(), root, Options{LegacyPrefix: "files"}) + require.NoError(t, err) + require.Empty(t, report.Candidates) + require.Zero(t, report.Renamed) + require.NoDirExists(t, filepath.Join(displaced(root), "insert_log")) + require.Equal(t, "canonical-data", readFile(t, filepath.Join(root, "insert_log/100/10/2001/_data/other.parquet"))) + + // Once repaired, another startup must not traverse the old empty skeleton. + walk := mockey.Mock(fs.WalkDir).To(func(fs.FS, string, fs.WalkDirFunc) error { + t.Fatal("repaired startup must not walk missing legacy layouts") + return nil + }).Build() + t.Cleanup(func() { walk.UnPatch() }) + _, err = Migrate(t.Context(), root, Options{LegacyPrefix: "files"}) + require.NoError(t, err) +} + +func TestMigrateDisplacedEmptyPreservesActiveNamespace(t *testing.T) { + root := t.TempDir() + t.Chdir(t.TempDir()) + empty := filepath.Join(displaced(root), "insert_log/1/2/3") + require.NoError(t, os.MkdirAll(empty, 0o755)) + legacyPrefix, err := filepath.Rel(root, displaced(root)) + require.NoError(t, err) + _, err = Migrate(t.Context(), root, Options{LegacyPrefix: legacyPrefix}) + require.NoError(t, err) + require.DirExists(t, empty) +} + +func TestMigrateDisplacedEmptyWaitsForSuccessfulMigration(t *testing.T) { + root := t.TempDir() + t.Chdir(t.TempDir()) + empty := filepath.Join(displaced(root), "text_log/1/2/3") + require.NoError(t, os.MkdirAll(empty, 0o755)) + key := "insert_log/1/2/3/data.parquet" + writeFile(t, filepath.Join(displaced(root), key), "source") + writeFile(t, filepath.Join(root, key), "conflicting-target") + _, err := Migrate(t.Context(), root, Options{LegacyPrefix: "files"}) + require.Error(t, err) + require.DirExists(t, empty) +} diff --git a/internal/storage/localmigrate/durability_test.go b/internal/storage/localmigrate/durability_test.go new file mode 100644 index 0000000000..c61cae34d1 --- /dev/null +++ b/internal/storage/localmigrate/durability_test.go @@ -0,0 +1,158 @@ +// Licensed to the LF AI & Data foundation under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 + +package localmigrate + +import ( + "io" + "os" + "path/filepath" + "testing" + + "github.com/bytedance/mockey" + "github.com/stretchr/testify/require" + "golang.org/x/sys/unix" +) + +func TestMigrateCopySyncsDestinationAncestorsBeforeSourceRemoval(t *testing.T) { + for _, publication := range []string{"fresh", "resumed"} { + t.Run(publication, func(t *testing.T) { + root, cwd, key := setupDurabilityCopy(t, publication == "resumed") + expected := destinationAncestorPaths(root, key) + var synced []string + var originalSync func(*os.File) error + syncPatch := mockey.Mock((*os.File).Sync).Origin(&originalSync).To(func(file *os.File) error { + if err := originalSync(file); err != nil { + return err + } + for _, directory := range expected { + if filepath.Clean(file.Name()) == directory { + synced = append(synced, directory) + } + } + return nil + }).Build() + defer syncPatch.UnPatch() + var originalRemove func(*os.Root, string) error + removed := 0 + removePatch := mockey.Mock((*os.Root).Remove).Origin(&originalRemove).To(func(opened *os.Root, name string) error { + if filepath.Join(opened.Name(), name) == filepath.Join(cwd, key) { + require.GreaterOrEqual(t, len(synced), len(expected)) + require.Equal(t, expected, synced[len(synced)-len(expected):], + "persist the complete destination directory chain before unlinking the source") + removed++ + } + return originalRemove(opened, name) + }).Build() + defer removePatch.UnPatch() + report, err := Migrate(t.Context(), root, Options{LegacyPrefix: "files"}) + require.NoError(t, err) + require.Equal(t, 1, removed) + require.Equal(t, 1, report.Copied) + require.Equal(t, "complete-index", readFile(t, filepath.Join(root, key))) + require.NoFileExists(t, filepath.Join(cwd, key)) + }) + } +} + +func TestMigrateCopyAncestorSyncFailureRetainsSourceAndRetries(t *testing.T) { + for _, publication := range []string{"fresh", "resumed"} { + for _, ancestor := range []string{"index_files/8/0/2/3", "index_files/8/0/2", "index_files/8/0", "index_files/8", "index_files", "."} { + t.Run(publication+"/"+ancestor, func(t *testing.T) { + root, cwd, key := setupDurabilityCopy(t, publication == "resumed") + failedDirectory := filepath.Join(root, ancestor) + failures := 0 + matchedSyncs := 0 + failOnSync := 1 + if publication == "fresh" && ancestor == filepath.Dir(key) { + failOnSync = 2 + } + var originalSync func(*os.File) error + syncPatch := mockey.Mock((*os.File).Sync).Origin(&originalSync).To(func(file *os.File) error { + if filepath.Clean(file.Name()) == failedDirectory { + matchedSyncs++ + // The leaf is synced once before the temporary file is + // renamed and again to persist publication. Fail the + // latter so every case exercises restart after publish. + if matchedSyncs == failOnSync { + failures++ + return unix.EIO + } + } + return originalSync(file) + }).Build() + defer syncPatch.UnPatch() + _, err := Migrate(t.Context(), root, Options{LegacyPrefix: "files"}) + require.ErrorIs(t, err, unix.EIO) + require.Equal(t, 1, failures) + require.Equal(t, "complete-index", readFile(t, filepath.Join(cwd, key))) + target := filepath.Join(root, key) + require.Equal(t, "complete-index", readFile(t, target)) + markers, err := filepath.Glob(filepath.Join(filepath.Dir(target), ".milvus-local-copy-*")) + require.NoError(t, err) + require.Len(t, markers, 1) + markerInfo, err := os.Stat(markers[0]) + require.NoError(t, err) + targetInfo, err := os.Stat(target) + require.NoError(t, err) + require.True(t, markerInfo.Mode().IsRegular()) + require.False(t, os.SameFile(markerInfo, targetInfo), "publication proof must not be a hard link") + require.Contains(t, readFile(t, markers[0]), "milvus-local-copy-v1") + syncPatch.UnPatch() + + copyCalls := 0 + copyPatch := mockey.Mock(io.Copy).To(func(io.Writer, io.Reader) (int64, error) { + copyCalls++ + return 0, unix.EIO + }).Build() + defer copyPatch.UnPatch() + report, err := Migrate(t.Context(), root, Options{LegacyPrefix: "files"}) + require.NoError(t, err, "retry should finish the already-published copy") + require.Zero(t, copyCalls) + require.Equal(t, 1, report.Copied) + require.Equal(t, "complete-index", readFile(t, target)) + require.NoFileExists(t, filepath.Join(cwd, key)) + require.NoFileExists(t, markers[0]) + }) + } + } +} + +func setupDurabilityCopy(t *testing.T, resumed bool) (root, cwd, key string) { + t.Helper() + root, cwd = t.TempDir(), t.TempDir() + key = "index_files/8/0/2/3/milvus_packed_inverted_index.v3" + t.Chdir(cwd) + writeFile(t, filepath.Join(cwd, key), "complete-index") + injectSourceRenameError(t, filepath.Join(cwd, filepath.Dir(filepath.Dir(key))), unix.EXDEV) + if resumed { + var originalRemove func(*os.Root, string) error + patch := mockey.Mock((*os.Root).Remove).Origin(&originalRemove).To(func(opened *os.Root, name string) error { + if filepath.Join(opened.Name(), name) == filepath.Join(cwd, key) { + return unix.EACCES + } + return originalRemove(opened, name) + }).Build() + defer patch.UnPatch() + _, err := Migrate(t.Context(), root, Options{LegacyPrefix: "files"}) + require.ErrorIs(t, err, unix.EACCES) + } + return root, cwd, key +} + +func destinationAncestorPaths(root, key string) []string { + var directories []string + for relative := filepath.Dir(key); ; relative = filepath.Dir(relative) { + directories = append(directories, filepath.Join(root, relative)) + if relative == "." { + return directories + } + } +} diff --git a/internal/storage/localmigrate/empty_dirs_test.go b/internal/storage/localmigrate/empty_dirs_test.go new file mode 100644 index 0000000000..8b475dee93 --- /dev/null +++ b/internal/storage/localmigrate/empty_dirs_test.go @@ -0,0 +1,191 @@ +// Licensed to the LF AI & Data foundation under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 + +package localmigrate + +import ( + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestMigrateLegacyCWDRemovesEmptyAncestors(t *testing.T) { + for _, key := range []string{ + "index_files/7/0/2/3/milvus_packed_inverted_index.v3", + "text_log/7/0/1/2/3/100/milvus_packed_text_index.v3", + } { + layout := strings.Split(key, "/")[0] + t.Run(layout, func(t *testing.T) { + root, cwd := t.TempDir(), t.TempDir() + t.Chdir(cwd) + writeFile(t, filepath.Join(cwd, key), "legacy-index") + _, err := Migrate(t.Context(), root, Options{LegacyPrefix: "files"}) + require.NoError(t, err) + require.Equal(t, "legacy-index", readFile(t, filepath.Join(root, key))) + require.NoDirExists(t, filepath.Join(cwd, layout), "remove the empty layout root, not just the packed leaf") + require.DirExists(t, cwd) + report, err := Migrate(t.Context(), root, Options{LegacyPrefix: "files"}) + require.NoError(t, err) + require.Empty(t, report.Candidates) + require.Zero(t, report.Renamed) + require.Zero(t, report.Copied) + }) + } +} + +func TestMigrateLegacyCWDCleansEmptySkeletonOnRestart(t *testing.T) { + for _, skeleton := range []string{ + "index_files/7/0/2/3", + "text_log/7/0/1/2/3/100", + } { + layout := strings.Split(skeleton, "/")[0] + t.Run(layout, func(t *testing.T) { + root, cwd := t.TempDir(), t.TempDir() + t.Chdir(cwd) + // Simulate interruption after the final packed file was moved: + // only eligible empty directories remain, so no file selects a leaf. + require.NoError(t, os.MkdirAll(filepath.Join(cwd, skeleton), 0o755)) + canonicalEmpty := filepath.Join(root, skeleton) + require.NoError(t, os.MkdirAll(canonicalEmpty, 0o755)) + report, err := Migrate(t.Context(), root, Options{LegacyPrefix: "files"}) + require.NoError(t, err) + require.Zero(t, report.Renamed) + require.Zero(t, report.Copied) + require.NoDirExists(t, filepath.Join(cwd, layout)) + require.DirExists(t, cwd) + require.DirExists(t, canonicalEmpty, "canonical empty directories are not legacy cleanup candidates") + }) + } +} + +func TestMigrateLegacyCWDEmptyCleanupPreservesUnrelatedEntries(t *testing.T) { + for _, key := range []string{ + "index_files/7/0/2/3/milvus_packed_inverted_index.v3", + "text_log/7/0/1/2/3/100/milvus_packed_text_index.v3", + } { + layout := strings.Split(key, "/")[0] + t.Run(layout, func(t *testing.T) { + root, cwd := t.TempDir(), t.TempDir() + t.Chdir(cwd) + writeFile(t, filepath.Join(cwd, key), "legacy-index") + unrelatedFile := filepath.Join(cwd, filepath.Dir(filepath.Dir(key)), "999", "legacy-index") + writeFile(t, unrelatedFile, "unrelated-data") + unknownEmpty := filepath.Join(cwd, layout, "notes", "empty") + tooDeepEmpty := filepath.Join(cwd, strings.Replace(filepath.Dir(key), "/7/", "/9/", 1), "111") + outsideEmpty := filepath.Join(cwd, "unrelated-empty") + for _, directory := range []string{unknownEmpty, tooDeepEmpty, outsideEmpty} { + require.NoError(t, os.MkdirAll(directory, 0o755)) + } + _, err := Migrate(t.Context(), root, Options{LegacyPrefix: "files"}) + require.NoError(t, err) + require.Equal(t, "legacy-index", readFile(t, filepath.Join(root, key))) + require.Equal(t, "unrelated-data", readFile(t, unrelatedFile)) + for _, directory := range []string{unknownEmpty, tooDeepEmpty, outsideEmpty, filepath.Join(cwd, layout)} { + require.DirExists(t, directory, "cleanup must stop at nonempty or unrecognized directories") + } + }) + } +} + +func TestMigrateLegacyCWDEmptyCleanupProtectsCanonicalRoot(t *testing.T) { + for _, nested := range []bool{false, true} { + name := "cwd-is-root" + if nested { + name = "root-inside-legacy-layout" + } + t.Run(name, func(t *testing.T) { + cwd := t.TempDir() + t.Chdir(cwd) + root := cwd + if nested { + root = filepath.Join(cwd, "index_files/7/0/2") + } + canonicalEmpty := filepath.Join(root, "3") + if !nested { + canonicalEmpty = filepath.Join(root, "index_files/7/0/2/3") + } + require.NoError(t, os.MkdirAll(canonicalEmpty, 0o755)) + _, err := Migrate(t.Context(), root, Options{LegacyPrefix: "files"}) + require.NoError(t, err) + require.DirExists(t, cwd) + require.DirExists(t, root) + require.DirExists(t, canonicalEmpty) + }) + } +} + +func TestMigrateLegacyCWDEmptyCleanupDoesNotDeleteReplacementFile(t *testing.T) { + for _, kind := range []string{"leaf", "ancestor"} { + t.Run(kind, func(t *testing.T) { + root, cwd := t.TempDir(), t.TempDir() + t.Chdir(cwd) + key := "index_files/7/0/2/3/milvus_packed_inverted_index.v3" + writeFile(t, filepath.Join(cwd, key), "legacy-index") + emptyLeaf := filepath.Join(cwd, "index_files/8/0/2/3") + require.NoError(t, os.MkdirAll(emptyLeaf, 0o755)) + replacement := emptyLeaf + if kind == "ancestor" { + replacement = filepath.Dir(emptyLeaf) + } + replaced := false + _, err := Migrate(t.Context(), root, Options{LegacyPrefix: "files", OnDirDone: func(Dir, time.Duration) { + if !replaced { + // Discovery already saw directories here. Change the leaf + // or its parent before cleanup; neither may unlink a file. + if kind == "ancestor" { + require.NoError(t, os.Remove(emptyLeaf)) + } + require.NoError(t, os.Remove(replacement)) + writeFile(t, replacement, "keep-replacement") + replaced = true + } + }}) + require.NoError(t, err) + require.True(t, replaced) + require.Equal(t, "keep-replacement", readFile(t, replacement)) + }) + } +} + +func TestMigrateLegacyCWDEmptyCleanupWaitsForSuccessfulMigration(t *testing.T) { + root, cwd := t.TempDir(), t.TempDir() + t.Chdir(cwd) + key := "index_files/7/0/2/3/milvus_packed_inverted_index.v3" + writeFile(t, filepath.Join(cwd, key), "legacy-index") + writeFile(t, filepath.Join(root, key), "conflicting-target") + empty := filepath.Join(cwd, "index_files/8/0/2/3") + require.NoError(t, os.MkdirAll(empty, 0o755)) + report, err := Migrate(t.Context(), root, Options{LegacyPrefix: "files"}) + require.Error(t, err) + require.NotEmpty(t, report.Conflicts) + require.DirExists(t, empty, "a failed migration must not proceed to empty-directory cleanup") + require.Equal(t, "legacy-index", readFile(t, filepath.Join(cwd, key))) + require.Equal(t, "conflicting-target", readFile(t, filepath.Join(root, key))) +} + +func TestMigrateLegacyCWDEmptyCleanupProtectsActiveAndReservedNamespaces(t *testing.T) { + for _, prefix := range []string{"files/insert_log/1/2/3", ".milvus-local-layout"} { + t.Run(prefix, func(t *testing.T) { + root := t.TempDir() + cwd := filepath.Join(root, prefix) + empty := filepath.Join(cwd, "index_files/7/0/2/3") + require.NoError(t, os.MkdirAll(empty, 0o755)) + t.Chdir(cwd) + _, err := Migrate(t.Context(), root, Options{LegacyPrefix: "files"}) + require.NoError(t, err) + require.DirExists(t, empty) + require.DirExists(t, cwd) + }) + } +} diff --git a/internal/storage/localmigrate/legacy_cwd_test.go b/internal/storage/localmigrate/legacy_cwd_test.go new file mode 100644 index 0000000000..976f98e868 --- /dev/null +++ b/internal/storage/localmigrate/legacy_cwd_test.go @@ -0,0 +1,155 @@ +// Licensed to the LF AI & Data foundation under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 + +package localmigrate + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestMigrateLegacyCWDUnifiedIndexesRenamesDeepestDirectories(t *testing.T) { + root, cwd := t.TempDir(), t.TempDir() + t.Chdir(cwd) + files := map[string]string{ + "text_log/7/0/1/2/3/100/milvus_packed_text_index.v3": "text-index", + "index_files/8/0/2/3/milvus_packed_inverted_index.v3": "scalar-index", + } + for key, content := range files { + writeFile(t, filepath.Join(cwd, key), content) + } + writeFile(t, filepath.Join(cwd, "notes.txt"), "unrelated") + report, err := Migrate(t.Context(), root, Options{LegacyPrefix: "files"}) + require.NoError(t, err) + assert.Equal(t, 2, report.Renamed) + assert.EqualValues(t, len("text-index")+len("scalar-index"), report.Bytes) + for key, content := range files { + assert.Equal(t, content, readFile(t, filepath.Join(root, key))) + assert.NoFileExists(t, filepath.Join(cwd, key)) + } + assert.Equal(t, "unrelated", readFile(t, filepath.Join(cwd, "notes.txt"))) + second, err := Migrate(t.Context(), root, Options{LegacyPrefix: "files"}) + require.NoError(t, err) + assert.Zero(t, second.Renamed) +} + +func TestMigrateLegacyCWDUnknownTargetConflicts(t *testing.T) { + root, cwd := t.TempDir(), t.TempDir() + t.Chdir(cwd) + key := "index_files/7/0/2/3/milvus_packed_inverted_index.v3" + writeFile(t, filepath.Join(cwd, key), "source") + writeFile(t, filepath.Join(root, key), "target") + report, err := Migrate(t.Context(), root, Options{LegacyPrefix: "files"}) + require.Error(t, err) + assert.NotEmpty(t, report.Conflicts) + assert.Equal(t, "source", readFile(t, filepath.Join(cwd, key))) +} + +func TestMigrateLegacyCWDMatchesDisplacedRoot(t *testing.T) { + for _, name := range []string{"direct", "symlink_alias"} { + t.Run(name, func(t *testing.T) { + root := t.TempDir() + cwd := displaced(root) + files := map[string]string{ + "text_log/7/0/1/2/3/100/milvus_packed_text_index.v3": "text-index", + "index_files/8/0/2/3/milvus_packed_inverted_index.v3": "scalar-index", + } + for key, content := range files { + writeFile(t, filepath.Join(cwd, key), content) + } + if name == "symlink_alias" { + alias := filepath.Join(t.TempDir(), "legacy") + require.NoError(t, os.Symlink(cwd, alias)) + t.Chdir(alias) + } else { + t.Chdir(cwd) + } + + report, err := Migrate(t.Context(), root, Options{LegacyPrefix: "files"}) + require.NoError(t, err) + assert.Empty(t, report.Conflicts) + assert.Len(t, report.Candidates, 1) + assert.Equal(t, 2, report.Renamed) + assert.EqualValues(t, len("text-index")+len("scalar-index"), report.Bytes) + for key, content := range files { + assert.Equal(t, content, readFile(t, filepath.Join(root, key))) + assert.NoFileExists(t, filepath.Join(cwd, key)) + } + + second, err := Migrate(t.Context(), root, Options{LegacyPrefix: "files"}) + require.NoError(t, err) + assert.Zero(t, second.Renamed) + }) + } +} + +func TestMigrateLegacyCWDDistinctFromDisplacedRootStillConflicts(t *testing.T) { + root, cwd := t.TempDir(), t.TempDir() + t.Chdir(cwd) + key := "index_files/8/0/2/3/milvus_packed_inverted_index.v3" + writeFile(t, filepath.Join(displaced(root), key), "displaced") + writeFile(t, filepath.Join(cwd, key), "cwd") + + report, err := Migrate(t.Context(), root, Options{LegacyPrefix: "files"}) + require.Error(t, err) + assert.NotEmpty(t, report.Conflicts) + assert.Zero(t, report.Renamed) + assert.NoFileExists(t, filepath.Join(root, key)) + assert.Equal(t, "displaced", readFile(t, filepath.Join(displaced(root), key))) + assert.Equal(t, "cwd", readFile(t, filepath.Join(cwd, key))) +} + +func TestMigrateLegacyCWDSkipsUnrelatedFiles(t *testing.T) { + root, cwd := t.TempDir(), t.TempDir() + t.Chdir(cwd) + for _, key := range []string{ + "notes.txt", + "index_files/7/0/2/3/legacy-index", + "index_files/7/0/2/3/milvus_packed_inverted_index.v2", + "text_log/7/0/1/2/3/100/index_meta.json", + } { + writeFile(t, filepath.Join(cwd, key), "unrelated") + } + report, err := Migrate(t.Context(), root, Options{LegacyPrefix: "files"}) + require.NoError(t, err) + assert.Zero(t, report.Renamed) + assert.Equal(t, "unrelated", readFile(t, filepath.Join(cwd, "notes.txt"))) + assert.NoDirExists(t, filepath.Join(root, "index_files")) +} + +func TestMigrateLegacyCWDNestedCanonicalRootIsSkipped(t *testing.T) { + cwd := t.TempDir() + t.Chdir(cwd) + root := filepath.Join(cwd, "data") + key := "index_files/7/0/2/3/milvus_packed_inverted_index.v3" + writeFile(t, filepath.Join(root, key), "canonical") + writeFile(t, filepath.Join(cwd, "index_files/8/0/2/3/milvus_packed_inverted_index.v3"), "legacy") + report, err := Migrate(t.Context(), root, Options{LegacyPrefix: "files"}) + require.NoError(t, err) + assert.Equal(t, 1, report.Renamed) + assert.Equal(t, "canonical", readFile(t, filepath.Join(root, key))) + assert.Equal(t, "legacy", readFile(t, filepath.Join(root, "index_files/8/0/2/3/milvus_packed_inverted_index.v3"))) +} + +func TestMigrateUnavailableCurrentDirectoryFailsBeforeMigration(t *testing.T) { + root, cwd := t.TempDir(), t.TempDir() + t.Chdir(cwd) + source := filepath.Join(displaced(root), "insert_log/1/2/3/data") + writeFile(t, source, "old-data") + require.NoError(t, os.Remove(cwd)) + _, err := Migrate(t.Context(), root, Options{LegacyPrefix: "files"}) + require.Error(t, err) + assert.NoFileExists(t, filepath.Join(root, "insert_log")) + assert.Equal(t, "old-data", readFile(t, source)) +} diff --git a/internal/storage/localmigrate/merge_copy_test.go b/internal/storage/localmigrate/merge_copy_test.go new file mode 100644 index 0000000000..6bca590377 --- /dev/null +++ b/internal/storage/localmigrate/merge_copy_test.go @@ -0,0 +1,339 @@ +// Licensed to the LF AI & Data foundation under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 + +package localmigrate + +import ( + "context" + "io" + "io/fs" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/bytedance/mockey" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "golang.org/x/sys/unix" + + "github.com/milvus-io/milvus/pkg/v3/util/merr" +) + +func TestMigrateMergeSplitJSONShreddingSegment(t *testing.T) { + t.Chdir(t.TempDir()) + root := t.TempDir() + segment := "insert_log/100/10/2001" + canonical := map[string]string{ + segment + "/_stats/json_stats.103/meta.json": "json-metadata", + segment + "/_stats/json_stats.103/shared_key_index/keys": "shared-keys", + } + legacy := map[string]string{ + segment + "/_data/data.parquet": "segment-data", + segment + "/_metadata/manifest-3.avro": "manifest", + segment + "/_stats/text_index.102/packed": "text-index", + segment + "/_stats/json_stats.103/shredding_data/0.parquet": "shredded-json", + } + for key, content := range canonical { + writeFile(t, filepath.Join(root, key), content) + } + for key, content := range legacy { + writeFile(t, filepath.Join(displaced(root), key), content) + } + _, err := Migrate(t.Context(), root, Options{LegacyPrefix: "files"}) + require.NoError(t, err) + for key, content := range canonical { + assert.Equal(t, content, readFile(t, filepath.Join(root, key))) + } + for key, content := range legacy { + assert.Equal(t, content, readFile(t, filepath.Join(root, key))) + assert.NoFileExists(t, filepath.Join(displaced(root), key)) + } + assert.NoDirExists(t, filepath.Join(displaced(root), "insert_log")) + assertMigrationRestartIsIdle(t, root) +} + +func TestMigrateMergePreservesCanonicalV1Segment(t *testing.T) { + t.Chdir(t.TempDir()) + root := t.TempDir() + v1 := "insert_log/100/10/2000/101/1234" + v3 := "insert_log/100/10/2001/_data/data.parquet" + writeFile(t, filepath.Join(root, v1), "v1-binlog") + writeFile(t, filepath.Join(displaced(root), v3), "v3-data") + _, err := Migrate(t.Context(), root, Options{LegacyPrefix: "files"}) + require.NoError(t, err) + assert.Equal(t, "v1-binlog", readFile(t, filepath.Join(root, v1))) + assert.Equal(t, "v3-data", readFile(t, filepath.Join(root, v3))) + assert.NoDirExists(t, filepath.Join(displaced(root), "insert_log")) + assertMigrationRestartIsIdle(t, root) +} + +func TestMigrateMergeDisplacedAndCWDIndexes(t *testing.T) { + root, cwd := t.TempDir(), t.TempDir() + t.Chdir(cwd) + displacedKey := "index_files/8/0/2/3/milvus_packed_inverted_index.v3" + cwdKey := "index_files/9/0/2/3/milvus_packed_inverted_index.v3" + writeFile(t, filepath.Join(displaced(root), displacedKey), "displaced-index") + writeFile(t, filepath.Join(cwd, cwdKey), "cwd-index") + _, err := Migrate(t.Context(), root, Options{LegacyPrefix: "files"}) + require.NoError(t, err) + assert.Equal(t, "displaced-index", readFile(t, filepath.Join(root, displacedKey))) + assert.Equal(t, "cwd-index", readFile(t, filepath.Join(root, cwdKey))) + assert.NoDirExists(t, filepath.Join(displaced(root), "index_files")) + assert.NoFileExists(t, filepath.Join(cwd, cwdKey)) + assertMigrationRestartIsIdle(t, root) +} + +func TestMigrateCopyFallbackCrossDevice(t *testing.T) { + for _, sourceKind := range []string{"displaced", "cwd"} { + t.Run(sourceKind, func(t *testing.T) { + root, cwd := t.TempDir(), t.TempDir() + t.Chdir(cwd) + key := "index_files/8/0/2/3/milvus_packed_inverted_index.v3" + sourceRoot, sourceDirectory := displaced(root), "index_files" + if sourceKind == "cwd" { + sourceRoot, sourceDirectory = cwd, filepath.Dir(key) + } + sourceFile := filepath.Join(sourceRoot, key) + writeFile(t, sourceFile, "complete-index") + injected := injectSourceRenameError(t, filepath.Join(sourceRoot, filepath.Dir(sourceDirectory)), unix.EXDEV) + _, err := Migrate(t.Context(), root, Options{LegacyPrefix: "files"}) + require.NoError(t, err) + require.Positive(t, *injected, "must exercise the cross-device fallback") + assert.Equal(t, "complete-index", readFile(t, filepath.Join(root, key))) + assert.NoDirExists(t, filepath.Join(sourceRoot, sourceDirectory)) + assertMigrationRestartIsIdle(t, root) + }) + } +} + +func TestMigrateRenamePermissionErrorDoesNotCopy(t *testing.T) { + root, cwd := t.TempDir(), t.TempDir() + t.Chdir(cwd) + key := "index_files/8/0/2/3/milvus_packed_inverted_index.v3" + writeFile(t, filepath.Join(cwd, key), "source-index") + injected := injectSourceRenameError(t, filepath.Join(cwd, filepath.Dir(filepath.Dir(key))), unix.EACCES) + _, err := Migrate(t.Context(), root, Options{LegacyPrefix: "files"}) + require.ErrorIs(t, err, unix.EACCES) + require.Positive(t, *injected) + assert.Equal(t, "source-index", readFile(t, filepath.Join(cwd, key))) + assert.NoFileExists(t, filepath.Join(root, key)) +} + +func TestMigrateCopyFailureLeavesSourceAndRetries(t *testing.T) { + root, cwd := t.TempDir(), t.TempDir() + t.Chdir(cwd) + key := "index_files/8/0/2/3/milvus_packed_inverted_index.v3" + source, target := filepath.Join(cwd, key), filepath.Join(root, key) + writeFile(t, source, "complete-original-index") + injected := injectSourceRenameError(t, filepath.Join(cwd, filepath.Dir(filepath.Dir(key))), unix.EXDEV) + copyCalls := 0 + copyPatch := mockey.Mock(io.Copy).To(func(dst io.Writer, src io.Reader) (int64, error) { + copyCalls++ + assert.Equal(t, "complete-original-index", readFile(t, source)) + assert.NoFileExists(t, target, "an incomplete copy must not become a canonical file") + n, err := dst.Write([]byte("partial")) + require.NoError(t, err) + return int64(n), io.ErrShortWrite + }).Build() + t.Cleanup(func() { copyPatch.UnPatch() }) + _, err := Migrate(t.Context(), root, Options{LegacyPrefix: "files"}) + require.ErrorIs(t, err, io.ErrShortWrite) + require.Positive(t, *injected) + require.Equal(t, 1, copyCalls) + assert.Equal(t, "complete-original-index", readFile(t, source)) + assert.NoFileExists(t, target) + copyPatch.UnPatch() + _, err = Migrate(t.Context(), root, Options{LegacyPrefix: "files"}) + require.NoError(t, err) + assert.Equal(t, "complete-original-index", readFile(t, target)) + assert.NoFileExists(t, source) + assertMigrationRestartIsIdle(t, root) +} + +func TestCopyFilePublicationRenameFailureRetainsSourceAndRetries(t *testing.T) { + sourceDir, targetDir := t.TempDir(), t.TempDir() + key := "index_files/8/0/2/3/milvus_packed_inverted_index.v3" + writeFile(t, filepath.Join(sourceDir, key), "complete-index") + require.NoError(t, os.MkdirAll(filepath.Join(targetDir, filepath.Dir(key)), 0o755)) + + source, err := os.OpenRoot(sourceDir) + require.NoError(t, err) + defer source.Close() + target, err := os.OpenRoot(targetDir) + require.NoError(t, err) + defer target.Close() + + var originalRename func(int, string, int, string) error + renameCalls := 0 + renamePatch := mockey.Mock(unix.Renameat).To(func(oldfd int, oldname string, newfd int, newname string) error { + if strings.HasSuffix(oldname, ".tmp") && newname == filepath.Base(key) { + renameCalls++ + return unix.EIO + } + return originalRename(oldfd, oldname, newfd, newname) + }).Origin(&originalRename).Build() + defer renamePatch.UnPatch() + + err = copyFile(t.Context(), source, key, target, key, &Report{}) + require.ErrorIs(t, err, unix.EIO) + require.Equal(t, 1, renameCalls) + require.Equal(t, "complete-index", readFile(t, filepath.Join(sourceDir, key))) + require.NoFileExists(t, filepath.Join(targetDir, key)) + artifacts, err := filepath.Glob(filepath.Join(targetDir, filepath.Dir(key), ".milvus-local-copy-*")) + require.NoError(t, err) + require.Empty(t, artifacts) + + renamePatch.UnPatch() + report := &Report{} + require.NoError(t, copyFile(t.Context(), source, key, target, key, report)) + require.Equal(t, 1, report.Copied) + require.Equal(t, "complete-index", readFile(t, filepath.Join(targetDir, key))) + require.NoFileExists(t, filepath.Join(sourceDir, key)) +} + +func TestCopyFileCanceledBeforeReadRetainsSource(t *testing.T) { + sourceDir, targetDir := t.TempDir(), t.TempDir() + key := "index_files/8/0/2/3/milvus_packed_inverted_index.v3" + writeFile(t, filepath.Join(sourceDir, key), "complete-index") + require.NoError(t, os.MkdirAll(filepath.Join(targetDir, filepath.Dir(key)), 0o755)) + + source, err := os.OpenRoot(sourceDir) + require.NoError(t, err) + defer source.Close() + target, err := os.OpenRoot(targetDir) + require.NoError(t, err) + defer target.Close() + + ctx, cancel := context.WithCancel(t.Context()) + cancel() + err = copyFile(ctx, source, key, target, key, &Report{}) + require.ErrorIs(t, err, context.Canceled) + require.Equal(t, "complete-index", readFile(t, filepath.Join(sourceDir, key))) + require.NoFileExists(t, filepath.Join(targetDir, key)) + artifacts, err := filepath.Glob(filepath.Join(targetDir, filepath.Dir(key), ".milvus-local-copy-*")) + require.NoError(t, err) + require.Empty(t, artifacts) +} + +func TestContextReaderStopsAfterCancellation(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + reader := &contextReader{ctx: ctx, reader: strings.NewReader("payload")} + buffer := make([]byte, 1) + + n, err := reader.Read(buffer) + require.NoError(t, err) + require.Equal(t, 1, n) + cancel() + + n, err = reader.Read(buffer) + require.ErrorIs(t, err, context.Canceled) + require.Zero(t, n) +} + +func TestMigrateCopyPublishedBeforeSourceRemovalRetries(t *testing.T) { + root, cwd := t.TempDir(), t.TempDir() + t.Chdir(cwd) + key := "index_files/8/0/2/3/milvus_packed_inverted_index.v3" + source, target := filepath.Join(cwd, key), filepath.Join(root, key) + writeFile(t, source, "complete-index") + injectSourceRenameError(t, filepath.Join(cwd, filepath.Dir(filepath.Dir(key))), unix.EXDEV) + var originalRemove func(*os.Root, string) error + removeCalls := 0 + removePatch := mockey.Mock((*os.Root).Remove).To(func(openedRoot *os.Root, name string) error { + if filepath.Join(openedRoot.Name(), name) == source { + removeCalls++ + assert.Equal(t, "complete-index", readFile(t, target), "publish a complete target before removing its source") + return unix.EACCES + } + return originalRemove(openedRoot, name) + }).Origin(&originalRemove).Build() + t.Cleanup(func() { removePatch.UnPatch() }) + _, err := Migrate(t.Context(), root, Options{LegacyPrefix: "files"}) + require.ErrorIs(t, err, unix.EACCES) + require.Equal(t, 1, removeCalls) + assert.Equal(t, "complete-index", readFile(t, source)) + assert.Equal(t, "complete-index", readFile(t, target)) + removePatch.UnPatch() + _, err = Migrate(t.Context(), root, Options{LegacyPrefix: "files"}) + require.NoError(t, err, "an already-published copy must not become a same-name conflict on retry") + assert.Equal(t, "complete-index", readFile(t, target)) + assert.NoFileExists(t, source) + assertMigrationRestartIsIdle(t, root) +} + +func TestMigrateMergeRejectsSameNameConflicts(t *testing.T) { + for _, conflictKind := range []string{"different-files", "source-file-target-directory", "source-directory-target-file"} { + t.Run(conflictKind, func(t *testing.T) { + t.Chdir(t.TempDir()) + root := t.TempDir() + key := "insert_log/100/10/2001/_data/conflict" + source, target := filepath.Join(displaced(root), key), filepath.Join(root, key) + sourceFile, targetFile := source, target + if conflictKind == "source-file-target-directory" { + targetFile = filepath.Join(target, "data") + } + if conflictKind == "source-directory-target-file" { + sourceFile = filepath.Join(source, "data") + } + writeFile(t, sourceFile, "legacy") + writeFile(t, targetFile, "canonical") + report, err := Migrate(t.Context(), root, Options{LegacyPrefix: "files"}) + require.ErrorIs(t, err, merr.ErrDataIntegrity) + assert.NotEmpty(t, report.Conflicts) + assert.Equal(t, "legacy", readFile(t, sourceFile)) + assert.Equal(t, "canonical", readFile(t, targetFile)) + }) + } +} + +func assertMigrationRestartIsIdle(t *testing.T, root string) { + t.Helper() + report, err := Migrate(t.Context(), root, Options{LegacyPrefix: "files"}) + require.NoError(t, err) + assert.Zero(t, report.Renamed) + assert.Zero(t, report.Bytes) +} + +// Only the old source filesystem rejects rename. Temporary-file publication +// inside the target filesystem must still execute the real system call. +func injectSourceRenameError(t *testing.T, sourceParent string, injectedErr error) *int { + t.Helper() + var parents []unix.Stat_t + require.NoError(t, filepath.WalkDir(sourceParent, func(name string, entry fs.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if entry.IsDir() { + var stat unix.Stat_t + if err := unix.Stat(name, &stat); err != nil { + return err + } + parents = append(parents, stat) + } + return nil + })) + var original func(int, string, int, string) error + injected := 0 + patch := mockey.Mock(unix.Renameat).To(func(oldfd int, oldname string, newfd int, newname string) error { + var actual unix.Stat_t + if err := unix.Fstat(oldfd, &actual); err == nil { + for _, expected := range parents { + if actual.Dev == expected.Dev && actual.Ino == expected.Ino { + injected++ + return injectedErr + } + } + } + return original(oldfd, oldname, newfd, newname) + }).Origin(&original).Build() + t.Cleanup(func() { patch.UnPatch() }) + return &injected +} diff --git a/internal/storage/localmigrate/migrate.go b/internal/storage/localmigrate/migrate.go new file mode 100644 index 0000000000..9e6dd0269d --- /dev/null +++ b/internal/storage/localmigrate/migrate.go @@ -0,0 +1,556 @@ +// Licensed to the LF AI & Data foundation under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package localmigrate repairs the double-root layout written by Milvus 3.0.0 +// and 3.0.1 and late 2.6 unified indexes written in the working directory. +// Migration merges directories without overwriting files, using rename where +// possible and copying across filesystems. No backup or migration journal is created. +// +// This package is unix-only: it needs a cross-root renameat and a +// directory-only unlinkat, neither of which os.Root offers (os.Root.Rename +// cannot cross roots, and os.Root.Remove would unlink a replacement file). +package localmigrate + +import ( + "context" + "fmt" + "io/fs" + "os" + "path/filepath" + "sort" + "strings" + "time" + + "github.com/gofrs/flock" + + "github.com/milvus-io/milvus/pkg/v3/util/merr" +) + +// LayoutDirs are layouts written through the formerly double-rooted Arrow +// filesystem. ChunkManager layouts such as delta_log and stats_log are not moved. +var LayoutDirs = []string{"insert_log", "text_log", "json_stats", "index_files", "index_v1"} + +const lockFileName = ".milvus-local-layout.lock" + +// reservedNamespace is a directory name this package keeps for itself under the +// storage root. It is distinct from lockFileName and is never a move source or +// a cleanup target, so a user directory that happens to use the name survives. +const reservedNamespace = ".milvus-local-layout" + +// maxReportedConflicts bounds the conflict list embedded in the returned error. +// Report.Conflicts keeps them all; an operator-facing message must not grow to +// the size of the storage root. +const maxReportedConflicts = 20 + +// Options controls automatic migration before the instance starts. +type Options struct { + // LegacyPrefix is minio.rootPath. Its insert_log namespace is still read in + // place by legacy manifests and must never be moved. + LegacyPrefix string + // OnPlan reports every directory this run decided to move, once, after + // discovery and conflict checking and before anything has moved. OnDirStart + // and OnDirDone then report each of those directories as it is moved, so a + // migration that takes minutes is visible while it runs instead of only in + // hindsight. All three are optional. + OnPlan func(dirs []Dir) + OnDirStart func(dir Dir) + OnDirDone func(dir Dir, elapsed time.Duration) +} + +// Dir describes one legacy directory the migration moves. Source and Target are +// absolute paths; Files and Bytes count the regular files below Source. +type Dir struct { + Source string + Target string + Files int + Bytes int64 +} + +// Report describes legacy directories moved in this run. +type Report struct { + Root string + Candidates []string + Conflicts []string + Renamed int + Copied int + Bytes int64 + Elapsed time.Duration +} + +func (r *Report) String() string { + return fmt.Sprintf("root=%s candidates=%v renamed=%d copied=%d bytes=%d conflicts=%d elapsed=%s", + r.Root, r.Candidates, r.Renamed, r.Copied, r.Bytes, len(r.Conflicts), r.Elapsed) +} + +// DisplacedRoots describes the one double-joined root; "/" has no displacement. +func DisplacedRoots(absRoot string) []string { + absRoot = filepath.Clean(absRoot) + if !filepath.IsAbs(absRoot) || absRoot == string(filepath.Separator) { + return nil + } + return []string{filepath.Join(absRoot, strings.TrimLeft(absRoot, string(filepath.Separator)))} +} + +type migrationSource struct { + kind string + root *os.Root + base string + physical string + cleanupDirs []string // Validated legacy directories in WalkDir order. +} + +type migrationDir struct { + source *migrationSource + path string + target string + files int + bytes int64 + dirs []string // Inspected directories, reused if only an empty skeleton remains. +} + +// describe renders the absolute source and target of d for reporting. +func (d migrationDir) describe(root *os.Root) Dir { + return Dir{ + Source: filepath.Join(d.source.root.Name(), d.path), + Target: filepath.Join(root.Name(), d.target), + Files: d.files, + Bytes: d.bytes, + } +} + +// Migrate merges old local layouts into the canonical storage root. Missing +// subtrees are renamed directly; cross-filesystem files are copied before their +// sources are removed. Existing directories are merged, never overwritten. +// A later startup resumes from the remaining sources without a migration journal. +// +// Canceling ctx stops the migration between filesystem reads and directories, which is +// safe precisely because it is resumable: the sources that have not moved yet +// are rediscovered on the next run. A partially copied temporary file is never +// published and is recopied from the intact source on the next run. +func Migrate(ctx context.Context, absRoot string, opts Options) (*Report, error) { + if !filepath.IsAbs(absRoot) { + return nil, merr.WrapErrParameterInvalidMsg("local storage root %q must be absolute", absRoot) + } + absRoot = filepath.Clean(absRoot) + startedAt := time.Now() + report := &Report{Root: absRoot} + defer func() { report.Elapsed = time.Since(startedAt) }() + if info, err := os.Stat(absRoot); err == nil && !info.IsDir() { + return report, merr.WrapErrParameterInvalidMsg("local storage root %q is not a directory", absRoot) + } + root, err := os.OpenRoot(absRoot) + if err != nil { + if os.IsNotExist(err) { + return report, nil + } + return report, merr.Wrapf(err, "open local storage root %s", absRoot) + } + defer root.Close() + cwd, err := openLegacyWorkingDirectory() + if err != nil { + return report, err + } + defer cwd.Close() + physicalRoot, err := filepath.EvalSymlinks(absRoot) + if err != nil { + return report, err + } + activeNamespace, err := legacyNamespace(physicalRoot, opts.LegacyPrefix) + if err != nil { + return report, err + } + if err := rejectSymlinks(root, lockFileName); err != nil { + return report, err + } + lock := flock.New(filepath.Join(root.Name(), lockFileName)) + locked, err := lock.TryLock() + if err != nil { + return report, merr.Wrap(err, "lock local layout migration") + } + if !locked { + return report, merr.WrapErrServiceUnavailableMsg("another local layout migration holds %s", lock.Path()) + } + defer lock.Unlock() + sources, err := migrationSources(root, cwd, physicalRoot, absRoot) + if err != nil { + return report, err + } + dirs, err := discoverDirectories(ctx, sources, physicalRoot, activeNamespace, report) + if err != nil { + return report, err + } + // CWD leaves are discovered through a map, so fix a stable order before + // anything is reported or moved: the plan, the move order and the log then + // agree across runs and are comparable after an interruption. + sort.SliceStable(dirs, func(i, j int) bool { + if dirs[i].source.root.Name() != dirs[j].source.root.Name() { + return dirs[i].source.root.Name() < dirs[j].source.root.Name() + } + return dirs[i].path < dirs[j].path + }) + if err := checkTargets(ctx, root, dirs, report); err != nil { + return report, err + } + if len(report.Conflicts) != 0 { + sort.Strings(report.Conflicts) + return report, merr.WrapErrDataIntegrityMsg("local layout files conflict (%d total): %v", + len(report.Conflicts), report.Conflicts[:min(len(report.Conflicts), maxReportedConflicts)]) + } + plan := make([]Dir, 0, len(dirs)) + for _, directory := range dirs { + plan = append(plan, directory.describe(root)) + } + if opts.OnPlan != nil { + opts.OnPlan(plan) + } + for index, directory := range dirs { + if err := ctx.Err(); err != nil { + return report, err + } + if opts.OnDirStart != nil { + opts.OnDirStart(plan[index]) + } + directoryStartedAt := time.Now() + if err := moveEntry(ctx, directory.source.root, directory.path, root, directory.target, report); err != nil { + return report, err + } + report.Bytes += directory.bytes + if opts.OnDirDone != nil { + opts.OnDirDone(plan[index], time.Since(directoryStartedAt)) + } + } + for _, source := range sources { + if err := removeEmptyLegacyDirectories(ctx, source); err != nil { + return report, err + } + } + return report, nil +} + +func openLegacyWorkingDirectory() (*os.Root, error) { + directory, err := os.Getwd() + if err != nil { + return nil, merr.Wrap(err, "resolve current working directory for local layout migration") + } + root, err := os.OpenRoot(filepath.Clean(directory)) + if err != nil { + return nil, merr.Wrapf(err, "open legacy working directory %s", directory) + } + return root, nil +} + +func migrationSources(root, cwd *os.Root, physicalRoot, configuredRoot string) ([]*migrationSource, error) { + var sources []*migrationSource + // R=/ has no displaced root, but may still have indexes under an old CWD. + if candidates := DisplacedRoots(configuredRoot); len(candidates) != 0 { + base, err := filepath.Rel(root.Name(), candidates[0]) + if err != nil { + return nil, err + } + sources = append(sources, &migrationSource{kind: "displaced", root: root, base: base, physical: physicalRoot}) + } + rootInfo, err := root.Stat(".") + if err != nil { + return nil, err + } + cwdInfo, err := cwd.Stat(".") + if err != nil { + return nil, err + } + if !os.SameFile(rootInfo, cwdInfo) { + // The displaced source already includes its text_log and index_files. + // Do not discover them twice when CWD names the same physical directory. + for _, source := range sources { + if info, err := source.root.Stat(source.base); err == nil && os.SameFile(info, cwdInfo) { + return sources, nil + } + // Leave missing or unsafe source validation to discovery below. + } + physicalCWD, err := filepath.EvalSymlinks(cwd.Name()) + if err != nil { + return nil, err + } + sources = append(sources, &migrationSource{kind: "cwd", root: cwd, base: ".", physical: physicalCWD}) + } + return sources, nil +} + +func legacyNamespace(root, prefix string) (string, error) { + prefix = filepath.Clean(prefix) + if !filepath.IsLocal(prefix) { + return "", merr.WrapErrParameterInvalidMsg("unsafe legacy minio.rootPath %q", prefix) + } + return filepath.Join(root, prefix, "insert_log"), nil +} + +func discoverDirectories(ctx context.Context, sources []*migrationSource, root, active string, report *Report) ([]migrationDir, error) { + var directories []migrationDir + for _, source := range sources { + if source.kind == "displaced" { + start := len(directories) + for _, layout := range LayoutDirs { + path := filepath.Join(source.base, layout) + if err := addDirectory(ctx, source, root, active, path, layout, &directories); err != nil { + return nil, err + } + } + if len(directories) != start { + report.Candidates = append(report.Candidates, filepath.Join(source.root.Name(), source.base)) + } + continue + } + // CWD indexes are moved at their deepest eligible directory. This keeps + // unrelated files in the old working directory untouched. + groups := make(map[string]*migrationDir) + for _, layout := range []string{"text_log", "index_files"} { + if err := collectLegacyIndexDirs(ctx, source, root, active, layout, groups); err != nil { + return nil, err + } + } + for _, directory := range groups { + directories = append(directories, *directory) + } + if len(groups) != 0 { + report.Candidates = append(report.Candidates, source.root.Name()) + } + } + return directories, nil +} + +func addDirectory(ctx context.Context, source *migrationSource, root, active, path, target string, directories *[]migrationDir) error { + if err := rejectSymlinks(source.root, path); err != nil { + return err + } + info, err := source.root.Lstat(path) + if os.IsNotExist(err) { + return nil + } + if err != nil { + return err + } + if !info.IsDir() { + return merr.WrapErrDataIntegrityMsg("legacy layout %s is not a directory", filepath.Join(source.root.Name(), path)) + } + directory := migrationDir{source: source, path: path, target: target} + if err := inspectDirectory(ctx, source.root, path, &directory); err != nil { + return err + } + if directory.files == 0 { + // An interrupted merge may have moved the last file but left its empty + // parents. Reuse discovery for cleanup after all migration work succeeds. + // Empty live/reserved namespaces must still be left untouched. + if protectNamespace(source, path, active, root) == nil { + source.cleanupDirs = append(source.cleanupDirs, directory.dirs...) + } + return nil + } + if err := protectNamespace(source, path, active, root); err != nil { + return err + } + *directories = append(*directories, directory) + return nil +} + +func collectLegacyIndexDirs(ctx context.Context, source *migrationSource, root, active, layout string, groups map[string]*migrationDir) error { + if err := rejectSymlinks(source.root, layout); err != nil { + return err + } + info, err := source.root.Lstat(layout) + if os.IsNotExist(err) { + return nil + } + if err != nil { + return err + } + if !info.IsDir() { + return merr.WrapErrDataIntegrityMsg("legacy index layout %s is not a directory", filepath.Join(source.root.Name(), layout)) + } + return fs.WalkDir(source.root.FS(), layout, func(name string, entry fs.DirEntry, walkErr error) error { + if err := ctx.Err(); err != nil { + return err + } + if walkErr != nil { + return walkErr + } + if entry.Type()&fs.ModeSymlink != 0 { + return merr.WrapErrDataIntegrityMsg("legacy source %s is a symlink; resolve manually", filepath.Join(source.root.Name(), name)) + } + if entry.IsDir() { + // The CWD is already resolved and WalkDir never follows symlinks. + physical := filepath.Join(source.physical, name) + if entersCanonicalRoot(source.physical, root, physical) { + return fs.SkipDir + } + // Include empty prefixes left by an interrupted migration, even when + // no packed file remains to identify a leaf. Never collect live paths + // or unrelated directories for cleanup. + if legacyIndexPath(name, true) && !pathsOverlap(physical, active) && + !pathsOverlap(physical, filepath.Join(root, reservedNamespace)) { + source.cleanupDirs = append(source.cleanupDirs, name) + } + return nil + } + if !legacyIndexPath(name, false) { + return nil + } + parent := filepath.Dir(name) + if directory := groups[parent]; directory != nil { + info, err := entry.Info() + if err != nil { + return err + } + directory.files++ + directory.bytes += info.Size() + return nil + } + directory := &migrationDir{source: source, path: parent, target: parent} + if err := protectNamespace(source, parent, active, root); err != nil { + return err + } + groups[parent] = directory + info, err := entry.Info() + if err != nil { + return err + } + if !info.Mode().IsRegular() { + return merr.WrapErrDataIntegrityMsg("legacy source %s is not a regular file", filepath.Join(source.root.Name(), name)) + } + directory.files = 1 + directory.bytes = info.Size() + return nil + }) +} + +func inspectDirectory(ctx context.Context, root *os.Root, path string, directory *migrationDir) error { + return fs.WalkDir(root.FS(), path, func(name string, entry fs.DirEntry, walkErr error) error { + if err := ctx.Err(); err != nil { + return err + } + if walkErr != nil { + return walkErr + } + if entry.Type()&fs.ModeSymlink != 0 { + return merr.WrapErrDataIntegrityMsg("legacy source %s is a symlink; resolve manually", filepath.Join(root.Name(), name)) + } + if entry.IsDir() { + if directory.files == 0 { + directory.dirs = append(directory.dirs, name) + } + return nil + } + info, err := entry.Info() + if err != nil { + return err + } + if !info.Mode().IsRegular() { + return merr.WrapErrDataIntegrityMsg("legacy source %s is not a regular file", filepath.Join(root.Name(), name)) + } + // Nonempty trees are moved normally; retain directory names only while + // this could still be an empty skeleton requiring cleanup. + directory.dirs = nil + directory.files++ + directory.bytes += info.Size() + return nil + }) +} + +func protectNamespace(source *migrationSource, path, active, root string) error { + original := filepath.Join(source.physical, path) + if source.kind == "displaced" { + original = filepath.Join(source.physical, source.base, filepath.Base(path)) + } + if pathsOverlap(original, active) { + return merr.WrapErrDataIntegrityMsg("legacy source %s overlaps the live manifest namespace %s; refusing to move", original, active) + } + if pathsOverlap(original, filepath.Join(root, reservedNamespace)) { + return merr.WrapErrDataIntegrityMsg("legacy source %s overlaps the reserved migration namespace", original) + } + return nil +} + +func withinDirectory(directory, name string) bool { + rel, err := filepath.Rel(directory, name) + return err == nil && filepath.IsLocal(rel) +} + +func pathsOverlap(a, b string) bool { + return withinDirectory(a, b) || withinDirectory(b, a) +} + +func entersCanonicalRoot(cwd, root, name string) bool { + return withinDirectory(cwd, root) && withinDirectory(root, name) +} + +// legacyIndexPath admits only the directory prefixes of eligible unified index files. +func legacyIndexPath(name string, directory bool) bool { + parts := strings.Split(filepath.ToSlash(name), "/") + ids := 4 + if parts[0] == "text_log" { + ids = 6 + } else if parts[0] != "index_files" { + return false + } + for i := 1; i < len(parts) && i <= ids; i++ { + if !isLegacyIndexID(parts[i]) { + return false + } + } + if directory { + return len(parts) <= ids+1 + } + return len(parts) == ids+2 && isLegacyUnifiedIndexFile(parts[len(parts)-1]) +} + +func isLegacyIndexID(value string) bool { + if value == "" { + return false + } + for _, ch := range value { + if ch < '0' || ch > '9' { + return false + } + } + return true +} + +func isLegacyUnifiedIndexFile(name string) bool { + // A discovery marker, also moved last so interrupted CWD leaves stay recognizable. + const prefix, suffix = "milvus_packed_", "_index.v3" + return len(name) > len(prefix)+len(suffix) && strings.HasPrefix(name, prefix) && strings.HasSuffix(name, suffix) +} + +// os.Root confines access but permits in-root symlinks. Migration must not +// follow those aliases into a different live directory, including at mutation. +func rejectSymlinks(root *os.Root, name string) error { + if !filepath.IsLocal(name) { + return merr.WrapErrDataIntegrityMsg("migration path %q is outside its opened root", name) + } + current := "" + for _, component := range strings.Split(filepath.Clean(name), string(filepath.Separator)) { + current = filepath.Join(current, component) + info, err := root.Lstat(current) + if os.IsNotExist(err) { + return nil + } + if err != nil { + return err + } + if info.Mode()&fs.ModeSymlink != 0 { + return merr.WrapErrDataIntegrityMsg("migration path %s is a symlink; resolve manually", filepath.Join(root.Name(), current)) + } + } + return nil +} diff --git a/internal/storage/localmigrate/migrate_test.go b/internal/storage/localmigrate/migrate_test.go new file mode 100644 index 0000000000..75ab3d043d --- /dev/null +++ b/internal/storage/localmigrate/migrate_test.go @@ -0,0 +1,189 @@ +// Licensed to the LF AI & Data foundation under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 + +package localmigrate + +import ( + "fmt" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/bytedance/mockey" + "github.com/gofrs/flock" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/milvus-io/milvus/pkg/v3/util/merr" +) + +func writeFile(t *testing.T, p, content string) { + t.Helper() + require.NoError(t, os.MkdirAll(filepath.Dir(p), 0o755)) + require.NoError(t, os.WriteFile(p, []byte(content), 0o600)) +} + +func readFile(t *testing.T, p string) string { + t.Helper() + b, err := os.ReadFile(p) + require.NoError(t, err) + return string(b) +} + +func displaced(root string) string { + return filepath.Join(root, strings.TrimPrefix(root, string(filepath.Separator))) +} + +func TestDisplacedRoots(t *testing.T) { + assert.Equal(t, []string{"/var/lib/milvus/data/var/lib/milvus/data"}, DisplacedRoots("/var/lib/milvus/data/")) + assert.Equal(t, []string{"/r/r"}, DisplacedRoots("/r")) + assert.Empty(t, DisplacedRoots("data")) + assert.Empty(t, DisplacedRoots("")) + assert.Empty(t, DisplacedRoots("/")) +} + +func TestMigrateRenamesDisplacedLayouts(t *testing.T) { + t.Chdir(t.TempDir()) + root := t.TempDir() + for _, layout := range []string{"insert_log", "text_log", "json_stats", "index_files", "index_v1"} { + writeFile(t, filepath.Join(displaced(root), layout, "1", "data"), layout) + } + writeFile(t, filepath.Join(displaced(root), "delta_log", "1", "data"), "untouched") + report, err := Migrate(t.Context(), root, Options{LegacyPrefix: "files"}) + require.NoError(t, err) + assert.Equal(t, 5, report.Renamed) + assert.EqualValues(t, 10+8+10+11+8, report.Bytes) + for _, layout := range []string{"insert_log", "text_log", "json_stats", "index_files", "index_v1"} { + assert.Equal(t, layout, readFile(t, filepath.Join(root, layout, "1", "data"))) + assert.NoDirExists(t, filepath.Join(displaced(root), layout)) + } + assert.Equal(t, "untouched", readFile(t, filepath.Join(displaced(root), "delta_log", "1", "data"))) + second, err := Migrate(t.Context(), root, Options{LegacyPrefix: "files"}) + require.NoError(t, err) + assert.Zero(t, second.Renamed) +} + +func TestMigrateMergesEmptyTargetDirectory(t *testing.T) { + t.Chdir(t.TempDir()) + root := t.TempDir() + key := "insert_log/1/2/3/data" + source := filepath.Join(displaced(root), key) + target := filepath.Join(root, "insert_log") + writeFile(t, source, "source") + require.NoError(t, os.MkdirAll(target, 0o755)) + report, err := Migrate(t.Context(), root, Options{LegacyPrefix: "files"}) + require.NoError(t, err) + assert.Empty(t, report.Conflicts) + assert.Equal(t, "source", readFile(t, filepath.Join(root, key))) + assert.NoFileExists(t, source) +} + +func TestMigrateLeavesLegacyMinioNamespaceUntouched(t *testing.T) { + for _, prefix := range []string{"files", "./files", "files//", "files/tmp/.."} { + t.Run(prefix, func(t *testing.T) { + testMigrateLeavesLegacyMinioNamespaceUntouched(t, prefix) + }) + } +} + +func testMigrateLeavesLegacyMinioNamespaceUntouched(t *testing.T, prefix string) { + t.Chdir(t.TempDir()) + root := t.TempDir() + key := "insert_log/1/2/5/_manifest/1.avro" + writeFile(t, filepath.Join(root, "files", key), "relative-manifest-data") + source := filepath.Join(displaced(root), key) + writeFile(t, source, "complete-key-data") + report, err := Migrate(t.Context(), root, Options{LegacyPrefix: prefix}) + require.NoError(t, err) + assert.Equal(t, 1, report.Renamed) + assert.Equal(t, "complete-key-data", readFile(t, filepath.Join(root, key))) + assert.Equal(t, "relative-manifest-data", readFile(t, filepath.Join(root, "files", key))) +} + +func TestMigrateValidatesRootAndLegacyPrefix(t *testing.T) { + t.Chdir(t.TempDir()) + for _, root := range []string{"", "relative/root"} { + _, err := Migrate(t.Context(), root, Options{}) + require.ErrorIs(t, err, merr.ErrParameterInvalid) + } + file := filepath.Join(t.TempDir(), "file") + writeFile(t, file, "data") + _, err := Migrate(t.Context(), file, Options{}) + require.ErrorIs(t, err, merr.ErrParameterInvalid) + for _, prefix := range []string{"../outside", "/absolute"} { + _, err := Migrate(t.Context(), t.TempDir(), Options{LegacyPrefix: prefix}) + require.Error(t, err) + } +} + +func TestMigrateFilesystemRootStillMigratesCWD(t *testing.T) { + for _, withIndex := range []bool{false, true} { + t.Run(fmt.Sprintf("index=%t", withIndex), func(t *testing.T) { + cwd, sandbox := t.TempDir(), t.TempDir() + t.Chdir(cwd) + key := "index_files/7/0/2/3/milvus_packed_inverted_index.v3" + if withIndex { + writeFile(t, filepath.Join(cwd, key), "old-index") + } + // Exercise the configured filesystem root without writing to the + // machine's actual root directory. + var original func(string) (*os.Root, error) + mock := mockey.Mock(os.OpenRoot).Origin(&original).To(func(name string) (*os.Root, error) { + if name == "/" { + return original(sandbox) + } + return original(name) + }).Build() + defer mock.UnPatch() + report, err := Migrate(t.Context(), "/", Options{LegacyPrefix: "files"}) + require.NoError(t, err) + if withIndex { + assert.Equal(t, "old-index", readFile(t, filepath.Join(sandbox, key))) + assert.NoFileExists(t, filepath.Join(cwd, key)) + } else { + assert.Zero(t, report.Renamed) + } + }) + } +} + +func TestMigrateLockContention(t *testing.T) { + t.Chdir(t.TempDir()) + root := t.TempDir() + writeFile(t, filepath.Join(displaced(root), "insert_log/1/data"), "data") + lock := flock.New(filepath.Join(root, lockFileName)) + require.NoError(t, lock.Lock()) + defer lock.Unlock() + _, err := Migrate(t.Context(), root, Options{LegacyPrefix: "files"}) + require.ErrorIs(t, err, merr.ErrServiceUnavailable) +} + +func TestMigrateRejectsSymlinks(t *testing.T) { + t.Chdir(t.TempDir()) + root, outside := t.TempDir(), t.TempDir() + writeFile(t, filepath.Join(outside, "sentinel"), "outside") + require.NoError(t, os.MkdirAll(filepath.Dir(displaced(root)), 0o755)) + require.NoError(t, os.Symlink(outside, displaced(root))) + _, err := Migrate(t.Context(), root, Options{LegacyPrefix: "files"}) + require.Error(t, err) + assert.Equal(t, "outside", readFile(t, filepath.Join(outside, "sentinel"))) +} + +func TestMigrateProtectsLiveManifestNamespace(t *testing.T) { + t.Chdir(t.TempDir()) + root := t.TempDir() + prefix := strings.TrimPrefix(root, string(filepath.Separator)) + source := filepath.Join(displaced(root), "insert_log/1/2/3/data") + writeFile(t, source, "live") + _, err := Migrate(t.Context(), root, Options{LegacyPrefix: prefix}) + require.ErrorIs(t, err, merr.ErrDataIntegrity) + assert.Equal(t, "live", readFile(t, source)) +} diff --git a/internal/storage/localmigrate/move.go b/internal/storage/localmigrate/move.go new file mode 100644 index 0000000000..ae6bd96288 --- /dev/null +++ b/internal/storage/localmigrate/move.go @@ -0,0 +1,412 @@ +// Licensed to the LF AI & Data foundation under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 + +package localmigrate + +import ( + "bytes" + "context" + "crypto/sha256" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "sort" + + "github.com/cockroachdb/errors" + "golang.org/x/sys/unix" + + "github.com/milvus-io/milvus/pkg/v3/util/merr" +) + +// Check actual entries, not overlapping directory prefixes: legacy JSON stats +// can split even a single segment's _stats directory between the two roots. +func checkTargets(ctx context.Context, root *os.Root, dirs []migrationDir, report *Report) error { + planned := make(map[string]bool) + for _, dir := range dirs { + err := fs.WalkDir(dir.source.root.FS(), dir.path, func(name string, entry fs.DirEntry, err error) error { + if ctxErr := ctx.Err(); ctxErr != nil { + return ctxErr + } + if err != nil { + return err + } + if !entry.IsDir() && !entry.Type().IsRegular() { + return merr.WrapErrDataIntegrityMsg("legacy source %s is not a regular file", name) + } + rel, err := filepath.Rel(dir.path, name) + if err != nil { + return err + } + target := filepath.Join(dir.target, rel) + isDir, seen := planned[target] + conflict := seen && (!isDir || !entry.IsDir()) + planned[target] = entry.IsDir() + info, err := root.Lstat(target) + switch { + case err == nil: + if !entry.IsDir() || !info.IsDir() { + published, err := publishedCopy(dir.source.root, name, root, target) + if err != nil { + return err + } + conflict = conflict || !published + } + case errors.Is(err, unix.ENOTDIR): + conflict = true + case !os.IsNotExist(err): + return err + } + if conflict { + report.Conflicts = append(report.Conflicts, filepath.Join(root.Name(), target)) + if entry.IsDir() { + return fs.SkipDir + } + } + return nil + }) + if err != nil { + return err + } + } + return nil +} + +func moveEntry(ctx context.Context, source *os.Root, name string, root *os.Root, target string, report *Report) error { + if err := ctx.Err(); err != nil { + return err + } + if err := rejectSymlinks(source, name); err != nil { + return err + } + if err := rejectSymlinks(root, target); err != nil { + return err + } + info, err := source.Lstat(name) + if err != nil { + return err + } + dest, err := root.Lstat(target) + if os.IsNotExist(err) { + if err := root.MkdirAll(filepath.Dir(target), 0o755); err != nil { + return err + } + if err := renameEntry(source, name, root, target); err == nil { + report.Renamed++ + return nil + } else if !errors.Is(err, unix.EXDEV) { + return merr.Wrapf(err, "rename local layout %s to %s", name, target) + } + if !info.IsDir() { + return copyFile(ctx, source, name, root, target, report) + } + if err := root.Mkdir(target, info.Mode().Perm()); err != nil { + return err + } + } else if err != nil { + return err + } else if !info.IsDir() || !dest.IsDir() { + published, err := publishedCopy(source, name, root, target) + if err != nil { + return err + } + if published { + return finishCopy(source, name, root, target, report) + } + return merr.WrapErrDataIntegrityMsg("local layout file already exists: %s", filepath.Join(root.Name(), target)) + } + entries, err := fs.ReadDir(source.FS(), name) + if err != nil { + return err + } + // Keep a CWD leaf recognizable after an interruption while moving auxiliary + // files: its packed index files are the last entries removed from the source. + sort.SliceStable(entries, func(i, j int) bool { + return !isLegacyUnifiedIndexFile(entries[i].Name()) && isLegacyUnifiedIndexFile(entries[j].Name()) + }) + for _, entry := range entries { + if err := moveEntry(ctx, source, filepath.Join(name, entry.Name()), root, filepath.Join(target, entry.Name()), report); err != nil { + return err + } + } + return source.Remove(name) // Only the now-empty source directory is removed. +} + +// removeEmptyLegacyDirectories prunes children before parents using the existing +// discovery order. Rmdir leaves nonempty directories and replacement files alone; +// it never removes the working directory itself or recursively deletes content. +func removeEmptyLegacyDirectories(ctx context.Context, source *migrationSource) error { + for i := len(source.cleanupDirs) - 1; i >= 0; i-- { + if err := ctx.Err(); err != nil { + return err + } + name := source.cleanupDirs[i] + if err := rejectSymlinks(source.root, name); err != nil { + if errors.Is(err, unix.ENOTDIR) { + continue + } + return err + } + parent, err := source.root.Open(filepath.Dir(name)) + if os.IsNotExist(err) || errors.Is(err, unix.ENOTDIR) { + continue + } + if err != nil { + return merr.Wrapf(err, "open legacy parent for cleanup %s", name) + } + err = unix.Unlinkat(int(parent.Fd()), filepath.Base(name), unix.AT_REMOVEDIR) + closeErr := parent.Close() + if err != nil && !os.IsNotExist(err) && !errors.Is(err, unix.ENOTEMPTY) && + !errors.Is(err, unix.EEXIST) && !errors.Is(err, unix.ENOTDIR) { + return merr.Wrapf(err, "remove empty legacy directory %s", name) + } + if closeErr != nil { + return merr.Wrapf(closeErr, "close legacy parent for cleanup %s", name) + } + } + return nil +} + +func renameEntry(source *os.Root, name string, root *os.Root, target string) error { + from, err := source.Open(filepath.Dir(name)) + if err != nil { + return err + } + defer from.Close() + to, err := root.Open(filepath.Dir(target)) + if err != nil { + return err + } + defer to.Close() + return unix.Renameat(int(from.Fd()), filepath.Base(name), int(to.Fd()), filepath.Base(target)) +} + +// The hash identifies one source-to-target publication, not its contents. +// The data temporary file is renamed into place; a small sidecar marker remains +// until the source removal is durable so a restart can recognize that target as +// this migration's own completed copy. +func copyArtifactBase(source *os.Root, name, target string) string { + sum := sha256.Sum256([]byte(source.Name() + "\x00" + name + "\x00" + target)) + return filepath.Join(filepath.Dir(target), fmt.Sprintf(".milvus-local-copy-%x", sum)) +} + +func copyTempPath(source *os.Root, name, target string) string { + return copyArtifactBase(source, name, target) + ".tmp" +} + +func copyMarkerPath(source *os.Root, name, target string) string { + return copyArtifactBase(source, name, target) + ".published" +} + +func copyMarkerPayload(source *os.Root, name, target string, size int64) []byte { + return []byte(fmt.Sprintf("milvus-local-copy-v1\n%s\n%d\n", copyArtifactBase(source, name, target), size)) +} + +func publishedCopy(source *os.Root, name string, root *os.Root, target string) (bool, error) { + markerPath := copyMarkerPath(source, name, target) + marker, err := root.Lstat(markerPath) + if os.IsNotExist(err) { + return false, nil + } + if err != nil { + return false, err + } + if !marker.Mode().IsRegular() { + return false, merr.WrapErrDataIntegrityMsg("invalid local migration publication marker for %s", target) + } + sourceInfo, err := source.Lstat(name) + if err != nil { + return false, err + } + expectedMarker := copyMarkerPayload(source, name, target, sourceInfo.Size()) + actualMarker, err := root.ReadFile(markerPath) + if err != nil { + return false, err + } + if !bytes.Equal(actualMarker, expectedMarker) { + return false, merr.WrapErrDataIntegrityMsg("invalid local migration publication marker for %s", target) + } + dest, err := root.Lstat(target) + if os.IsNotExist(err) { + return false, nil + } + return err == nil && dest.Mode().IsRegular() && dest.Size() == sourceInfo.Size(), err +} + +func copyFile(ctx context.Context, source *os.Root, name string, root *os.Root, target string, report *Report) error { + if err := ctx.Err(); err != nil { + return err + } + in, err := source.Open(name) + if err != nil { + return err + } + defer in.Close() + info, err := in.Stat() + if err != nil { + return err + } + if !info.Mode().IsRegular() { + return merr.WrapErrDataIntegrityMsg("legacy source %s is not a regular file", name) + } + temp := copyTempPath(source, name, target) + marker := copyMarkerPath(source, name, target) + for _, artifact := range []string{temp, marker} { + if err := rejectSymlinks(root, artifact); err != nil { + return err + } + if previous, err := root.Lstat(artifact); err == nil { + if !previous.Mode().IsRegular() { + return merr.WrapErrDataIntegrityMsg("invalid local migration copy artifact for %s", target) + } + } else if !os.IsNotExist(err) { + return err + } + // A previous interrupted copy was never published. Recopy from the + // intact source; never expose a partially written file under the final + // name. + if err := root.Remove(artifact); err != nil && !os.IsNotExist(err) { + return err + } + } + out, err := root.OpenFile(temp, os.O_CREATE|os.O_EXCL|os.O_WRONLY, info.Mode().Perm()) + if err != nil { + return err + } + published := false + defer func() { + out.Close() + if !published { + _ = root.Remove(temp) + _ = root.Remove(marker) + } + }() + if _, err := io.Copy(out, &contextReader{ctx: ctx, reader: in}); err != nil { + return merr.Wrapf(err, "copy local layout %s to %s", name, target) + } + if err := ctx.Err(); err != nil { + return err + } + if err := out.Sync(); err != nil { + return err + } + if err := out.Close(); err != nil { + return err + } + markerFile, err := root.OpenFile(marker, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600) + if err != nil { + return err + } + markerOpen := true + defer func() { + if markerOpen { + markerFile.Close() + } + }() + if _, err := markerFile.Write(copyMarkerPayload(source, name, target, info.Size())); err != nil { + return err + } + if err := markerFile.Sync(); err != nil { + return err + } + if err := markerFile.Close(); err != nil { + return err + } + markerOpen = false + // Persist the proof and complete temporary file before publishing it. If + // the process stops after the rename, the next startup can finish source + // removal instead of treating its own target as an unrelated conflict. + if err := syncDirectory(root, filepath.Dir(target)); err != nil { + return err + } + if err := renameEntry(root, temp, root, target); err != nil { + return err + } + published = true + if err := ctx.Err(); err != nil { + return err + } + return finishCopy(source, name, root, target, report) +} + +type contextReader struct { + ctx context.Context + reader io.Reader +} + +func (r *contextReader) Read(buffer []byte) (int, error) { + if err := r.ctx.Err(); err != nil { + return 0, err + } + return r.reader.Read(buffer) +} + +func syncDirectory(root *os.Root, directory string) error { + opened, err := root.Open(directory) + if err != nil { + return err + } + err = opened.Sync() + closeErr := opened.Close() + if err != nil { + return err + } + return closeErr +} + +// Persist the target entry and every ancestor link that MkdirAll may have +// created. Syncing only the leaf directory does not make new parents durable. +// Repeat this on recovery too: an existing directory may be from an interrupted +// migration that never finished syncing its parents. +func syncCopyTargetDirectories(root *os.Root, directory string) error { + for { + parent, err := root.Open(directory) + if err != nil { + return err + } + err = parent.Sync() + closeErr := parent.Close() + if err != nil { + return err + } + if closeErr != nil { + return closeErr + } + if directory == "." { + return nil + } + directory = filepath.Dir(directory) + } +} + +func finishCopy(source *os.Root, name string, root *os.Root, target string, report *Report) error { + if err := syncCopyTargetDirectories(root, filepath.Dir(target)); err != nil { + return err + } + sourceParent, err := source.Open(filepath.Dir(name)) + if err != nil { + return err + } + defer sourceParent.Close() + if err := source.Remove(name); err != nil { + return err + } + // Persist source removal before dropping the publication marker. Otherwise + // a power loss could resurrect the source without proof that target is ours. + if err := sourceParent.Sync(); err != nil { + return err + } + // Best-effort: the copy is already published and the source is gone, so the + // migration is complete. The sidecar is only retry provenance, not data. + _ = root.Remove(copyMarkerPath(source, name, target)) + report.Copied++ + return nil +} diff --git a/internal/storage/localmigrate/progress_test.go b/internal/storage/localmigrate/progress_test.go new file mode 100644 index 0000000000..edfa40ed2c --- /dev/null +++ b/internal/storage/localmigrate/progress_test.go @@ -0,0 +1,140 @@ +// Licensed to the LF AI & Data foundation under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package localmigrate + +import ( + "context" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// The plan is what a slow or interrupted migration has to be reconciled against, +// so every directory must be named before anything moves, and again as it moves +// and once it is done. +func TestMigrateReportsPlanAndEveryDirectory(t *testing.T) { + t.Chdir(t.TempDir()) + root := t.TempDir() + layouts := []string{"index_files", "insert_log", "json_stats"} + for _, layout := range layouts { + writeFile(t, filepath.Join(displaced(root), layout, "1", "data"), layout) + } + + var plan, starts, dones []Dir + var elapsed []time.Duration + report, err := Migrate(t.Context(), root, Options{ + LegacyPrefix: "files", + OnPlan: func(dirs []Dir) { plan = append(plan, dirs...) }, + OnDirStart: func(dir Dir) { starts = append(starts, dir) }, + OnDirDone: func(dir Dir, took time.Duration) { + dones = append(dones, dir) + elapsed = append(elapsed, took) + }, + }) + require.NoError(t, err) + + // Discovery walks a map for CWD leaves, so the order is fixed explicitly. + sources := make([]string, 0, len(plan)) + targets := make([]string, 0, len(plan)) + for _, dir := range plan { + sources = append(sources, dir.Source) + targets = append(targets, dir.Target) + assert.Equal(t, 1, dir.Files) + assert.Positive(t, dir.Bytes) + } + wantSources := make([]string, 0, len(layouts)) + wantTargets := make([]string, 0, len(layouts)) + for _, layout := range layouts { + wantSources = append(wantSources, filepath.Join(displaced(root), layout)) + wantTargets = append(wantTargets, filepath.Join(root, layout)) + } + assert.Equal(t, wantSources, sources) + assert.Equal(t, wantTargets, targets) + + // Every planned directory is reported starting and finishing, in plan order. + assert.Equal(t, plan, starts) + assert.Equal(t, plan, dones) + assert.Len(t, elapsed, len(layouts)) + assert.Positive(t, report.Elapsed) +} + +func TestMigrateWithoutLegacyDirectoriesReportsAnEmptyPlan(t *testing.T) { + t.Chdir(t.TempDir()) + root := t.TempDir() + + planned := 0 + moved := 0 + report, err := Migrate(t.Context(), root, Options{ + LegacyPrefix: "files", + OnPlan: func(dirs []Dir) { planned = len(dirs) }, + OnDirStart: func(Dir) { moved++ }, + }) + + require.NoError(t, err) + assert.Zero(t, planned) + assert.Zero(t, moved) + assert.Zero(t, report.Renamed+report.Copied) +} + +// An expired deadline must surface as a failure rather than a partially applied +// layout that looks finished. +func TestMigrateExpiredDeadlineFailsBeforeMoving(t *testing.T) { + t.Chdir(t.TempDir()) + root := t.TempDir() + key := filepath.Join("insert_log", "1", "data") + writeFile(t, filepath.Join(displaced(root), key), "payload") + + ctx, cancel := context.WithDeadline(t.Context(), time.Now().Add(-time.Second)) + defer cancel() + _, err := Migrate(ctx, root, Options{LegacyPrefix: "files"}) + + require.ErrorIs(t, err, context.DeadlineExceeded) + // The source is untouched, so a later run still has everything to move. + assert.Equal(t, "payload", readFile(t, filepath.Join(displaced(root), key))) +} + +// Cancellation is safe because the migration is resumable: whatever has not +// moved is rediscovered, so a restart with more time finishes the job. +func TestMigrateCancelledMidwayResumesOnRestart(t *testing.T) { + t.Chdir(t.TempDir()) + root := t.TempDir() + layouts := []string{"index_files", "insert_log", "json_stats"} + for _, layout := range layouts { + writeFile(t, filepath.Join(displaced(root), layout, "1", "data"), layout) + } + + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + first, err := Migrate(ctx, root, Options{ + LegacyPrefix: "files", + // Stop after the first directory is fully in place. + OnDirDone: func(Dir, time.Duration) { cancel() }, + }) + require.ErrorIs(t, err, context.Canceled) + require.Equal(t, 1, first.Renamed+first.Copied) + assert.Equal(t, "index_files", readFile(t, filepath.Join(root, "index_files", "1", "data"))) + + second, err := Migrate(t.Context(), root, Options{LegacyPrefix: "files"}) + require.NoError(t, err) + assert.Equal(t, len(layouts)-1, second.Renamed+second.Copied) + for _, layout := range layouts { + assert.Equal(t, layout, readFile(t, filepath.Join(root, layout, "1", "data"))) + } +} diff --git a/internal/storage/record_reader.go b/internal/storage/record_reader.go index 93752bdb95..5c09cc28d8 100644 --- a/internal/storage/record_reader.go +++ b/internal/storage/record_reader.go @@ -221,7 +221,7 @@ func newIterativePackedRecordReader( type ManifestReader struct { fieldBinlogs []*datapb.FieldBinlog manifest string - reader *packed.FFIPackedReader + reader manifestArrowReader bufferSize int64 arrowSchema *arrow.Schema @@ -231,10 +231,16 @@ type ManifestReader struct { storageConfig *indexpb.StorageConfig storagePluginContext *indexcgopb.StoragePluginContext externalSpecContext packed.ExternalSpecContext + textColumnConfigs []packed.TextColumnConfig neededColumns []string } +type manifestArrowReader interface { + ReadNext() (arrow.Record, error) + Close() error +} + // NewManifestReaderFromBinlogs creates a ManifestReader from binlogs func NewManifestReaderFromBinlogs(fieldBinlogs []*datapb.FieldBinlog, schema *schemapb.CollectionSchema, @@ -309,6 +315,31 @@ func NewManifestReaderWithExtfs( storageConfig *indexpb.StorageConfig, storagePluginContext *indexcgopb.StoragePluginContext, extfs packed.ExternalSpecContext, +) (*ManifestReader, error) { + return newManifestReader(manifest, schema, bufferSize, storageConfig, storagePluginContext, extfs, nil) +} + +// NewTextDecodedManifestReader opens a manifest through milvus-storage's +// SegmentReader so TEXT reference columns are resolved through source-specific +// LOB paths and returned as logical UTF8 strings. +func NewTextDecodedManifestReader( + manifest string, + schema *schemapb.CollectionSchema, + bufferSize int64, + storageConfig *indexpb.StorageConfig, + textColumnConfigs []packed.TextColumnConfig, +) (*ManifestReader, error) { + return newManifestReader(manifest, schema, bufferSize, storageConfig, nil, packed.ExternalSpecContext{}, textColumnConfigs) +} + +func newManifestReader( + manifest string, + schema *schemapb.CollectionSchema, + bufferSize int64, + storageConfig *indexpb.StorageConfig, + storagePluginContext *indexcgopb.StoragePluginContext, + extfs packed.ExternalSpecContext, + textColumnConfigs []packed.TextColumnConfig, ) (*ManifestReader, error) { columnResolver := typeutil.NewStorageColumnResolver(schema, typeutil.WithStorageColumnExternalSpec(extfs.Spec)) arrowSchema, err := ConvertToArrowSchemaWithNameResolver( @@ -328,7 +359,7 @@ func NewManifestReaderWithExtfs( // RecordToInsertData conversion does not accidentally decode internal LOB // references as user text. Any source type coercion must stay in the // external-source normalization path, not in the internal manifest path. - if !typeutil.IsExternalCollection(schema) || columnResolver.IsMilvusTable() { + if len(textColumnConfigs) == 0 && (!typeutil.IsExternalCollection(schema) || columnResolver.IsMilvusTable()) { arrowSchema = overrideTextFieldsToBinaryByFields( columnResolver.ManifestStoredFields(), arrowSchema, @@ -360,6 +391,7 @@ func NewManifestReaderWithExtfs( storageConfig: storageConfig, storagePluginContext: storagePluginContext, externalSpecContext: extfs, + textColumnConfigs: textColumnConfigs, neededColumns: neededColumns, } @@ -373,6 +405,15 @@ func NewManifestReaderWithExtfs( } func (mr *ManifestReader) init() error { + if len(mr.textColumnConfigs) > 0 { + reader, err := packed.NewFFISegmentReader(mr.manifest, mr.arrowSchema, mr.neededColumns, + mr.bufferSize, mr.storageConfig, mr.textColumnConfigs) + if err != nil { + return err + } + mr.reader = reader + return nil + } reader, err := packed.NewFFIPackedReader(mr.manifest, mr.arrowSchema, mr.neededColumns, mr.bufferSize, mr.storageConfig, diff --git a/internal/storage/remote_chunk_manager.go b/internal/storage/remote_chunk_manager.go index 156f432c23..113133e44e 100644 --- a/internal/storage/remote_chunk_manager.go +++ b/internal/storage/remote_chunk_manager.go @@ -321,6 +321,14 @@ func (mcm *RemoteChunkManager) MultiRemove(ctx context.Context, keys []string) e // RemoveWithPrefix removes all objects with the same prefix @prefix from minio. func (mcm *RemoteChunkManager) RemoveWithPrefix(ctx context.Context, prefix string) error { + // An empty prefix matches every object in the bucket, so this API would + // erase the whole instance. LocalChunkManager refuses it for the same + // reason; the guard belongs here rather than in each caller. + if len(prefix) == 0 { + errMsg := "empty prefix is not allowed for ChunkManager remove operation" + mlog.Warn(ctx, errMsg) + return merr.WrapErrStorageMsg("%s", errMsg) + } // removeObject in parallel. runningGroup, _ := errgroup.WithContext(ctx) runningGroup.SetLimit(10) diff --git a/internal/storage/rw.go b/internal/storage/rw.go index 0f8fcc4639..7cba024356 100644 --- a/internal/storage/rw.go +++ b/internal/storage/rw.go @@ -443,6 +443,48 @@ func NewManifestRecordReader(ctx context.Context, manifestPath string, neededSch return NewAbsentFieldFillRecordReader(inner, neededSchema, present), nil } +// NewTextDecodedManifestRecordReader resolves persisted TEXT references through +// source-specific LOB paths and returns logical UTF8 columns. It is intended for +// compaction that must rewrite LOB payloads into a different partition namespace. +func NewTextDecodedManifestRecordReader( + _ context.Context, + manifestPath string, + neededSchema *schemapb.CollectionSchema, + textColumnConfigs []packed.TextColumnConfig, + option ...RwOption, +) (RecordReader, error) { + rwOptions := DefaultReaderOptions() + for _, opt := range option { + opt(rwOptions) + } + if err := rwOptions.validate(); err != nil { + return nil, err + } + present := rwOptions.presentFields + if present == nil { + var err error + present, err = packed.GetManifestFieldIDs(manifestPath, rwOptions.storageConfig) + if err != nil { + return nil, err + } + } + presentSchema, err := filterSchemaToPresentFields(neededSchema, present) + if err != nil { + return nil, err + } + inner, err := NewTextDecodedManifestReader( + manifestPath, + presentSchema, + rwOptions.bufferSize, + rwOptions.storageConfig, + textColumnConfigs, + ) + if err != nil { + return nil, err + } + return NewAbsentFieldFillRecordReader(inner, neededSchema, present), nil +} + // filterSchemaToPresentFields returns a copy of schema keeping only the fields // (and struct sub-fields) whose FieldID is physically present, so the packed // reader is asked to read only stored columns; absent fields are then filled by diff --git a/internal/storage/rw_test.go b/internal/storage/rw_test.go index ebdf055f69..f03d79f626 100644 --- a/internal/storage/rw_test.go +++ b/internal/storage/rw_test.go @@ -41,8 +41,10 @@ import ( "github.com/milvus-io/milvus/internal/storagecommon" "github.com/milvus-io/milvus/internal/storagev2/packed" "github.com/milvus-io/milvus/pkg/v3/common" + "github.com/milvus-io/milvus/pkg/v3/objectstorage" "github.com/milvus-io/milvus/pkg/v3/proto/datapb" "github.com/milvus-io/milvus/pkg/v3/proto/indexpb" + "github.com/milvus-io/milvus/pkg/v3/proto/querypb" "github.com/milvus-io/milvus/pkg/v3/util/merr" "github.com/milvus-io/milvus/pkg/v3/util/metautil" "github.com/milvus-io/milvus/pkg/v3/util/paramtable" @@ -812,6 +814,20 @@ func (s *PackedBinlogRecordSuite) TestV3StatsWrittenUnderBasePath() { assert.NotContains(s.T(), p, "stats_log", "bloom filter stat path must not use legacy stats_log/ layout") } + + // QueryNode resolves V3 bloom-filter paths from SegmentLoadInfo and passes + // them unchanged to ChunkManager.MultiRead. Exercise that exact boundary so + // a regression to bucket-relative local paths cannot be hidden by the test + // joining localStorage.path itself. + loadInfo := &querypb.SegmentLoadInfo{ManifestPath: manifestPath} + resolvedPaths, err := packed.NewStatsResolverFromLoadInfo(loadInfo).BloomFilterPaths(pkField.GetFieldID()) + require.NoError(s.T(), err) + require.Equal(s.T(), bfStat.Paths, resolvedPaths) + + chunkManager := NewLocalChunkManager(objectstorage.RootPath(dir)) + values, err := chunkManager.MultiRead(s.ctx, resolvedPaths) + require.NoError(s.T(), err) + require.Len(s.T(), values, len(resolvedPaths)) } func genRowWithBM25(magic int64) map[int64]interface{} { diff --git a/internal/storage/types.go b/internal/storage/types.go index 20214b9ae9..e563070f6c 100644 --- a/internal/storage/types.go +++ b/internal/storage/types.go @@ -51,7 +51,20 @@ type ChunkObjectInfo struct { // ChunkManager is to manager chunks. // Include Read, Write, Remove chunks. type ChunkManager interface { - // RootPath returns current root path. + // Paths passed to ChunkManager are complete keys in the backend namespace, + // and implementations never prepend RootPath themselves: + // - remote keys are bucket-relative object keys that already include the + // storage prefix minio.rootPath (for example, files/insert_log/...); + // they are not URIs and do not start with a slash + // - local keys are complete filesystem paths that already include the + // storage prefix localStorage.path (for example, + // /var/lib/milvus/data/insert_log/...) + // Storage V3 manifests and the loon filesystem use exactly the same keys + // (the loon filesystem is rooted at the bucket, or at "/" for local; see + // storagev2.LoonFSRootPath), so a path never needs conversion between the + // manifest, the ChunkManager and the disk. WalkWithPrefix returns FilePath + // values in the same namespace. + // RootPath returns the storage prefix (minio.rootPath or localStorage.path). RootPath() string // Path returns path of @filePath. Path(ctx context.Context, filePath string) (string, error) diff --git a/internal/storagev2/filesystem_metrics.go b/internal/storagev2/filesystem_metrics.go index 76e4e69daf..ac9ca47d22 100644 --- a/internal/storagev2/filesystem_metrics.go +++ b/internal/storagev2/filesystem_metrics.go @@ -151,7 +151,9 @@ func makePropertiesFromConfig(storageConfig *indexpb.StorageConfig) (C.LoonPrope keys = append(keys, propAccessKeyValue) values = append(values, v) } - if v := storageConfig.GetRootPath(); v != "" { + // Same root rule as packed.MakePropertiesFromStorageConfig, otherwise the + // metrics lookup addresses a filesystem that was never created. + if v := LoonFSRootPath(storageConfig); v != "" { keys = append(keys, propRootPath) values = append(values, v) } diff --git a/internal/storagev2/filesystem_metrics_test.go b/internal/storagev2/filesystem_metrics_test.go index 8e3bcdb088..77cb18d2de 100644 --- a/internal/storagev2/filesystem_metrics_test.go +++ b/internal/storagev2/filesystem_metrics_test.go @@ -60,46 +60,29 @@ func TestGetFilesystemMetricsWithConfig(t *testing.T) { } } +// Every local storage config maps to the single loon filesystem rooted at "/" +// (see LoonFSRootPath), so two local configs with different root paths share +// one metrics entry whose display key names that root. func TestListFilesystemMetrics(t *testing.T) { - metricsBefore, err := ListFilesystemMetrics() - require.NoError(t, err) - existingDisplayKeys := make(map[string]struct{}, len(metricsBefore)) - for _, fsMetrics := range metricsBefore { - existingDisplayKeys[fsMetrics.DisplayKey] = struct{}{} - } - - dirA := t.TempDir() - dirB := t.TempDir() - expectedDisplayPrefixes := map[string]struct{}{ - "file://" + dirA + "#fs:": {}, - "file://" + dirB + "#fs:": {}, - } - for _, dir := range []string{dirA, dirB} { - _, err := GetFilesystemMetricsWithConfig(&indexpb.StorageConfig{ + for _, dir := range []string{t.TempDir(), t.TempDir()} { + fsMetrics, err := GetFilesystemMetricsWithConfig(&indexpb.StorageConfig{ StorageType: "local", RootPath: dir, }) require.NoError(t, err) + require.NotNil(t, fsMetrics) } metricsList, err := ListFilesystemMetrics() require.NoError(t, err) - newFilesystemCount := 0 + localPrefix := "file://" + LoonLocalFSRootPath + "#fs:" + localEntries := 0 for _, fsMetrics := range metricsList { - if _, ok := existingDisplayKeys[fsMetrics.DisplayKey]; ok { + if !strings.HasPrefix(fsMetrics.DisplayKey, localPrefix) { continue } - newFilesystemCount++ - matched := false - for prefix := range expectedDisplayPrefixes { - if strings.HasPrefix(fsMetrics.DisplayKey, prefix) { - delete(expectedDisplayPrefixes, prefix) - matched = true - break - } - } - assert.True(t, matched, fsMetrics.DisplayKey) + localEntries++ assert.GreaterOrEqual(t, fsMetrics.ReadCount, int64(0)) assert.GreaterOrEqual(t, fsMetrics.WriteCount, int64(0)) assert.GreaterOrEqual(t, fsMetrics.ReadBytes, int64(0)) @@ -109,8 +92,7 @@ func TestListFilesystemMetrics(t *testing.T) { assert.GreaterOrEqual(t, fsMetrics.MultiPartUploadCreated, int64(0)) assert.GreaterOrEqual(t, fsMetrics.MultiPartUploadFinished, int64(0)) } - require.Equal(t, 2, newFilesystemCount) - require.Empty(t, expectedDisplayPrefixes) + require.Equal(t, 1, localEntries, "local storage shares one loon filesystem rooted at /") } func TestNilConfig(t *testing.T) { diff --git a/internal/storagev2/loon_root.go b/internal/storagev2/loon_root.go new file mode 100644 index 0000000000..62f2e3dc23 --- /dev/null +++ b/internal/storagev2/loon_root.go @@ -0,0 +1,42 @@ +// Licensed to the LF AI & Data foundation under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storagev2 + +import "github.com/milvus-io/milvus/pkg/v3/proto/indexpb" + +// LoonLocalFSRootPath is the fs.root_path handed to loon for storageType=local. +// +// Milvus keys already carry the storage prefix: localStorage.path for local +// (/var/lib/milvus/data/insert_log/...) and minio.rootPath for remote +// (files/insert_log/...). The loon filesystem therefore has to be rooted at the +// namespace root, which is the bucket for remote (milvus-storage never applies +// root_path there) and "/" for local. Rooting the local SubTreeFileSystem at +// localStorage.path would join the prefix twice (milvus-storage #351, #53051) +// and make complete keys differ from their physical locations. Every Go site +// that turns a StorageConfig into loon properties must go through +// LoonFSRootPath; the C++ counterpart is +// LoonFSRootPath in storage/loon_ffi/util.h. +const LoonLocalFSRootPath = "/" + +// LoonFSRootPath returns the fs.root_path for storageConfig: "/" for local, +// the configured root path otherwise. +func LoonFSRootPath(storageConfig *indexpb.StorageConfig) string { + if storageConfig.GetStorageType() == "local" { + return LoonLocalFSRootPath + } + return storageConfig.GetRootPath() +} diff --git a/internal/storagev2/loon_root_test.go b/internal/storagev2/loon_root_test.go new file mode 100644 index 0000000000..02ed3b6b3f --- /dev/null +++ b/internal/storagev2/loon_root_test.go @@ -0,0 +1,38 @@ +// Licensed to the LF AI & Data foundation under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storagev2 + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/milvus-io/milvus/pkg/v3/proto/indexpb" +) + +// Milvus keys already carry the storage prefix, so the loon filesystem must be +// rooted at the namespace root: "/" for local, and the configured root path +// (which milvus-storage does not apply for object storage) for remote. +func TestLoonFSRootPath(t *testing.T) { + assert.Equal(t, "/", LoonFSRootPath(&indexpb.StorageConfig{StorageType: "local", RootPath: "/var/lib/milvus/data"})) + assert.Equal(t, "/", LoonFSRootPath(&indexpb.StorageConfig{StorageType: "local", RootPath: "data"})) + assert.Equal(t, "/", LoonFSRootPath(&indexpb.StorageConfig{StorageType: "local"})) + assert.Equal(t, "files", LoonFSRootPath(&indexpb.StorageConfig{StorageType: "remote", RootPath: "files"})) + assert.Equal(t, "files", LoonFSRootPath(&indexpb.StorageConfig{StorageType: "minio", RootPath: "files"})) + assert.Equal(t, "", LoonFSRootPath(&indexpb.StorageConfig{StorageType: "remote"})) + assert.Equal(t, "", LoonFSRootPath(nil)) +} diff --git a/internal/storagev2/packed/ffi_common.go b/internal/storagev2/packed/ffi_common.go index 8322bdbde1..a767a1d1e9 100644 --- a/internal/storagev2/packed/ffi_common.go +++ b/internal/storagev2/packed/ffi_common.go @@ -18,6 +18,7 @@ import ( "github.com/cockroachdb/errors" + "github.com/milvus-io/milvus/internal/storagev2" _ "github.com/milvus-io/milvus/internal/util/cgo" "github.com/milvus-io/milvus/pkg/v3/proto/indexpb" "github.com/milvus-io/milvus/pkg/v3/util/merr" @@ -82,10 +83,9 @@ func ExtfsPrefixForCollection(collectionID int64) string { return fmt.Sprintf("extfs.%d.", collectionID) } -// MakePropertiesFromStorageConfig creates a Properties object from StorageConfig -// This function converts a StorageConfig structure into a Properties object by -// calling the FFI properties_create function. All configuration fields from -// StorageConfig are mapped to corresponding key-value pairs in Properties. +// MakePropertiesFromStorageConfig creates a Properties object from StorageConfig. +// All configuration fields are mapped to corresponding property key-value pairs, +// with the local filesystem root normalized separately from the key prefix. func MakePropertiesFromStorageConfig(storageConfig *indexpb.StorageConfig, extraKVs map[string]string) (*C.LoonProperties, error) { if storageConfig == nil { return nil, merr.WrapErrStorageMsg("storageConfig is required") @@ -112,9 +112,9 @@ func MakePropertiesFromStorageConfig(storageConfig *indexpb.StorageConfig, extra keys = append(keys, PropertyFSAccessKeyValue) values = append(values, storageConfig.GetSecretAccessKey()) } - if storageConfig.GetRootPath() != "" { + if fsRoot := storagev2.LoonFSRootPath(storageConfig); fsRoot != "" { keys = append(keys, PropertyFSRootPath) - values = append(values, storageConfig.GetRootPath()) + values = append(values, fsRoot) } if storageConfig.GetStorageType() != "" { keys = append(keys, PropertyFSStorageType) diff --git a/internal/storagev2/packed/ffi_root_path_test.go b/internal/storagev2/packed/ffi_root_path_test.go new file mode 100644 index 0000000000..5405ca5662 --- /dev/null +++ b/internal/storagev2/packed/ffi_root_path_test.go @@ -0,0 +1,47 @@ +// Licensed to the LF AI & Data foundation under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package packed + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/milvus-io/milvus/pkg/v3/proto/indexpb" +) + +func TestMakePropertiesFromStorageConfig_RootPath(t *testing.T) { + for _, tc := range []struct { + storageType string + rootPath string + expected string + }{ + {storageType: "local", rootPath: "/var/lib/milvus/data", expected: "/"}, + {storageType: "remote", rootPath: "files", expected: "files"}, + } { + t.Run(tc.storageType, func(t *testing.T) { + props, err := MakePropertiesFromStorageConfig(&indexpb.StorageConfig{ + StorageType: tc.storageType, + RootPath: tc.rootPath, + }, nil) + require.NoError(t, err) + defer FreeProperties(props) + assert.Equal(t, tc.expected, loonPropertyString(props, PropertyFSRootPath)) + }) + } +} diff --git a/internal/storagev2/packed/manifest_commit_test.go b/internal/storagev2/packed/manifest_commit_test.go index b017def041..a29c9e38cb 100644 --- a/internal/storagev2/packed/manifest_commit_test.go +++ b/internal/storagev2/packed/manifest_commit_test.go @@ -46,7 +46,7 @@ func manifestTestStorageConfig(t *testing.T) *indexpb.StorageConfig { // loon transaction (and therefore without bumping the version). func TestCommitManifestUpdates_EmptyShortCircuit(t *testing.T) { cfg := manifestTestStorageConfig(t) - basePath := "files/commit_empty_test/seg1" + basePath := path.Join(cfg.RootPath, "commit_empty_test/seg1") cases := []struct { name string @@ -71,9 +71,9 @@ func TestCommitManifestUpdates_EmptyShortCircuit(t *testing.T) { // via a single CommitManifestUpdates call. func TestCommitManifestUpdates_StatsOnly(t *testing.T) { cfg := manifestTestStorageConfig(t) - basePath := "files/commit_stats_only/seg1" + basePath := path.Join(cfg.RootPath, "commit_stats_only/seg1") - statPath := path.Join(cfg.RootPath, basePath, "_stats/bloom_filter.100/1") + statPath := path.Join(basePath, "_stats/bloom_filter.100/1") require.NoError(t, WriteFile(cfg, statPath, []byte("bloom-blob"))) got, err := CommitManifestUpdates(basePath, ManifestEarliest, cfg, &ManifestUpdates{ @@ -98,11 +98,11 @@ func TestCommitManifestUpdates_StatsOnly(t *testing.T) { // BM25 memory_size while excluding text/JSON index stats. func TestStatsBinlogSizeFromManifest(t *testing.T) { cfg := manifestTestStorageConfig(t) - basePath := "files/stats_binlog_size/seg1" + basePath := path.Join(cfg.RootPath, "stats_binlog_size/seg1") - bloomPath := path.Join(cfg.RootPath, basePath, "_stats/bloom_filter.100/1") - bm25Path := path.Join(cfg.RootPath, basePath, "_stats/bm25.101/1") - textPath := path.Join(cfg.RootPath, basePath, "_stats/text.102/1") + bloomPath := path.Join(basePath, "_stats/bloom_filter.100/1") + bm25Path := path.Join(basePath, "_stats/bm25.101/1") + textPath := path.Join(basePath, "_stats/text.102/1") require.NoError(t, WriteFile(cfg, bloomPath, []byte("bloom-blob"))) require.NoError(t, WriteFile(cfg, bm25Path, []byte("bm25-blob"))) require.NoError(t, WriteFile(cfg, textPath, []byte("text-blob"))) @@ -132,9 +132,9 @@ func TestStatsBinlogSizeFromManifest_ReadError(t *testing.T) { // TestCommitManifestUpdates_DeltaOnly exercises the delta-only path. func TestCommitManifestUpdates_DeltaOnly(t *testing.T) { cfg := manifestTestStorageConfig(t) - basePath := "files/commit_delta_only/seg1" + basePath := path.Join(cfg.RootPath, "commit_delta_only/seg1") - deltaPath := path.Join(cfg.RootPath, basePath, "_delta/delta-1") + deltaPath := path.Join(basePath, "_delta/delta-1") require.NoError(t, WriteFile(cfg, deltaPath, []byte("delta-payload"))) got, err := CommitManifestUpdates(basePath, ManifestEarliest, cfg, &ManifestUpdates{ @@ -156,7 +156,7 @@ func TestCommitManifestUpdates_DeltaOnly(t *testing.T) { // bundle. func TestCommitManifestUpdates_AllSections(t *testing.T) { cfg := manifestTestStorageConfig(t) - basePath := "files/commit_all_sections/seg1" + basePath := path.Join(cfg.RootPath, "commit_all_sections/seg1") schema := arrow.NewSchema([]arrow.Field{ { @@ -186,9 +186,9 @@ func TestCommitManifestUpdates_AllSections(t *testing.T) { require.NoError(t, err) defer out.Destroy() - statPath := path.Join(cfg.RootPath, basePath, "_stats/bloom_filter.100/1") + statPath := path.Join(basePath, "_stats/bloom_filter.100/1") require.NoError(t, WriteFile(cfg, statPath, []byte("bloom-blob"))) - deltaPath := path.Join(cfg.RootPath, basePath, "_delta/delta-1") + deltaPath := path.Join(basePath, "_delta/delta-1") require.NoError(t, WriteFile(cfg, deltaPath, []byte("delta-payload"))) got, err := CommitManifestUpdates(basePath, ManifestEarliest, cfg, &ManifestUpdates{ @@ -212,7 +212,7 @@ func TestCommitManifestUpdates_AllSections(t *testing.T) { // add_column_group (function-backfill) branch in applyManifestUpdates. func TestCommitManifestUpdates_AddNewColumnGroups(t *testing.T) { cfg := manifestTestStorageConfig(t) - basePath := "files/commit_new_cgs/seg1" + basePath := path.Join(cfg.RootPath, "commit_new_cgs/seg1") schema := arrow.NewSchema([]arrow.Field{ { @@ -254,7 +254,7 @@ func TestCommitManifestUpdates_AddNewColumnGroups(t *testing.T) { func TestCommitManifestUpdates_AddEmptyColumnGroup(t *testing.T) { cfg := manifestTestStorageConfig(t) - basePath := "files/commit_empty_cg/seg1" + basePath := path.Join(cfg.RootPath, "commit_empty_cg/seg1") schema := arrow.NewSchema([]arrow.Field{{ Name: "101", @@ -288,7 +288,7 @@ func TestCommitManifestUpdates_AddEmptyColumnGroup(t *testing.T) { // ColumnGroups handle multiple times (and on a nil receiver) is safe. func TestColumnGroups_DestroyIdempotent(t *testing.T) { cfg := manifestTestStorageConfig(t) - basePath := "files/cg_destroy_test/seg1" + basePath := path.Join(cfg.RootPath, "cg_destroy_test/seg1") schema := arrow.NewSchema([]arrow.Field{ { @@ -330,7 +330,7 @@ func TestColumnGroups_DestroyIdempotent(t *testing.T) { // the close FFI on an exhausted handle. func TestFFIPackedWriter_DoubleCloseRejected(t *testing.T) { cfg := manifestTestStorageConfig(t) - basePath := "files/ffi_double_close/seg1" + basePath := path.Join(cfg.RootPath, "ffi_double_close/seg1") schema := arrow.NewSchema([]arrow.Field{ { diff --git a/internal/storagev2/packed/manifest_index_test.go b/internal/storagev2/packed/manifest_index_test.go index 317fd48a65..27abd313cb 100644 --- a/internal/storagev2/packed/manifest_index_test.go +++ b/internal/storagev2/packed/manifest_index_test.go @@ -52,8 +52,8 @@ func indexEntry(t *testing.T, basePath, indexPrefix string, indexID, buildID int // absolute legacy prefix, with every scalar field intact. func TestCommitManifestUpdates_IndexRoundTrip(t *testing.T) { cfg := manifestTestStorageConfig(t) - basePath := "files/commit_index_roundtrip/seg1" - indexPrefix := "files/index_v1/1/2/3/500/1" + basePath := path.Join(cfg.GetRootPath(), "commit_index_roundtrip/seg1") + indexPrefix := path.Join(cfg.GetRootPath(), "index_v1/1/2/3/500/1") committed, err := CommitManifestUpdates(basePath, ManifestEarliest, cfg, &ManifestUpdates{ Indexes: []ManifestIndexInfo{indexEntry(t, basePath, indexPrefix, 10, 500)}, @@ -90,17 +90,19 @@ func TestCommitManifestUpdates_IndexRoundTrip(t *testing.T) { // DataCoord publish a rebuild without an explicit drop. func TestCommitManifestUpdates_IndexRepublishReplaces(t *testing.T) { cfg := manifestTestStorageConfig(t) - basePath := "files/commit_index_republish/seg1" + basePath := path.Join(cfg.GetRootPath(), "commit_index_republish/seg1") + firstPrefix := path.Join(cfg.GetRootPath(), "index_v1/1/2/3/500/1") + secondPrefix := path.Join(cfg.GetRootPath(), "index_v1/1/2/3/501/2") first, err := CommitManifestUpdates(basePath, ManifestEarliest, cfg, &ManifestUpdates{ - Indexes: []ManifestIndexInfo{indexEntry(t, basePath, "files/index_v1/1/2/3/500/1", 10, 500)}, + Indexes: []ManifestIndexInfo{indexEntry(t, basePath, firstPrefix, 10, 500)}, }) require.NoError(t, err) _, version, err := UnmarshalManifestPath(first) require.NoError(t, err) second, err := CommitManifestUpdates(basePath, version, cfg, &ManifestUpdates{ - Indexes: []ManifestIndexInfo{indexEntry(t, basePath, "files/index_v1/1/2/3/501/2", 10, 501)}, + Indexes: []ManifestIndexInfo{indexEntry(t, basePath, secondPrefix, 10, 501)}, }) require.NoError(t, err) @@ -108,17 +110,17 @@ func TestCommitManifestUpdates_IndexRepublishReplaces(t *testing.T) { require.NoError(t, err) require.Len(t, indexes, 1) require.EqualValues(t, 501, indexes[0].BuildID) - require.Equal(t, "files/index_v1/1/2/3/501/2", indexes[0].Path) + require.Equal(t, secondPrefix, indexes[0].Path) } func TestCommitManifestUpdates_DropIndex(t *testing.T) { cfg := manifestTestStorageConfig(t) - basePath := "files/commit_index_drop/seg1" + basePath := path.Join(cfg.GetRootPath(), "commit_index_drop/seg1") published, err := CommitManifestUpdates(basePath, ManifestEarliest, cfg, &ManifestUpdates{ Indexes: []ManifestIndexInfo{ - indexEntry(t, basePath, "files/index_v1/1/2/3/500/1", 10, 500), - indexEntry(t, basePath, "files/index_v1/1/2/3/600/1", 11, 600), + indexEntry(t, basePath, path.Join(cfg.GetRootPath(), "index_v1/1/2/3/500/1"), 10, 500), + indexEntry(t, basePath, path.Join(cfg.GetRootPath(), "index_v1/1/2/3/600/1"), 11, 600), }, }) require.NoError(t, err) diff --git a/internal/storagev2/packed/packed_reader_ffi_test.go b/internal/storagev2/packed/packed_reader_ffi_test.go index 5cc56abd0f..0be193983b 100644 --- a/internal/storagev2/packed/packed_reader_ffi_test.go +++ b/internal/storagev2/packed/packed_reader_ffi_test.go @@ -4,6 +4,7 @@ import ( "fmt" "io" "math" + "path" "testing" "github.com/apache/arrow/go/v17/arrow" @@ -53,7 +54,7 @@ func TestPackedFFIReader(t *testing.T) { }, }, nil) - basePath := "files/packed_reader_test/1" + basePath := path.Join(dir, "packed_reader_test/1") version := int64(0) // Build record batch @@ -204,7 +205,7 @@ func TestPackedFFIReaderPartialColumns(t *testing.T) { }, }, nil) - basePath := "files/packed_reader_partial_test/1" + basePath := path.Join(dir, "packed_reader_partial_test/1") version := int64(0) // Build record batch @@ -342,7 +343,7 @@ func TestPackedFFIReaderMultipleBatches(t *testing.T) { }, }, nil) - basePath := "files/packed_reader_multi_batch_test/1" + basePath := path.Join(dir, "packed_reader_multi_batch_test/1") version := int64(0) columnGroups := []storagecommon.ColumnGroup{ diff --git a/internal/storagev2/packed/packed_writer_ffi_test.go b/internal/storagev2/packed/packed_writer_ffi_test.go index 72f17ad86b..35e03a5ac6 100644 --- a/internal/storagev2/packed/packed_writer_ffi_test.go +++ b/internal/storagev2/packed/packed_writer_ffi_test.go @@ -2,6 +2,7 @@ package packed import ( "math" + "path" "testing" "github.com/apache/arrow/go/v17/arrow" @@ -112,7 +113,7 @@ func TestGetManifestFieldIDs_InvalidColumnName(t *testing.T) { }, }, nil) columnGroups := []storagecommon.ColumnGroup{{Columns: []int{0}, GroupID: storagecommon.DefaultShortColumnGroupID}} - basePath := "files/packed_writer_invalid_column/1" + basePath := path.Join(CreateStorageConfig().GetRootPath(), "packed_writer_invalid_column/1") cfg := CreateStorageConfig() writer, err := NewFFIPackedWriter(basePath, schema, columnGroups, cfg, nil) require.NoError(t, err) @@ -159,7 +160,7 @@ func TestGetManifestFieldIDs_FromPackedWriterManifest(t *testing.T) { }, }, nil) columnGroups := []storagecommon.ColumnGroup{{Columns: []int{0, 1}, GroupID: storagecommon.DefaultShortColumnGroupID}} - basePath := "files/packed_writer_field_ids/1" + basePath := path.Join(CreateStorageConfig().GetRootPath(), "packed_writer_field_ids/1") cfg := CreateStorageConfig() writer, err := NewFFIPackedWriter(basePath, schema, columnGroups, cfg, nil) require.NoError(t, err) @@ -214,7 +215,7 @@ func TestResolveManifestSingleWriterFormat_FromPackedWriterManifest(t *testing.T }, }, nil) columnGroups := []storagecommon.ColumnGroup{{Columns: []int{0}, GroupID: storagecommon.DefaultShortColumnGroupID, Fields: []int64{100}}} - basePath := "files/packed_writer_format/1" + basePath := path.Join(CreateStorageConfig().GetRootPath(), "packed_writer_format/1") cfg := CreateStorageConfig() writer, err := NewFFIPackedWriter(basePath, schema, columnGroups, cfg, nil) require.NoError(t, err) @@ -250,7 +251,7 @@ func TestResolveManifestSingleWriterFormat_FiltersMixedAddColumnGroups(t *testin pt.Reset(pt.LocalStorageCfg.Path.Key) }) - basePath := "files/packed_writer_mixed_format/1" + basePath := path.Join(CreateStorageConfig().GetRootPath(), "packed_writer_mixed_format/1") cfg := CreateStorageConfig() writeColumn := func(name string, fieldID int64, format string, asNewColumnGroup bool) WriterOutput { schema := arrow.NewSchema([]arrow.Field{ @@ -347,7 +348,7 @@ func TestPackedFFIWriter(t *testing.T) { }, }, nil) - basePath := "files/packed_writer_test/1" + basePath := path.Join(CreateStorageConfig().GetRootPath(), "packed_writer_test/1") version := int64(0) for i := 0; i < batch; i++ { @@ -450,7 +451,7 @@ func TestFFIPackedWriter_CloseThenCommitUpdates(t *testing.T) { {Columns: []int{0}, GroupID: storagecommon.DefaultShortColumnGroupID}, } - basePath := "files/close_commit_test/1" + basePath := path.Join(CreateStorageConfig().GetRootPath(), "close_commit_test/1") cfg := CreateStorageConfig() w, err := NewFFIPackedWriter(basePath, schema, columnGroups, cfg, nil) require.NoError(t, err) diff --git a/internal/storagev2/packed/segment_reader_ffi.go b/internal/storagev2/packed/segment_reader_ffi.go new file mode 100644 index 0000000000..bda3917444 --- /dev/null +++ b/internal/storagev2/packed/segment_reader_ffi.go @@ -0,0 +1,183 @@ +// Licensed to the LF AI & Data foundation under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package packed + +/* +#cgo pkg-config: milvus-storage + +#include +#include "milvus-storage/ffi_c.h" +#include "arrow/c/abi.h" +*/ +import "C" + +import ( + "io" + "unsafe" + + "github.com/apache/arrow/go/v17/arrow" + "github.com/apache/arrow/go/v17/arrow/cdata" + + "github.com/milvus-io/milvus/pkg/v3/proto/indexpb" + "github.com/milvus-io/milvus/pkg/v3/util/merr" +) + +// FFISegmentReader resolves TEXT LOB references through the partition paths in +// textColumns and returns logical UTF8 columns. Compaction uses it when source +// and output manifests belong to different partition namespaces. +type FFISegmentReader struct { + handle C.LoonSegmentReaderHandle + recordReader arrayRecordReader + schema *arrow.Schema +} + +type arrayRecordReader interface { + Read() (arrow.Record, error) +} + +func NewFFISegmentReader( + manifestPath string, + schema *arrow.Schema, + neededColumns []string, + bufferSize int64, + storageConfig *indexpb.StorageConfig, + textColumns []TextColumnConfig, +) (*FFISegmentReader, error) { + if storageConfig == nil { + return nil, merr.WrapErrStorageMsg("storageConfig must not be nil") + } + basePath, version, err := UnmarshalManifestPath(manifestPath) + if err != nil { + return nil, err + } + + var cas cdata.CArrowSchema + cdata.ExportArrowSchema(schema, &cas) + cSchema := (*C.struct_ArrowSchema)(unsafe.Pointer(&cas)) + defer cdata.ReleaseCArrowSchema(&cas) + + cProperties, err := MakePropertiesFromStorageConfig(storageConfig, nil) + if err != nil { + return nil, err + } + defer C.loon_properties_free(cProperties) + + cConfig, cleanupConfig := buildCSegmentReaderConfig(textColumns, bufferSize) + defer cleanupConfig() + cNeededColumns, numColumns, cleanupColumns := buildCSegmentReaderColumns(neededColumns) + defer cleanupColumns() + cBasePath := C.CString(basePath) + defer C.free(unsafe.Pointer(cBasePath)) + + var handle C.LoonSegmentReaderHandle + result := C.loon_segment_reader_open( + cBasePath, + C.int64_t(version), + cSchema, + cNeededColumns, + numColumns, + cConfig, + cProperties, + &handle, + ) + if err := HandleLoonFFIResult(result); err != nil { + return nil, err + } + + var stream cdata.CArrowArrayStream + result = C.loon_segment_reader_get_stream(handle, (*C.struct_ArrowArrayStream)(unsafe.Pointer(&stream))) + if err := HandleLoonFFIResult(result); err != nil { + C.loon_segment_reader_destroy(handle) + return nil, err + } + recordReader, err := cdata.ImportCRecordReader(&stream, schema) + if err != nil { + C.loon_segment_reader_destroy(handle) + return nil, merr.WrapErrStorage(err, "failed to import TEXT-decoding segment reader") + } + + return &FFISegmentReader{handle: handle, recordReader: recordReader, schema: schema}, nil +} + +func buildCSegmentReaderConfig(textColumns []TextColumnConfig, bufferSize int64) (*C.LoonSegmentReaderConfig, func()) { + cConfig := (*C.LoonSegmentReaderConfig)(C.calloc(1, C.sizeof_LoonSegmentReaderConfig)) + cConfig.read_buffer_size = C.int64_t(bufferSize) + cConfig.num_lob_columns = C.size_t(len(textColumns)) + if len(textColumns) == 0 { + cConfig.lob_columns = nil + return cConfig, func() { C.free(unsafe.Pointer(cConfig)) } + } + + cColumns := (*C.LoonLobColumnConfig)(C.calloc(C.size_t(len(textColumns)), C.sizeof_LoonLobColumnConfig)) + columnSlice := unsafe.Slice(cColumns, len(textColumns)) + for i, column := range textColumns { + columnSlice[i].field_id = C.int64_t(column.FieldID) + columnSlice[i].lob_base_path = C.CString(column.LobBasePath) + } + cConfig.lob_columns = cColumns + return cConfig, func() { + for i := range columnSlice { + C.free(unsafe.Pointer(columnSlice[i].lob_base_path)) + } + C.free(unsafe.Pointer(cColumns)) + C.free(unsafe.Pointer(cConfig)) + } +} + +func buildCSegmentReaderColumns(columns []string) (**C.char, C.int64_t, func()) { + cColumns := make([]*C.char, len(columns)) + for i, column := range columns { + cColumns[i] = C.CString(column) + } + cleanup := func() { + for _, column := range cColumns { + C.free(unsafe.Pointer(column)) + } + } + if len(cColumns) == 0 { + return nil, 0, cleanup + } + return (**C.char)(unsafe.Pointer(&cColumns[0])), C.int64_t(len(cColumns)), cleanup +} + +func (r *FFISegmentReader) ReadNext() (arrow.Record, error) { + if r.recordReader == nil { + return nil, io.EOF + } + record, err := r.recordReader.Read() + if err != nil { + if err == io.EOF { + return nil, io.EOF + } + return nil, merr.WrapErrStorage(err, "failed to read TEXT-decoded segment batch") + } + return record, nil +} + +func (r *FFISegmentReader) Close() error { + if r.handle == 0 { + return nil + } + r.recordReader = nil + C.loon_segment_reader_destroy(r.handle) + r.handle = 0 + return nil +} + +func (r *FFISegmentReader) Schema() *arrow.Schema { + return r.schema +} diff --git a/internal/storagev2/packed/transaction_test.go b/internal/storagev2/packed/transaction_test.go index d10b9dc152..fe5a4fa3ff 100644 --- a/internal/storagev2/packed/transaction_test.go +++ b/internal/storagev2/packed/transaction_test.go @@ -537,7 +537,7 @@ func TestCreateMilvusTableManifestFromSegmentManifests_ImportsSourceDeltalogs(t func TestGetDeltaLogsFromManifestWithExtfsResolvesRelativeSourceDeltalogs(t *testing.T) { cfg := manifestTestStorageConfig(t) - sourceBasePath := "files/source/segment/10" + sourceBasePath := path.Join(cfg.GetRootPath(), "source/segment/10") sourceManifest := createBaseManifest(t, sourceBasePath, cfg) sourceDeltaPath := path.Join(sourceBasePath, "_delta/9001") diff --git a/internal/streamingnode/server/wal/vchannel/segment/pack_writer.go b/internal/streamingnode/server/wal/vchannel/segment/pack_writer.go index 76d042f94b..0ea695b585 100644 --- a/internal/streamingnode/server/wal/vchannel/segment/pack_writer.go +++ b/internal/streamingnode/server/wal/vchannel/segment/pack_writer.go @@ -2,7 +2,6 @@ package segment import ( "context" - "path" "github.com/samber/lo" "google.golang.org/protobuf/proto" @@ -18,13 +17,11 @@ import ( "github.com/milvus-io/milvus/internal/storage" "github.com/milvus-io/milvus/internal/storagecommon" "github.com/milvus-io/milvus/internal/storagev2/packed" - "github.com/milvus-io/milvus/pkg/v3/common" "github.com/milvus-io/milvus/pkg/v3/mq/msgstream" "github.com/milvus-io/milvus/pkg/v3/proto/datapb" "github.com/milvus-io/milvus/pkg/v3/proto/indexpb" "github.com/milvus-io/milvus/pkg/v3/proto/streamingpb" "github.com/milvus-io/milvus/pkg/v3/util/merr" - "github.com/milvus-io/milvus/pkg/v3/util/metautil" "github.com/milvus-io/milvus/pkg/v3/util/paramtable" "github.com/milvus-io/milvus/pkg/v3/util/retry" "github.com/milvus-io/milvus/pkg/v3/util/typeutil" @@ -396,7 +393,9 @@ func manifestPathForGrowingPack(meta *streamingpb.SegmentAssignmentMeta) string if meta.GetStorageVersion() != storage.StorageV3 { return "" } - k := metautil.JoinIDPath(meta.GetCollectionId(), meta.GetPartitionId(), meta.GetSegmentId()) - basePath := path.Join(paramtable.Get().MinioCfg.RootPath.GetValue(), common.SegmentInsertLogPath, k) + // Use the primary storage root: localStorage.path for local storage, + // minio.rootPath for remote storage, consistent with other V3 writers. + basePath := storage.SegmentManifestBasePath(packed.CreateStorageConfig().GetRootPath(), + meta.GetCollectionId(), meta.GetPartitionId(), meta.GetSegmentId()) return packed.MarshalManifestPath(basePath, packed.ManifestEarliest) } diff --git a/internal/streamingnode/server/wal/vchannel/segment/pack_writer_test.go b/internal/streamingnode/server/wal/vchannel/segment/pack_writer_test.go index 53ea862d64..2d611c9ca3 100644 --- a/internal/streamingnode/server/wal/vchannel/segment/pack_writer_test.go +++ b/internal/streamingnode/server/wal/vchannel/segment/pack_writer_test.go @@ -2,6 +2,7 @@ package segment import ( "context" + "path" "testing" "github.com/stretchr/testify/assert" @@ -10,6 +11,7 @@ import ( "github.com/milvus-io/milvus-proto/go-api/v3/msgpb" "github.com/milvus-io/milvus-proto/go-api/v3/schemapb" "github.com/milvus-io/milvus/internal/storage" + "github.com/milvus-io/milvus/internal/storagev2/packed" "github.com/milvus-io/milvus/pkg/v3/proto/datapb" "github.com/milvus-io/milvus/pkg/v3/proto/messagespb" "github.com/milvus-io/milvus/pkg/v3/proto/streamingpb" @@ -39,6 +41,75 @@ func TestCurrentSplitForGrowingPackFillsNewSplitFormats(t *testing.T) { } } +func TestManifestPathForGrowingPackUsesPrimaryStorageRoot(t *testing.T) { + params := paramtable.Get() + meta := &streamingpb.SegmentAssignmentMeta{ + CollectionId: 1, + PartitionId: 2, + SegmentId: 3, + StorageVersion: storage.StorageV3, + } + localRoot := t.TempDir() + + for _, testCase := range []struct { + name string + storageType string + minioRoot string + wantRoot string + }{ + {name: "local", storageType: "local", minioRoot: "files", wantRoot: localRoot}, + {name: "remote", storageType: "remote", minioRoot: "files", wantRoot: "files"}, + {name: "remote_bucket_root", storageType: "remote", minioRoot: "/", wantRoot: "."}, + } { + t.Run(testCase.name, func(t *testing.T) { + require.NoError(t, params.Save(params.CommonCfg.StorageType.Key, testCase.storageType)) + // minio.rootPath stays set in both cases: under local storage it + // must not leak into the key. + require.NoError(t, params.Save(params.MinioCfg.RootPath.Key, testCase.minioRoot)) + require.NoError(t, params.Save(params.LocalStorageCfg.Path.Key, localRoot)) + t.Cleanup(func() { + _ = params.Reset(params.CommonCfg.StorageType.Key) + _ = params.Reset(params.MinioCfg.RootPath.Key) + _ = params.Reset(params.LocalStorageCfg.Path.Key) + }) + + base, version, err := packed.UnmarshalManifestPath(manifestPathForGrowingPack(meta)) + require.NoError(t, err) + assert.Equal(t, path.Join(testCase.wantRoot, "insert_log", "1", "2", "3"), base) + assert.Equal(t, packed.ManifestEarliest, version) + }) + } +} + +func TestManifestPathForGrowingPackPreservesPersistedPath(t *testing.T) { + for _, manifestPath := range []string{ + packed.MarshalManifestPath("files/insert_log/1/2/3", 7), + packed.MarshalManifestPath(path.Join(t.TempDir(), "insert_log/1/2/3"), 9), + } { + meta := &streamingpb.SegmentAssignmentMeta{ + CollectionId: 1, + PartitionId: 2, + SegmentId: 3, + StorageVersion: storage.StorageV3, + PersistedStorage: &streamingpb.L1SegmentPersistedStorage{ + ManifestPath: manifestPath, + }, + } + assert.Equal(t, manifestPath, manifestPathForGrowingPack(meta)) + } +} + +func TestManifestPathForGrowingPackSkipsNonV3(t *testing.T) { + for _, storageVersion := range []int64{storage.StorageV1, storage.StorageV2} { + assert.Empty(t, manifestPathForGrowingPack(&streamingpb.SegmentAssignmentMeta{ + CollectionId: 1, + PartitionId: 2, + SegmentId: 3, + StorageVersion: storageVersion, + })) + } +} + func TestCurrentSplitFromPersistedStorageRestoresFormat(t *testing.T) { schema := &schemapb.CollectionSchema{Fields: []*schemapb.FieldSchema{ {FieldID: 100, DataType: schemapb.DataType_Int64, IsPrimaryKey: true}, diff --git a/internal/util/segcore/segment.go b/internal/util/segcore/segment.go index fd3b741f59..c6e6a2e368 100644 --- a/internal/util/segcore/segment.go +++ b/internal/util/segcore/segment.go @@ -21,6 +21,7 @@ import ( "google.golang.org/protobuf/proto" "github.com/milvus-io/milvus-proto/go-api/v3/commonpb" + "github.com/milvus-io/milvus/internal/metastore/kv/binlog" "github.com/milvus-io/milvus/internal/storage" "github.com/milvus-io/milvus/internal/storagev2/packed" "github.com/milvus-io/milvus/internal/util/cgo" @@ -30,7 +31,6 @@ import ( "github.com/milvus-io/milvus/pkg/v3/proto/segcorepb" "github.com/milvus-io/milvus/pkg/v3/util/merr" "github.com/milvus-io/milvus/pkg/v3/util/metautil" - "github.com/milvus-io/milvus/pkg/v3/util/paramtable" "github.com/milvus-io/milvus/pkg/v3/util/tsoutil" ) @@ -485,8 +485,9 @@ func resolveStatsWithBasePaths(src *querypb.SegmentLoadInfo) ( } } - // V2: compute basePaths from rootPath + stats metadata. - rootPath := paramtable.Get().MinioCfg.RootPath.GetValue() + // Legacy non-manifest stats use the primary storage root: an absolute + // localStorage.path for local storage, minio.rootPath for object storage. + rootPath := binlog.GetRootPath() textBasePaths := make(map[int64]string, len(textStats)) for fieldID, stats := range textStats { diff --git a/internal/util/segcore/segment_test.go b/internal/util/segcore/segment_test.go index 9f11723e35..210eab1910 100644 --- a/internal/util/segcore/segment_test.go +++ b/internal/util/segcore/segment_test.go @@ -2,10 +2,12 @@ package segcore_test import ( "context" + "path" "path/filepath" "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "google.golang.org/protobuf/proto" "github.com/milvus-io/milvus-proto/go-api/v3/commonpb" @@ -471,6 +473,58 @@ func TestConvertToSegcoreSegmentLoadInfo_IndexStorePathVersion(t *testing.T) { assert.Equal(t, indexpb.IndexStorePathVersion_INDEX_STORE_PATH_VERSION_COLLECTION_ROOTED, result.GetIndexInfos()[0].GetIndexStorePathVersion()) } +func TestConvertToSegcoreSegmentLoadInfo_LegacyStatsRoot(t *testing.T) { + paramtable.Init() + params := paramtable.Get() + for _, item := range []*paramtable.ParamItem{¶ms.CommonCfg.StorageType, ¶ms.LocalStorageCfg.Path, ¶ms.MinioCfg.RootPath} { + key, value := item.Key, item.GetValue() + t.Cleanup(func() { require.NoError(t, params.Save(key, value)) }) + } + const localRoot = "/var/lib/milvus/data" + require.NoError(t, params.Save(params.LocalStorageCfg.Path.Key, localRoot)) + require.NoError(t, params.Save(params.MinioCfg.RootPath.Key, "files")) + for _, backend := range []struct{ name, root string }{{"local", localRoot}, {"remote", "files"}} { + t.Run(backend.name, func(t *testing.T) { + require.NoError(t, params.Save(params.CommonCfg.StorageType.Key, backend.name)) + for _, version := range []int64{storage.StorageV1, storage.StorageV2} { + for _, fullPaths := range []bool{false, true} { + textBase := backend.root + "/text_log/10/1/2/3/4/106" + jsonBase := backend.root + "/json_stats/3/11/1/2/3/4/107" + textFiles := []string{"index.v3"} + jsonFiles := []string{"meta.json", "shared_key_index/0", "shredding_data/0/0"} + inputText := append([]string(nil), textFiles...) + inputJSON := append([]string(nil), jsonFiles...) + if fullPaths { + for i := range inputText { + inputText[i] = path.Join(textBase, inputText[i]) + } + for i := range inputJSON { + inputJSON[i] = path.Join(jsonBase, inputJSON[i]) + } + } + src := &querypb.SegmentLoadInfo{ + StorageVersion: version, CollectionID: 2, PartitionID: 3, SegmentID: 4, + TextStatsLogs: map[int64]*datapb.TextIndexStats{ + 106: {FieldID: 106, BuildID: 10, Version: 1, Files: inputText}, + }, + JsonKeyStatsLogs: map[int64]*datapb.JsonKeyStats{ + 107: {FieldID: 107, BuildID: 11, Version: 1, JsonKeyStatsDataFormat: 3, Files: inputJSON}, + }, + } + original := proto.Clone(src) + result, err := segcore.ConvertToSegcoreSegmentLoadInfo(src) + require.NoError(t, err, "version=%d fullPaths=%t", version, fullPaths) + assert.Equal(t, textBase, result.TextStatsLogs[106].BasePath) + assert.Equal(t, jsonBase, result.JsonKeyStatsLogs[107].BasePath) + assert.Equal(t, textFiles, result.TextStatsLogs[106].Files) + assert.Equal(t, jsonFiles, result.JsonKeyStatsLogs[107].Files) + assert.True(t, proto.Equal(original, src), "conversion must not rewrite source metadata") + } + } + }) + } +} + func TestConvertToSegcoreSegmentLoadInfo_V3ManifestErrorReturnsError(t *testing.T) { src := &querypb.SegmentLoadInfo{ StorageVersion: storage.StorageV3, diff --git a/pkg/util/metautil/binlog.go b/pkg/util/metautil/binlog.go index 8ae885a2ba..46a709b97a 100644 --- a/pkg/util/metautil/binlog.go +++ b/pkg/util/metautil/binlog.go @@ -196,7 +196,12 @@ func ExtractJSONKeyStatsRelativePath(file string) string { // BuildJSONKeyStatsV3Prefix returns the V3 remote base path for JSON key stats. // Format: {segmentBasePath}/_stats/json_stats.{fieldID} func BuildJSONKeyStatsV3Prefix(segmentBasePath string, fieldID int64) string { - return path.Join(segmentBasePath, "_stats", "json_stats."+strconv.FormatInt(fieldID, 10)) + suffix := "_stats/json_stats." + strconv.FormatInt(fieldID, 10) + if segmentBasePath == "" { + return suffix + } + // The manifest base is a complete key; preserve literal remote components. + return strings.TrimRight(segmentBasePath, "/") + "/" + suffix } // JSONStatsSegmentInfo is the segment metadata needed to derive a JSON stats base path. diff --git a/pkg/util/metautil/binlog_test.go b/pkg/util/metautil/binlog_test.go index f48fcd7cbd..073fd5e4c4 100644 --- a/pkg/util/metautil/binlog_test.go +++ b/pkg/util/metautil/binlog_test.go @@ -26,6 +26,22 @@ import ( "github.com/milvus-io/milvus/pkg/v3/util/typeutil" ) +func TestBuildJSONKeyStatsV3PrefixPreservesBase(t *testing.T) { + for _, base := range []string{"files/insert_log/1/2/3", "/data/insert_log/1/2/3", "files/tmp/../insert_log/1/2/3", "files//insert_log/1/2/3"} { + for _, trailing := range []string{"", "/"} { + got := BuildJSONKeyStatsV3Prefix(base+trailing, 100) + if want := base + "/_stats/json_stats.100"; got != want { + t.Fatalf("base %q: got %q, want %q", base+trailing, got, want) + } + } + } + for base, want := range map[string]string{"": "_stats/json_stats.100", "/": "/_stats/json_stats.100"} { + if got := BuildJSONKeyStatsV3Prefix(base, 100); got != want { + t.Fatalf("base %q: got %q, want %q", base, got, want) + } + } +} + func TestParseInsertLogPath(t *testing.T) { type args struct { path string diff --git a/pkg/util/paramtable/component_param.go b/pkg/util/paramtable/component_param.go index 65a66489cc..0ab063c4c1 100644 --- a/pkg/util/paramtable/component_param.go +++ b/pkg/util/paramtable/component_param.go @@ -166,7 +166,7 @@ func (p *ComponentParam) init(bt *BaseTable) { p.MixCoordCfg.init(bt) p.ProxyCfg.init(bt) p.QueryCoordCfg.init(bt) - p.QueryNodeCfg.init(bt) + p.QueryNodeCfg.init(bt, p.LocalStorageCfg.Path.GetValue()) p.DataCoordCfg.init(bt) p.DataNodeCfg.init(bt) p.StreamingCfg.init(bt) @@ -4488,7 +4488,7 @@ func formatDurationWithMillisecondFallback(v string) string { return v } -func (p *queryNodeConfig) init(base *BaseTable) { +func (p *queryNodeConfig) init(base *BaseTable, localStoragePath string) { p.StrictGroupAcceptanceThreshold = ParamItem{ Key: "queryNode.groupBy.strictGroupAcceptanceThreshold", Version: "2.6.23", DefaultValue: "0.1", Export: true, @@ -5178,7 +5178,6 @@ This defaults to true, indicating that Milvus creates temporary index for growin Doc: "Deprecated: The folder that storing data files for mmap, setting to a path will enable Milvus to load data with mmap", Formatter: func(v string) string { if len(v) == 0 { - localStoragePath := getLocalStoragePath(base) return path.Join(localStoragePath, "mmap") } return v @@ -5451,7 +5450,6 @@ Max read concurrency must greater than or equal to 1, and less than or equal to Formatter: func(v string) string { if len(v) == 0 { // use local storage path to check correct device - localStoragePath := getLocalStoragePath(base) if _, err := os.Stat(localStoragePath); os.IsNotExist(err) { if err := os.MkdirAll(localStoragePath, os.ModePerm); err != nil { mlog.Fatal(context.TODO(), "failed to mkdir", mlog.String("localStoragePath", localStoragePath), mlog.Err(err)) @@ -9687,12 +9685,3 @@ func (params *ComponentParam) Reset(key string) error { func (params *ComponentParam) GetWithDefault(key string, dft string) string { return params.baseTable.GetWithDefault(key, dft) } - -func getLocalStoragePath(base *BaseTable) string { - localStoragePath := base.Get("localStorage.path") - if len(localStoragePath) == 0 { - localStoragePath = defaultLocalStoragePath - mlog.Warn(context.TODO(), "localStorage.path is not set, using default value", mlog.String("localStorage.path", localStoragePath)) - } - return localStoragePath -} diff --git a/pkg/util/paramtable/component_param_test.go b/pkg/util/paramtable/component_param_test.go index af9063aa39..ac1aee5a94 100644 --- a/pkg/util/paramtable/component_param_test.go +++ b/pkg/util/paramtable/component_param_test.go @@ -47,6 +47,40 @@ func TestQueryNodeStrictGroupSettings(t *testing.T) { assert.Equal(t, 100, cfg.StrictGroupProbeCandidates.GetAsInt()) } +func TestComponentParamDerivedLocalStoragePathsUseCanonicalFrozenValue(t *testing.T) { + workingDir := t.TempDir() + t.Chdir(workingDir) + configDir := t.TempDir() + t.Setenv("MILVUSCONF", configDir) + configFile := filepath.Join(configDir, "local-storage.yaml") + root := filepath.Join(t.TempDir(), "canonical") + require.NoError(t, os.WriteFile(configFile, []byte("localStorage:\n path: \" "+root+"/./ \"\n"), 0o600)) + + bt := NewBaseTable(SkipRemote(true), SkipEnv(true), Files([]string{filepath.Base(configFile)}), Interval(10*time.Millisecond)) + t.Cleanup(bt.mgr.Close) + params := &ComponentParam{} + params.Init(bt) + + wantRoot := filepath.Clean(root) + require.Equal(t, wantRoot, params.LocalStorageCfg.Path.GetValue()) + require.Equal(t, filepath.Join(wantRoot, "pprof"), params.ProfileCfg.PprofPath.GetValue()) + require.Equal(t, filepath.Join(wantRoot, "mmap"), params.QueryNodeCfg.MmapDirPath.GetValue()) + initialDiskCapacity := params.QueryNodeCfg.DiskCapacityLimit.GetValue() + + // File/config-center updates replace the raw source value before the + // Forbidden handler rejects them. Derived paths must keep using the same + // canonical startup root as LocalStorageCfg.Path, not that rejected raw value. + require.NoError(t, os.WriteFile(configFile, []byte("localStorage:\n path: relative/data\n"), 0o600)) + require.Eventually(t, func() bool { + return bt.Get("localStorage.path") == "relative/data" + }, time.Second, 10*time.Millisecond) + require.Equal(t, wantRoot, params.LocalStorageCfg.Path.GetValue()) + require.Equal(t, filepath.Join(wantRoot, "pprof"), params.ProfileCfg.PprofPath.GetValue()) + require.Equal(t, filepath.Join(wantRoot, "mmap"), params.QueryNodeCfg.MmapDirPath.GetValue()) + require.Equal(t, initialDiskCapacity, params.QueryNodeCfg.DiskCapacityLimit.GetValue()) + require.NoDirExists(t, filepath.Join(workingDir, "relative")) +} + func shouldPanic(t *testing.T, name string, f func()) { defer func() { recover() }() f() diff --git a/pkg/util/paramtable/param_item.go b/pkg/util/paramtable/param_item.go index dd25892133..8c1639d92a 100644 --- a/pkg/util/paramtable/param_item.go +++ b/pkg/util/paramtable/param_item.go @@ -260,22 +260,23 @@ func (pi *ParamItem) getWithRaw() (result, raw string, err error) { if s := pi.tempValue.Load(); s != nil { return pi.gateValue(*s), *s, nil } - if pi.manager == nil { panic(fmt.Sprintf("manager is nil %s", pi.Key)) } // raw is always the primary key's value, used for CAS comparison. // effectiveRaw is the value actually used for computing result (may come from fallback). - _, raw, err = pi.manager.GetConfig(pi.Key) + source, raw, err := pi.manager.GetConfig(pi.Key) effectiveRaw := raw + effectiveSource := source if err != nil || raw == pi.DefaultValue { // try fallback if the entry is not exist or default value, // because default value may already defined in milvus.yaml // and we don't want the fallback keys be overridden. for _, key := range pi.FallbackKeys { - var fallbackRaw string - _, fallbackRaw, err = pi.manager.GetConfig(key) + fallbackSource, fallbackRaw, fallbackErr := pi.manager.GetConfig(key) + err = fallbackErr if err == nil { + effectiveSource = fallbackSource effectiveRaw = fallbackRaw break } @@ -286,6 +287,15 @@ func (pi *ParamItem) getWithRaw() (result, raw string, err error) { effectiveRaw = pi.DefaultValue raw = pi.DefaultValue } + // Config sources replace their value before Manager.OnEvent can reject the + // change. Keep a forbidden item's formatted startup value when that happens. + // RuntimeSource remains an explicit process-local override used by tests and + // operational tooling. + if pi.Forbidden && effectiveSource != config.RuntimeSource { + if initial := pi.lastValue.Load(); initial != nil { + return *initial, raw, nil + } + } result = pi.gateValue(effectiveRaw) if pi.Formatter != nil { result = pi.Formatter(result) diff --git a/pkg/util/paramtable/param_item_test.go b/pkg/util/paramtable/param_item_test.go index cd1d9e3cf7..abfde5a6a7 100644 --- a/pkg/util/paramtable/param_item_test.go +++ b/pkg/util/paramtable/param_item_test.go @@ -85,6 +85,22 @@ func TestParamGroupDeleteDoesNotSurfaceTombstone(t *testing.T) { assert.Empty(t, group.GetValue()) } +func TestForbiddenParamItemAllowsRuntimeOverride(t *testing.T) { + manager := config.NewManager() + manager.SetConfig("test.static.path", "initial") + + param := &ParamItem{ + Key: "test.static.path", + Forbidden: true, + Formatter: strings.ToUpper, + } + param.Init(manager) + require.Equal(t, "INITIAL", param.GetValue()) + + manager.SetConfig("test.static.path", "runtime") + require.Equal(t, "RUNTIME", param.GetValue()) +} + func TestGetWithRaw_FallbackKeyCacheSuccess(t *testing.T) { // When primary key equals DefaultValue and a fallback key has a different value, // getWithRaw should return the fallback value as result but the primary key's diff --git a/pkg/util/paramtable/service_param.go b/pkg/util/paramtable/service_param.go index f2821aace1..5db92b5c8b 100644 --- a/pkg/util/paramtable/service_param.go +++ b/pkg/util/paramtable/service_param.go @@ -23,6 +23,7 @@ import ( "net/url" "os" "path" + "path/filepath" "strconv" "strings" "time" @@ -33,6 +34,7 @@ import ( "github.com/milvus-io/milvus/pkg/v3/mlog" "github.com/milvus-io/milvus/pkg/v3/util" "github.com/milvus-io/milvus/pkg/v3/util/etcd" + "github.com/milvus-io/milvus/pkg/v3/util/merr" "github.com/milvus-io/milvus/pkg/v3/util/metricsinfo" ) @@ -116,7 +118,7 @@ func (p *ServiceParam) init(bt *BaseTable) { p.KafkaCfg.Init(bt) p.RocksmqCfg.Init(bt) p.MinioCfg.Init(bt) - p.ProfileCfg.Init(bt) + p.ProfileCfg.init(bt, p.LocalStorageCfg.Path.GetValue()) } func (p *ServiceParam) RocksmqEnable() bool { @@ -611,13 +613,29 @@ func (p *LocalStorageConfig) Init(base *BaseTable) { Version: "2.0.0", DefaultValue: defaultLocalStoragePath, Doc: `Local path to where vector data are stored during a search or a query to avoid repetitve access to MinIO or S3 service. +Must be an absolute filesystem path. Milvus refuses to start if this value is relative or empty. +Migration of data written using a relative path in older versions is not supported. Caution: Changing this parameter after using Milvus for a period of time will affect your access to old data. It is recommended to change this parameter before starting Milvus for the first time.`, - Export: true, + // Every local storage key is a complete filesystem path that starts + // with this value, and the loon local filesystem is rooted at "/", so + // a relative path here would silently depend on the process working + // directory. Reject it during configuration initialization. + Formatter: formatLocalStoragePath, + Forbidden: true, + Export: true, } p.Path.Init(base.mgr) } +func formatLocalStoragePath(value string) string { + value = strings.TrimSpace(value) + if !filepath.IsAbs(value) { + panic(merr.WrapErrParameterInvalidMsg("localStorage.path must be an absolute filesystem path, got %q", value)) + } + return filepath.Clean(value) +} + type MetaStoreConfig struct { MetaStoreType ParamItem `refreshable:"false"` PaginationSize ParamItem `refreshable:"true"` @@ -2088,6 +2106,10 @@ type ProfileConfig struct { } func (p *ProfileConfig) Init(base *BaseTable) { + p.init(base, formatLocalStoragePath(base.GetWithDefault("localStorage.path", defaultLocalStoragePath))) +} + +func (p *ProfileConfig) init(base *BaseTable, localStoragePath string) { p.PprofPath = ParamItem{ Key: "profile.pprof.path", Version: "2.5.5", @@ -2095,7 +2117,6 @@ func (p *ProfileConfig) Init(base *BaseTable) { Doc: "The folder that storing pprof files, by default will use localStoragePath/pprof", Formatter: func(v string) string { if len(v) == 0 { - localStoragePath := getLocalStoragePath(base) return path.Join(localStoragePath, "pprof") } return v diff --git a/pkg/util/paramtable/service_param_test.go b/pkg/util/paramtable/service_param_test.go index 9b7ec84c00..653344bef2 100644 --- a/pkg/util/paramtable/service_param_test.go +++ b/pkg/util/paramtable/service_param_test.go @@ -17,18 +17,119 @@ package paramtable import ( + "os" + "path/filepath" "strconv" "testing" "time" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/milvus-io/milvus/pkg/v3/config" "github.com/milvus-io/milvus/pkg/v3/util" + "github.com/milvus-io/milvus/pkg/v3/util/merr" "github.com/milvus-io/milvus/pkg/v3/util/metricsinfo" "github.com/milvus-io/milvus/pkg/v3/util/typeutil" ) +// Every local storage key is a complete filesystem path that starts with +// localStorage.path, and the loon local filesystem is rooted at "/", so the +// configured value must never depend on the process working directory. +func TestLocalStorageConfig_PathIsAbsolute(t *testing.T) { + for _, tc := range []struct { + name string + configured string + want string + }{ + {name: "default", want: filepath.Clean(defaultLocalStoragePath)}, + {name: "absolute", configured: "/var/lib/milvus/data", want: "/var/lib/milvus/data"}, + {name: "clean_absolute", configured: " /var/lib/milvus/./old/../data/ ", want: "/var/lib/milvus/data"}, + {name: "filesystem_root", configured: "/", want: "/"}, + } { + t.Run(tc.name, func(t *testing.T) { + bt := NewBaseTable(SkipRemote(true), SkipEnv(true), Files([]string{})) + t.Cleanup(bt.mgr.Close) + if tc.configured != "" { + require.NoError(t, bt.Save("localStorage.path", tc.configured)) + } + var params ServiceParam + require.NotPanics(t, func() { params.init(bt) }) + require.Equal(t, tc.want, params.LocalStorageCfg.Path.GetValue()) + require.True(t, filepath.IsAbs(params.LocalStorageCfg.Path.GetValue())) + }) + } +} + +func TestLocalStorageConfig_RejectsInvalidPathDuringInit(t *testing.T) { + assertInvalidInit := func(t *testing.T, bt *BaseTable) { + t.Helper() + t.Cleanup(bt.mgr.Close) + var panicValue any + func() { + defer func() { panicValue = recover() }() + var params ServiceParam + params.init(bt) + }() + require.NotNil(t, panicValue, "service initialization must reject the configured path before starting components") + panicErr, ok := panicValue.(error) + require.True(t, ok, "initialization must panic with a typed configuration error, got %T", panicValue) + require.ErrorIs(t, panicErr, merr.ErrParameterInvalid) + require.Contains(t, panicErr.Error(), "localStorage.path must be an absolute filesystem path") + } + for _, tc := range []struct { + name string + value string + }{ + {name: "relative", value: "relative/data"}, + {name: "current_directory_relative", value: "./data"}, + {name: "parent_directory_relative", value: "../data"}, + {name: "current_directory", value: "."}, + {name: "empty", value: ""}, + {name: "blank", value: " \t\n "}, + } { + t.Run("saved/"+tc.name, func(t *testing.T) { + bt := NewBaseTable(SkipRemote(true), SkipEnv(true), Files([]string{})) + // Save before initialization exercises the startup path, rather + // than calling the formatter through a later GetValue. + require.NoError(t, bt.Save("localStorage.path", tc.value)) + assertInvalidInit(t, bt) + }) + t.Run("environment/"+tc.name, func(t *testing.T) { + t.Setenv("LOCALSTORAGE_PATH", tc.value) + bt := NewBaseTable(SkipRemote(true), Files([]string{})) + assertInvalidInit(t, bt) + }) + } +} + +func TestLocalStorageConfig_PathDoesNotChangeAtRuntime(t *testing.T) { + dir := t.TempDir() + t.Setenv("MILVUSCONF", dir) + configFile := filepath.Join(dir, "local-storage.yaml") + require.NoError(t, os.WriteFile(configFile, []byte("localStorage:\n path: /var/lib/milvus/data\n"), 0o600)) + bt := NewBaseTable(SkipRemote(true), SkipEnv(true), Files([]string{filepath.Base(configFile)}), Interval(10*time.Millisecond)) + t.Cleanup(bt.mgr.Close) + + var params ServiceParam + require.NotPanics(t, func() { params.init(bt) }) + require.Equal(t, "/var/lib/milvus/data", params.LocalStorageCfg.Path.GetValue()) + + // File and config-center sources replace their stored value before emitting + // an update event. A forbidden path must retain its startup value even then. + require.NoError(t, os.WriteFile(configFile, []byte("localStorage:\n path: relative/data\n"), 0o600)) + require.Eventually(t, func() bool { + return bt.Get("localStorage.path") == "relative/data" + }, time.Second, 10*time.Millisecond) + require.NotPanics(t, func() { + require.Equal(t, "/var/lib/milvus/data", params.LocalStorageCfg.Path.GetValue()) + }) + value, raw, err := params.LocalStorageCfg.Path.getWithRaw() + require.NoError(t, err) + require.Equal(t, "/var/lib/milvus/data", value) + require.Equal(t, "relative/data", raw) +} + func TestServiceParam(t *testing.T) { var SParams ServiceParam bt := NewBaseTable(SkipRemote(true)) diff --git a/scripts/segcore_error_boundary_baseline.txt b/scripts/segcore_error_boundary_baseline.txt index 67a5239643..99af3023d4 100644 --- a/scripts/segcore_error_boundary_baseline.txt +++ b/scripts/segcore_error_boundary_baseline.txt @@ -167,6 +167,7 @@ milvus_storage internal/core/src/storage/ArrowFileSystemChunkManager.h milvus_storage internal/core/src/storage/FileManager.h milvus_storage internal/core/src/storage/IndexEntryEncryptedLocalWriter.cpp milvus_storage internal/core/src/storage/IndexEntryEncryptedLocalWriter.h +milvus_storage internal/core/src/storage/minio/MinioChunkManager.cpp milvus_storage internal/core/src/storage/loon_ffi/ffi_reader_c.cpp milvus_storage internal/core/src/storage/loon_ffi/loon_error_code.h milvus_storage internal/core/src/storage/loon_ffi/milvus_table_c.cpp