diff --git a/CLAUDE.md b/CLAUDE.md index 8536ac7a93..ffc4fd90ae 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -34,6 +34,7 @@ NEVER answer based on documentation alone or code alone. NEVER skip Step 2 — t - [**Observability**](docs/agent_guides/observability/README.md): Logging, metrics, tracing, and observability debug workflows. - [**Streaming System**](docs/agent_guides/streaming-system/streaming-system.md): Write path, WAL, DDL/DCL execution, replication && CDC. +- [**Storage Paths**](docs/agent_guides/storage/path_contract.md): Read before changing storage paths, manifests, local migration, or GC. ## Testing diff --git a/cmd/roles/local_layout.go b/cmd/roles/local_layout.go new file mode 100644 index 0000000000..286b661cf7 --- /dev/null +++ b/cmd/roles/local_layout.go @@ -0,0 +1,143 @@ +// Licensed to the LF AI & Data foundation under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package roles + +import ( + "context" + "time" + + "github.com/milvus-io/milvus/internal/storage/localmigrate" + "github.com/milvus-io/milvus/pkg/v3/mlog" + "github.com/milvus-io/milvus/pkg/v3/util/paramtable" +) + +// maxLoggedConflicts bounds the conflict sample in the failure log. The full set +// stays in the returned Report; a startup log must not grow to the size of the +// storage root. +const maxLoggedConflicts = 100 + +// migrateLocalStorageLayoutOrDie moves the double-joined 3.0 local layout and +// recognized 2.6 working-directory index directories before any component opens +// data. It is enabled automatically for standalone with common.storageType=local. +// Directories are merged without overwriting files; cross-filesystem moves copy. +func (mr *MilvusRoles) migrateLocalStorageLayoutOrDie(ctx context.Context, params *paramtable.ComponentParam) bool { + if !mr.Local || params.CommonCfg.StorageType.GetValue() != "local" { + return true + } + migrationCtx, cancel := mr.localStorageMigrationContext(ctx) + defer cancel() + + report, err := mr.migrateLocalStorageLayout(migrationCtx, params) + if err != nil { + if migrationCtx.Err() != nil { + mlog.Info(ctx, "local storage layout migration canceled before component startup", + mlog.String("root", params.LocalStorageCfg.Path.GetValue()), + mlog.Err(err)) + return false + } + mlog.Fatal(ctx, "local storage layout migration failed, refusing to start", + mlog.String("root", params.LocalStorageCfg.Path.GetValue()), + mlog.String("report", report.String()), + mlog.Int("conflictCount", len(report.Conflicts)), + mlog.Strings("conflictSample", firstStrings(report.Conflicts, maxLoggedConflicts)), + mlog.Err(err)) + return false + } + // A shutdown can race with the final migration operation. Do not continue + // into etcd/component startup after the signal handler has requested exit. + if migrationCtx.Err() != nil { + mlog.Info(ctx, "local storage layout migration completed during shutdown; skipping component startup", + mlog.String("root", params.LocalStorageCfg.Path.GetValue())) + return false + } + // Always report the outcome. "Checked the root and found nothing to move" + // and "never ran" must not look identical in a startup log. + mlog.Info(ctx, "local storage layout migration finished", + mlog.String("report", report.String())) + return true +} + +func (mr *MilvusRoles) localStorageMigrationContext(parent context.Context) (context.Context, context.CancelFunc) { + ctx, cancel := context.WithCancel(parent) + if mr.closed == nil { + return ctx, cancel + } + go func() { + select { + case <-mr.closed: + cancel() + case <-ctx.Done(): + } + }() + return ctx, cancel +} + +func (mr *MilvusRoles) migrateLocalStorageLayout(ctx context.Context, params *paramtable.ComponentParam) (*localmigrate.Report, error) { + if !mr.Local || params.CommonCfg.StorageType.GetValue() != "local" { + return nil, nil + } + root := params.LocalStorageCfg.Path.GetValue() + + logger := mlog.With(mlog.String("root", root)) + logger.Info(ctx, "checking local storage layout for legacy directories to migrate") + + planned, started, completed := 0, 0, 0 + return localmigrate.Migrate(ctx, root, localmigrate.Options{ + LegacyPrefix: params.MinioCfg.RootPath.GetValue(), + OnPlan: func(dirs []localmigrate.Dir) { + planned = len(dirs) + files, bytes := 0, int64(0) + for _, dir := range dirs { + files += dir.Files + bytes += dir.Bytes + } + logger.Info(ctx, "local storage layout migration plan decided", + mlog.Int("directories", planned), + mlog.Int("files", files), + mlog.Int64("bytes", bytes)) + // Report the full plan before moving any directory. + for index, dir := range dirs { + logger.Info(ctx, "local storage layout migration will move directory", + mlog.Int("index", index+1), mlog.Int("total", planned), + mlog.String("source", dir.Source), mlog.String("target", dir.Target), + mlog.Int("files", dir.Files), mlog.Int64("bytes", dir.Bytes)) + } + }, + OnDirStart: func(dir localmigrate.Dir) { + started++ + logger.Info(ctx, "moving local storage layout directory", + mlog.Int("index", started), mlog.Int("total", planned), + mlog.String("source", dir.Source), mlog.String("target", dir.Target), + mlog.Int("files", dir.Files), mlog.Int64("bytes", dir.Bytes)) + }, + OnDirDone: func(dir localmigrate.Dir, elapsed time.Duration) { + completed++ + logger.Info(ctx, "moved local storage layout directory", + mlog.Int("index", completed), mlog.Int("total", planned), + mlog.String("source", dir.Source), mlog.String("target", dir.Target), + mlog.Int("files", dir.Files), mlog.Int64("bytes", dir.Bytes), + mlog.Duration("elapsed", elapsed)) + }, + }) +} + +func firstStrings(values []string, limit int) []string { + if len(values) <= limit { + return values + } + return values[:limit] +} diff --git a/cmd/roles/local_layout_test.go b/cmd/roles/local_layout_test.go new file mode 100644 index 0000000000..4e3870edd0 --- /dev/null +++ b/cmd/roles/local_layout_test.go @@ -0,0 +1,224 @@ +// Licensed to the LF AI & Data foundation under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package roles + +import ( + "context" + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/milvus-io/milvus/internal/storage/localmigrate" + "github.com/milvus-io/milvus/pkg/v3/util/merr" + "github.com/milvus-io/milvus/pkg/v3/util/paramtable" +) + +func TestLocalStorageMigrationContextCancelsOnShutdown(t *testing.T) { + mr := NewMilvusRoles() + ctx, cancel := mr.localStorageMigrationContext(context.Background()) + defer cancel() + + close(mr.closed) + select { + case <-ctx.Done(): + require.ErrorIs(t, ctx.Err(), context.Canceled) + case <-time.After(time.Second): + t.Fatal("migration context was not canceled after shutdown") + } +} + +func localLayoutTestParams(t *testing.T, storageType, root string) *paramtable.ComponentParam { + t.Helper() + base := paramtable.NewBaseTable(paramtable.SkipRemote(true), paramtable.SkipEnv(true)) + require.NoError(t, base.Save("common.storageType", storageType)) + require.NoError(t, base.Save("localStorage.path", root)) + params := ¶mtable.ComponentParam{} + params.Init(base) + return params +} + +func writeLocalLayoutTestFile(t *testing.T, root, key, contents string) { + t.Helper() + filename := filepath.Join(root, key) + require.NoError(t, os.MkdirAll(filepath.Dir(filename), 0o755)) + require.NoError(t, os.WriteFile(filename, []byte(contents), 0o600)) +} + +func TestMigrateLocalStorageLayoutAutomaticSources(t *testing.T) { + root, cwd := t.TempDir(), t.TempDir() + params := localLayoutTestParams(t, "local", root) + mr := &MilvusRoles{Local: true} + t.Chdir(cwd) + keys := map[string]string{ + cwd: "text_log/7/0/1/2/3/100/milvus_packed_text_index.v3", + localmigrate.DisplacedRoots(root)[0]: "insert_log/1/2/3/_data/a.parquet", + } + for sourceRoot, key := range keys { + writeLocalLayoutTestFile(t, sourceRoot, key, key) + } + report, err := mr.migrateLocalStorageLayout(t.Context(), params) + require.NoError(t, err) + require.Equal(t, 2, report.Renamed) + for sourceRoot, key := range keys { + require.NoFileExists(t, filepath.Join(sourceRoot, key)) + contents, err := os.ReadFile(filepath.Join(root, key)) + require.NoError(t, err) + require.Equal(t, key, string(contents)) + } + // A new startup finds no old directories. + mr = &MilvusRoles{Local: true} + params = localLayoutTestParams(t, "local", root) + report, err = mr.migrateLocalStorageLayout(t.Context(), params) + require.NoError(t, err) + require.Zero(t, report.Renamed) + require.Zero(t, report.Bytes) + // Simulate normal GC. A restart does not resurrect deleted canonical data. + for _, key := range keys { + require.NoError(t, os.Remove(filepath.Join(root, key))) + } + report, err = mr.migrateLocalStorageLayout(t.Context(), params) + require.NoError(t, err) + require.Zero(t, report.Renamed) + for _, key := range keys { + require.NoFileExists(t, filepath.Join(root, key)) + } +} + +func TestMigrateLocalStorageLayoutDisabled(t *testing.T) { + for _, test := range []struct { + name string + standalone bool + storageType string + }{ + {name: "remote standalone", standalone: true, storageType: "remote"}, + {name: "local distributed", storageType: "local"}, + {name: "remote distributed", storageType: "remote"}, + } { + t.Run(test.name, func(t *testing.T) { + root, cwd := t.TempDir(), t.TempDir() + params := localLayoutTestParams(t, test.storageType, root) + mr := &MilvusRoles{Local: test.standalone} + t.Chdir(cwd) + key := "text_log/7/0/1/2/3/100/milvus_packed_text_index.v3" + writeLocalLayoutTestFile(t, cwd, key, "old-index") + writeLocalLayoutTestFile(t, localmigrate.DisplacedRoots(root)[0], key, "displaced-index") + report, err := mr.migrateLocalStorageLayout(t.Context(), params) + require.NoError(t, err) + require.Nil(t, report) + require.NoFileExists(t, filepath.Join(root, key)) + require.NoFileExists(t, filepath.Join(root, ".milvus-local-layout.lock")) + require.NoDirExists(t, filepath.Join(root, ".milvus-local-layout")) + require.FileExists(t, filepath.Join(cwd, key)) + require.FileExists(t, filepath.Join(localmigrate.DisplacedRoots(root)[0], key)) + }) + } +} + +func TestMigrateLocalStorageLayoutConflictStopsStartup(t *testing.T) { + root, cwd := t.TempDir(), t.TempDir() + params := localLayoutTestParams(t, "local", root) + mr := &MilvusRoles{Local: true} + t.Chdir(cwd) + key := "index_files/7/0/2/3/milvus_packed_inverted_index.v3" + writeLocalLayoutTestFile(t, cwd, key, "old-index") + writeLocalLayoutTestFile(t, root, key, "new-index") + report, err := mr.migrateLocalStorageLayout(t.Context(), params) + require.ErrorIs(t, err, merr.ErrDataIntegrity) + require.Equal(t, []string{filepath.Join(root, key)}, report.Conflicts) + contents, err := os.ReadFile(filepath.Join(root, key)) + require.NoError(t, err) + require.Equal(t, "new-index", string(contents)) +} + +func TestMigrateLocalStorageLayoutWithoutLegacyFiles(t *testing.T) { + for _, layout := range []string{"existing-root", "missing-root"} { + t.Run(layout, func(t *testing.T) { + root, cwd := t.TempDir(), t.TempDir() + if layout == "missing-root" { + root = filepath.Join(root, "new-install") + } + params := localLayoutTestParams(t, "local", root) + if layout == "missing-root" { + // Configuration creates this empty directory when reading disk + // capacity. Remove it so migration really receives a missing root. + require.NoError(t, os.Remove(root)) + require.NoDirExists(t, root) + } + mr := &MilvusRoles{Local: true} + t.Chdir(cwd) + _, err := mr.migrateLocalStorageLayout(t.Context(), params) + require.NoError(t, err) + require.NoDirExists(t, filepath.Join(root, ".milvus-local-layout")) + if layout == "missing-root" { + require.NoDirExists(t, root, "migration must not create a missing storage root") + require.NoFileExists(t, filepath.Join(root, ".milvus-local-layout.lock")) + } + }) + } +} + +func TestMigrateLocalStorageLayoutRetriesAfterCopyFailure(t *testing.T) { + root, cwd := t.TempDir(), t.TempDir() + t.Chdir(cwd) + key := "index_files/7/0/2/3/milvus_packed_inverted_index.v3" + writeLocalLayoutTestFile(t, cwd, key, "old-index") + // A regular file in place of the destination directory makes rename fail. + blocker := filepath.Join(root, "index_files") + require.NoError(t, os.WriteFile(blocker, []byte("not-a-directory"), 0o600)) + mr := &MilvusRoles{Local: true} + _, err := mr.migrateLocalStorageLayout(t.Context(), localLayoutTestParams(t, "local", root)) + require.Error(t, err) + source, err := os.ReadFile(filepath.Join(cwd, key)) + require.NoError(t, err) + require.Equal(t, "old-index", string(source)) + + // Once the filesystem problem is resolved, a fresh standalone startup + // recovers directly from the same visible source without a shell handoff. + require.NoError(t, os.Remove(blocker)) + mr = &MilvusRoles{Local: true} + report, err := mr.migrateLocalStorageLayout(t.Context(), localLayoutTestParams(t, "local", root)) + require.NoError(t, err) + require.Equal(t, 1, report.Renamed) + require.Equal(t, 1, report.Renamed) + require.NoFileExists(t, filepath.Join(cwd, key)) + contents, err := os.ReadFile(filepath.Join(root, key)) + require.NoError(t, err) + require.Equal(t, "old-index", string(contents)) +} + +func TestMigrateLocalStorageLayoutProtectsLegacyManifestNamespace(t *testing.T) { + root, cwd := t.TempDir(), t.TempDir() + t.Chdir(cwd) + params := localLayoutTestParams(t, "local", root) + // This configured legacy prefix resolves to the same physical namespace as + // the double-root source. A relative manifest still reads there after reload. + relativeRoot, err := filepath.Rel(string(filepath.Separator), root) + require.NoError(t, err) + require.NoError(t, params.Save("minio.rootPath", filepath.ToSlash(relativeRoot))) + key := "insert_log/1/2/3/_data/a.parquet" + sourceRoot := localmigrate.DisplacedRoots(root)[0] + writeLocalLayoutTestFile(t, sourceRoot, key, "still-referenced") + mr := &MilvusRoles{Local: true} + _, err = mr.migrateLocalStorageLayout(t.Context(), params) + require.Error(t, err) + require.FileExists(t, filepath.Join(sourceRoot, key)) + require.NoFileExists(t, filepath.Join(root, key)) + require.NoDirExists(t, filepath.Join(root, ".milvus-local-layout", "backups")) +} diff --git a/cmd/roles/roles.go b/cmd/roles/roles.go index 4a84c2bb2c..6a4264bb23 100644 --- a/cmd/roles/roles.go +++ b/cmd/roles/roles.go @@ -435,6 +435,10 @@ func (mr *MilvusRoles) Run() { } params := paramtable.Get() + // Recover visible legacy local files before etcd and components start. + if !mr.migrateLocalStorageLayoutOrDie(ctx, params) { + return + } if params.EtcdCfg.UseEmbedEtcd.GetAsBool() { // Start etcd server. if err := etcd.InitEtcdServer( diff --git a/cmd/roles/roles_test.go b/cmd/roles/roles_test.go index 6db6477b1b..152e9df1f6 100644 --- a/cmd/roles/roles_test.go +++ b/cmd/roles/roles_test.go @@ -77,7 +77,9 @@ func TestFilesystemMetricsRegisteredWithRolesRegistry(t *testing.T) { "milvus_storage_filesystem_multi_part_upload_created": {}, "milvus_storage_filesystem_multi_part_upload_finished": {}, } - expectedDisplayKeyPrefix := "file://" + dir + "#fs:" + // Local storage shares one loon filesystem rooted at "/", whatever the + // configured localStorage.path is (see storagev2.LoonFSRootPath). + expectedDisplayKeyPrefix := "file://" + storagev2.LoonLocalFSRootPath + "#fs:" for _, family := range gathered { if _, ok := missingFamilies[family.GetName()]; !ok { continue diff --git a/configs/milvus.yaml b/configs/milvus.yaml index 55fd35f9e1..e3b486ec72 100644 --- a/configs/milvus.yaml +++ b/configs/milvus.yaml @@ -87,6 +87,8 @@ tikv: localStorage: # Local path to where vector data are stored during a search or a query to avoid repetitve access to MinIO or S3 service. + # Must be an absolute filesystem path. Milvus refuses to start if this value is relative or empty. + # Migration of data written using a relative path in older versions is not supported. # Caution: Changing this parameter after using Milvus for a period of time will affect your access to old data. # It is recommended to change this parameter before starting Milvus for the first time. path: /var/lib/milvus/data/ diff --git a/docs/agent_guides/README.md b/docs/agent_guides/README.md index ccea123700..b6b0ce4a6f 100644 --- a/docs/agent_guides/README.md +++ b/docs/agent_guides/README.md @@ -8,6 +8,8 @@ changing that subsystem. observability debugging. - [Streaming system](streaming-system/streaming-system.md): WAL, channels, DDL/DCL execution, recovery storage, and replication. +- [Storage path contract](storage/path_contract.md): complete keys, local upgrade + compatibility, and GC ownership boundaries. Update these guides with the code when an invariant, interface, or ownership boundary changes. diff --git a/docs/agent_guides/storage/path_contract.md b/docs/agent_guides/storage/path_contract.md new file mode 100644 index 0000000000..13bdb7d623 --- /dev/null +++ b/docs/agent_guides/storage/path_contract.md @@ -0,0 +1,113 @@ +# Storage path contract + +Read this before changing storage paths, manifests, local migration, or GC. +Update it with the code when a path or ownership rule changes. + +## Complete keys + +ChunkManager receives complete keys; it does not prepend its root. + +| Backend | Key | Example | +| --- | --- | --- | +| Local | Absolute filesystem path including `localStorage.path` | `/var/lib/milvus/data/insert_log/1/2/3/...` | +| Remote | Bucket-relative object key, normally including `minio.rootPath` | `files/insert_log/1/2/3/...` | + +**New local segment/index namespaces never use minio.rootPath.** Updates to an +existing legacy manifest retain its base path, including the historical prefix; +do not rebase that manifest while appending data or stats. A remote key does not include +the bucket name or a URI scheme. Local loon/Arrow filesystems use `/` as their +filesystem root so a complete key is not joined to the storage root twice. + +Explore is a separate layout: local temporary manifests live under +`localStorage.path/__explore_temp__/`; remote manifests live under +`__explore_temp__/` at the bucket root, without `minio.rootPath`. + +## Path ownership + +- Writers use shared layout helpers: `storage.SegmentManifestBasePath`, + `SegmentPartitionBasePath`, and the binlog/index helpers in `metautil`. +- Readers consume the complete key recorded in metadata or the manifest. + They must not prepend another root. +- Some API fields describe relative file names rather than complete keys. + TEXT stats results contain complete keys; JSON stats results contain names + relative to their field's stats directory. Resolve those names once at the + consuming boundary. +- Cleanup uses the same layout as the producer. Preserve remote keys exactly: + `.`, `..`, and repeated separators are literal object-key components. + +## Local upgrade compatibility + +Let `R = localStorage.path` and `M = the legacy minio.rootPath`. + +| Existing layout | Upgrade behavior | +| --- | --- | +| `R//...` | Use directly | +| Manifest base `M/insert_log/...`, files at `R/M/insert_log/...` | Resolve the loaded manifest to the existing absolute path; read in place | +| `R///...` | Move displaced layouts into the standard root at standalone startup | +| Unified indexes under the old process CWD | Move recognized index directories if they remain accessible | + +Migration uses rename, merges overlapping directories, and falls back to copy +on cross-filesystem moves. It does not overwrite conflicting files. Restart +resumes from remaining sources. Shutdown cancellation is checked while walking +and between file-copy reads; partial temporary copies are never published. There +is no configured migration timeout, reverse migration, or backup. +Old relative `localStorage.path` configurations are not supported. + +`normalizeLocalManifestPath` applies only to locally owned segments, not +remote data or foreign/self-contained snapshots. It updates the loaded view, +without writing the catalog; a later ordinary update may persist the path. +Keep the old `minio.rootPath` unchanged while relative legacy manifests remain. +Compatibility rules must not become new-write rules. + +TEXT LOB references contain a file ID and row offset, not their source storage +prefix. Compaction may reuse those encoded references only when every source +manifest and the output manifest have the same partition base. When a legacy +local source under `R/M/insert_log//

` is compacted into a canonical output +under `R/insert_log//

`, the source reader must decode TEXT through the +legacy LOB directory and the output writer must create new LOB files under the +canonical directory. Copying references or merely registering the old LOB file +metadata in the output manifest is not sufficient: readers reconstruct the LOB +directory from the output manifest base. + +## Necessary checks, at clear boundaries + +- Configuration initialization requires an absolute local root and freezes it + against runtime configuration updates. +- Metadata loading resolves recognized legacy paths and checks their identity. +- Deletion validates its segment/field ownership and applicable root boundary. + Recursive segment prefixes end in `/`, so `2001` cannot match `20010`. +- Avoid repeated URI or path-spelling blacklists. Check the actual range or + ownership constraint, and share a helper when the semantics are identical. +- Migration recognizes old CWD indexes by layout and + `milvus_packed__index.v3` names, rather than moving arbitrary CWD data. + Packed files move last during a merge so interrupted leaves stay recognizable. + Symlinks inside migration paths are refused; automatic alias repair is not + supported. Root confinement alone does not protect an in-root live alias. + +## GC coverage and limits + +- V1/V2: delete recorded files and periodically scan standard layouts for + orphans, including independent `bm25_stats` files. +- Dropped V3 segments: remove the manifest base directory, including data and + stats, subject to retention and snapshot protection. +- V3 orphan scan: enumerate `insert_log` and reclaim files without segment + metadata after the protection period. Files belonging to registered V3 + segments are skipped, even if an individual file is absent from the manifest. +- Therefore an uncommitted stats file inside a live V3 segment is not guaranteed + to be reclaimed by DataCoord's orphan scan. +- Legacy `R/M/insert_log` orphan/LOB layouts and old Explore temporary paths + have separate cleanup gaps. Do not claim this PR repairs all such leftovers. + Startup cannot recover indexes hidden with an old container's CWD. + +## Implementation and regression tests + +- Roots and writers: `internal/storagev2/loon_root.go`, + `internal/core/src/storage/loon_ffi/util.cpp`, + `internal/storage/binlog_record_writer.go`. +- Upgrade: `cmd/roles/local_layout.go`, `internal/storage/localmigrate/*_test.go`. +- Compatibility: `internal/datacoord/meta_local_manifest*.go` and + `snapshot_local_manifest_test.go`. +- Cleanup: `internal/datacoord/garbage_collector*.go` and `task_stats*.go`. + +Source review and unit tests do not replace a real version-to-version upgrade +test. Report which versions, layouts, and failure cases were actually exercised. diff --git a/internal/compaction/lob_compaction.go b/internal/compaction/lob_compaction.go index 3fe79c9a8f..dcd08f07d0 100644 --- a/internal/compaction/lob_compaction.go +++ b/internal/compaction/lob_compaction.go @@ -19,6 +19,7 @@ package compaction import ( "context" "fmt" + "path" "github.com/milvus-io/milvus-proto/go-api/v3/schemapb" "github.com/milvus-io/milvus/internal/storagev2/packed" @@ -43,10 +44,12 @@ const ( // // forced strategies: // - clustering compaction: always REWRITE_ALL (data is repartitioned) -// - sort compaction: always REUSE_ALL (row order changes but same data) +// - sort compaction: REUSE_ALL when source and output share a partition base +// (row order changes but the LOB namespace stays valid) // - schema-bump compaction: always REUSE_ALL (1->1, only materializes a new // non-LOB output column + bumps schema version; existing TEXT LOB data is -// unchanged, so its refs are carried as-is — never REWRITE_ALL) +// unchanged). Compactors must override this to REWRITE_ALL when a full +// rewrite moves the output to a different partition base. // - mix compaction with multiple outputs (N->M, M>1): always REWRITE_ALL // (LOB refs cannot be duplicated across output manifests without inflating ValidRows) // - L0 delete compaction: always SKIP (only applies delete logs, segment LOB refs unchanged) @@ -271,6 +274,10 @@ type LOBCompactionContext struct { ForcedStrategy LOBCompactionStrategy // isForced indicates whether strategy is forced (skip hole ratio calculation) IsForced bool + // DecodeTextFromSource is set when source and output partition bases differ. + // The compactor must decode references with the source base before writing + // new LOB files under the output base. + DecodeTextFromSource bool } // SegmentRowStats tracks row statistics for a segment during compaction @@ -355,6 +362,75 @@ func (ctx *LOBCompactionContext) ComputeStrategies(textFieldIDs []int64, thresho } } +// ForceRewriteAll overrides the normal compaction-type or hole-ratio decision. +// It is used when encoded TEXT references would cross a partition namespace: +// a reference contains only file ID and row offset, while readers reconstruct +// its LOB directory from the output manifest's partition base. +func (ctx *LOBCompactionContext) ForceRewriteAll(textFieldIDs []int64) { + if ctx == nil { + return + } + ctx.ForcedStrategy = LOBStrategyRewriteAll + ctx.IsForced = true + for _, fieldID := range textFieldIDs { + decision := ctx.Decisions[fieldID] + decision.FieldID = fieldID + decision.Strategy = LOBStrategyRewriteAll + ctx.Decisions[fieldID] = decision + } +} + +// ForceRewriteAllAcrossPartitionBases selects source-side TEXT decoding and +// canonical output rewriting in addition to overriding the field strategies. +func (ctx *LOBCompactionContext) ForceRewriteAllAcrossPartitionBases(textFieldIDs []int64) { + ctx.ForceRewriteAll(textFieldIDs) + if ctx != nil { + ctx.DecodeTextFromSource = true + } +} + +// LOBSourcePartitionBaseMismatch reports whether any source manifest resolves +// TEXT LOB references relative to a different partition base than the output. +// REUSE_ALL is safe only when every source and the output share this base. +func LOBSourcePartitionBaseMismatch(sourceManifests map[int64]string, outputPartitionBase string) (bool, error) { + outputPartitionBase = path.Clean(outputPartitionBase) + for _, manifestPath := range sourceManifests { + if manifestPath == "" { + continue + } + segmentBasePath, _, err := packed.UnmarshalManifestPath(manifestPath) + if err != nil { + return false, err + } + if path.Clean(path.Dir(segmentBasePath)) != outputPartitionBase { + return true, nil + } + } + return false, nil +} + +// GetSourceTextColumnConfigs derives the source LOB directories used by the +// TEXT-decoding SegmentReader for a single manifest. +func (ctx *LOBCompactionContext) GetSourceTextColumnConfigs(manifestPath string) ([]packed.TextColumnConfig, error) { + if ctx == nil || !ctx.DecodeTextFromSource { + return nil, nil + } + segmentBasePath, _, err := packed.UnmarshalManifestPath(manifestPath) + if err != nil { + return nil, err + } + partitionBasePath := path.Dir(segmentBasePath) + fieldIDs := ctx.GetRewriteAllFieldIDs() + configs := make([]packed.TextColumnConfig, 0, len(fieldIDs)) + for _, fieldID := range fieldIDs { + configs = append(configs, packed.TextColumnConfig{ + FieldID: fieldID, + LobBasePath: path.Join(partitionBasePath, "lobs", fmt.Sprint(fieldID)), + }) + } + return configs, nil +} + // GetStrategy returns the strategy for a specific field. func (ctx *LOBCompactionContext) GetStrategy(fieldID int64) LOBCompactionStrategy { if decision, ok := ctx.Decisions[fieldID]; ok { @@ -655,7 +731,7 @@ func (ctx *LOBCompactionContext) GetTextColumnConfigs(partitionBasePath string, InlineThreshold: inlineThreshold, MaxLobFileBytes: maxLobFileBytes, FlushThresholdBytes: flushThresholdBytes, - RewriteMode: true, + RewriteMode: !ctx.DecodeTextFromSource, }) } diff --git a/internal/compaction/lob_compaction_test.go b/internal/compaction/lob_compaction_test.go index 516ca2d94b..6e1b8ab5ba 100644 --- a/internal/compaction/lob_compaction_test.go +++ b/internal/compaction/lob_compaction_test.go @@ -26,6 +26,90 @@ import ( "github.com/milvus-io/milvus/pkg/v3/proto/datapb" ) +func TestLOBSourcePartitionBaseMismatch(t *testing.T) { + outputPartitionBase := "/var/lib/milvus/data/insert_log/1/2" + tests := []struct { + name string + manifests map[int64]string + want bool + wantErr bool + }{ + { + name: "same canonical partition", + manifests: map[int64]string{ + 10: packed.MarshalManifestPath(outputPartitionBase+"/10", 1), + 11: packed.MarshalManifestPath(outputPartitionBase+"/11", 2), + }, + }, + { + name: "legacy partition", + manifests: map[int64]string{ + 10: packed.MarshalManifestPath("/var/lib/milvus/data/files/insert_log/1/2/10", 1), + }, + want: true, + }, + { + name: "mixed partitions", + manifests: map[int64]string{ + 10: packed.MarshalManifestPath(outputPartitionBase+"/10", 1), + 11: packed.MarshalManifestPath("/var/lib/milvus/data/files/insert_log/1/2/11", 1), + }, + want: true, + }, + { + name: "invalid manifest", + manifests: map[int64]string{10: "not-json"}, + wantErr: true, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got, err := LOBSourcePartitionBaseMismatch(tc.manifests, outputPartitionBase) + if tc.wantErr { + assert.Error(t, err) + return + } + assert.NoError(t, err) + assert.Equal(t, tc.want, got) + }) + } +} + +func TestLOBCompactionContext_ForceRewriteAll(t *testing.T) { + ctx := NewLOBCompactionContext() + ctx.SetCompactionType(datapb.CompactionType_SortCompaction, 1, 1) + ctx.ComputeStrategies([]int64{101, 102}, DefaultLOBHoleRatioThreshold) + + ctx.ForceRewriteAll([]int64{101, 102}) + + assert.True(t, ctx.IsForced) + assert.Equal(t, LOBStrategyRewriteAll, ctx.ForcedStrategy) + assert.False(t, ctx.HasReuseAllFields()) + assert.True(t, ctx.ShouldRewriteAnyField()) + assert.Equal(t, LOBStrategyRewriteAll, ctx.GetStrategy(101)) + assert.Equal(t, LOBStrategyRewriteAll, ctx.GetStrategy(102)) +} + +func TestLOBCompactionContext_CrossPartitionRewriteConfigs(t *testing.T) { + ctx := NewLOBCompactionContext() + ctx.SetCompactionType(datapb.CompactionType_SortCompaction, 1, 1) + ctx.ComputeStrategies([]int64{101}, DefaultLOBHoleRatioThreshold) + ctx.ForceRewriteAllAcrossPartitionBases([]int64{101}) + + sourceManifest := packed.MarshalManifestPath("/data/files/insert_log/1/2/10", 3) + sourceConfigs, err := ctx.GetSourceTextColumnConfigs(sourceManifest) + assert.NoError(t, err) + if assert.Len(t, sourceConfigs, 1) { + assert.Equal(t, "/data/files/insert_log/1/2/lobs/101", sourceConfigs[0].LobBasePath) + } + outputConfigs := ctx.GetTextColumnConfigs("/data/insert_log/1/2", 64, 1024, 512) + if assert.Len(t, outputConfigs, 1) { + assert.Equal(t, "/data/insert_log/1/2/lobs/101", outputConfigs[0].LobBasePath) + assert.False(t, outputConfigs[0].RewriteMode, "input is already decoded by the source reader") + } +} + func TestGetForcedStrategy_AllTypes(t *testing.T) { tests := []struct { name string diff --git a/internal/core/src/clustering/KmeansClusteringTest.cpp b/internal/core/src/clustering/KmeansClusteringTest.cpp index 8d0c6fa12e..3ed2cf324b 100644 --- a/internal/core/src/clustering/KmeansClusteringTest.cpp +++ b/internal/core/src/clustering/KmeansClusteringTest.cpp @@ -386,11 +386,9 @@ TEST(KmeansClusteringTest, ReadFromManifestStorageV3) { ASSERT_EQ(v3.TotalRows(), num_rows); ASSERT_EQ(v3.NumColumnGroups(), 1); - // The fixture wrote to the LOCAL filesystem, and - // MakeInternalPropertiesFromStorageConfig copies storage_type/root_path - // verbatim into the loon properties. So the storage config must be - // local-typed and rooted at root_path (ManifestPathJson() returns a - // relative base_path; loon resolves it against PROPERTY_FS_ROOT_PATH). + // The fixture wrote to the local filesystem and ManifestPathJson() returns + // an absolute base_path. MakeInternalPropertiesFromStorageConfig uses "/" + // as the local filesystem root, leaving that complete path unchanged. // Leaving StorageConfig at its default ("minio") would send the manifest // reader to object storage. auto storage_config = gen_local_storage_config(root_path); @@ -424,8 +422,7 @@ TEST(KmeansClusteringTest, ReadFromManifestStorageV3) { pb_storage_config->set_root_path(storage_config.root_path); // Mirror analyze_c.cpp: the field schema is carried in FieldDataMeta, and - // the loon properties come from the storage config through the very - // production helper under test - not from MakeInternalLocalProperies. + // the loon properties come from the production storage-config helper. milvus::storage::FieldDataMeta field_data_meta{ collection_id, partition_id, 0, vec_fid.get(), info.field_schema()}; milvus::storage::IndexMeta index_meta{ diff --git a/internal/core/src/common/VectorArrayStorageV2Test.cpp b/internal/core/src/common/VectorArrayStorageV2Test.cpp index f8b540b90a..4f5b09ad2c 100644 --- a/internal/core/src/common/VectorArrayStorageV2Test.cpp +++ b/internal/core/src/common/VectorArrayStorageV2Test.cpp @@ -138,8 +138,9 @@ class TestVectorArrayStorageV2 : public testing::Test { auto fs = milvus::segcore::GetDefaultArrowFileSystem(); // Prepare paths and column groups - std::vector paths = {"test_data/0/10000.parquet", - "test_data/101/10001.parquet"}; + std::vector paths = { + TestLocalPath + "test_data/0/10000.parquet", + TestLocalPath + "test_data/101/10001.parquet"}; // Create directories for the parquet files for (const auto& path : paths) { @@ -293,7 +294,7 @@ class TestVectorArrayStorageV2 : public testing::Test { void TearDown() override { auto fs = milvus::segcore::GetDefaultArrowFileSystem(); - (void)fs->DeleteDir("test_data"); + (void)fs->DeleteDir(TestLocalPath + "test_data"); } protected: @@ -312,7 +313,8 @@ TEST_F(TestVectorArrayStorageV2, BuildEmbListHNSWIndex) { ASSERT_TRUE(segment_->HasFieldData(vector_array_field_id)); // Get the storage v2 parquet file paths that were already written in SetUp - std::vector paths = {"test_data/101/10001.parquet"}; + std::vector paths = {TestLocalPath + + "test_data/101/10001.parquet"}; // Use the existing Arrow file system from SetUp auto fs = milvus::segcore::GetDefaultArrowFileSystem(); @@ -427,7 +429,8 @@ TEST_F(TestVectorArrayStorageV2, BuildEmbListHNSWIndexWithMmap) { ASSERT_TRUE(segment_->HasFieldData(vector_array_field_id)); // Get the storage v2 parquet file paths that were already written in SetUp - std::vector paths = {"test_data/101/10001.parquet"}; + std::vector paths = {TestLocalPath + + "test_data/101/10001.parquet"}; // Use the existing Arrow file system from SetUp auto fs = milvus::segcore::GetDefaultArrowFileSystem(); @@ -565,7 +568,8 @@ TEST_F(TestVectorArrayStorageV2, BuildEncodedEmbListHNSWIndexWithMmap) { auto vector_array_field_id = fields_["vector_array"]; ASSERT_TRUE(segment_->HasFieldData(vector_array_field_id)); - std::vector paths = {"test_data/101/10001.parquet"}; + std::vector paths = {TestLocalPath + + "test_data/101/10001.parquet"}; auto fs = milvus::segcore::GetDefaultArrowFileSystem(); auto storage_config = gen_local_storage_config(TestLocalPath); auto cm = CreateChunkManager(storage_config); diff --git a/internal/core/src/exec/expression/LikeConjunctExprTest.cpp b/internal/core/src/exec/expression/LikeConjunctExprTest.cpp index eb48eabb33..b6753f7e0f 100644 --- a/internal/core/src/exec/expression/LikeConjunctExprTest.cpp +++ b/internal/core/src/exec/expression/LikeConjunctExprTest.cpp @@ -32,6 +32,7 @@ #include "common/Types.h" #include "common/protobuf_utils.h" #include "common/type_c.h" +#include "exec/expression/ExprBatchTestUtils.h" #include "exec/expression/function/FunctionFactory.h" #include "expr/ITypeExpr.h" #include "filemanager/InputStream.h" @@ -86,7 +87,7 @@ TEST(LikeConjunctExpr, TestMultiFieldMultiLikeWithRetrieve) { int64_t segment_id = 3; int64_t index_version = 4000; - EXEC_EVAL_EXPR_BATCH_SIZE.store(3); + milvus::test::ExprBatchSizeGuard batch_size_guard(3); auto schema = std::make_shared(); auto pk_fid = schema->AddDebugField("pk", DataType::INT64); @@ -94,8 +95,9 @@ TEST(LikeConjunctExpr, TestMultiFieldMultiLikeWithRetrieve) { auto content_fid = schema->AddDebugField("content", DataType::VARCHAR); schema->set_primary_field_id(pk_fid); - // Use TestLocalPath to match LocalChunkManagerSingleton initialized in init_gtest.cpp - auto storage_config = gen_local_storage_config(TestLocalPath); + // AppendIndexV2 loads through RemoteChunkManagerSingleton, so build the + // index under the same TestRemotePath root. + auto storage_config = get_default_local_storage_config(); auto cm = CreateChunkManager(storage_config); auto fs = storage::InitArrowFileSystem(storage_config); @@ -202,13 +204,14 @@ TEST(LikeConjunctExpr, TestMultiFieldMultiLikeWithRetrieve) { insert_data.SetTimestamps(0, 100); auto serialized_bytes = insert_data.Serialize(storage::Remote); - auto log_path = fmt::format("{}{}/{}/{}/{}/{}", - TestLocalPath, - collection_id, - partition_id, - segment_id, - field_id.get(), - 0); + auto log_path = + fmt::format("{}insert_log/like_conjunct/{}/{}/{}/{}/{}", + storage_config.root_path, + collection_id, + partition_id, + segment_id, + field_id.get(), + 0); cm_w.Write( log_path, serialized_bytes.data(), serialized_bytes.size()); @@ -358,6 +361,4 @@ TEST(LikeConjunctExpr, TestMultiFieldMultiLikeWithRetrieve) { } EXPECT_EQ(actual_pks, expected_pks); - - EXEC_EVAL_EXPR_BATCH_SIZE.store(DEFAULT_EXEC_EVAL_EXPR_BATCH_SIZE); } diff --git a/internal/core/src/index/FMIndexTest.cpp b/internal/core/src/index/FMIndexTest.cpp index 4e4dc1b542..c2ef922bdb 100644 --- a/internal/core/src/index/FMIndexTest.cpp +++ b/internal/core/src/index/FMIndexTest.cpp @@ -1067,7 +1067,7 @@ TEST(FMIndex, ExecutorPathDeclinedOpsFallBackToScan) { false); auto index_meta = gen_index_meta( segment_id, field_id.get(), index_build_id, index_version); - auto storage_config = gen_local_storage_config(TestLocalPath); + auto storage_config = get_default_local_storage_config(); auto cm = CreateChunkManager(storage_config); auto fs = storage::InitArrowFileSystem(storage_config); @@ -1096,8 +1096,8 @@ TEST(FMIndex, ExecutorPathDeclinedOpsFallBackToScan) { insert_data.SetFieldDataMeta(field_meta); insert_data.SetTimestamps(0, 100); auto serialized_bytes = insert_data.Serialize(storage::Remote); - auto log_path = fmt::format("{}{}/{}/{}/{}/{}", - TestLocalPath, + auto log_path = fmt::format("{}insert_log/fm_index/{}/{}/{}/{}/{}", + storage_config.root_path, collection_id, partition_id, segment_id, @@ -1256,7 +1256,7 @@ TEST(FMIndex, ExecutorPathMatchRechecksVarchar) { false); auto index_meta = gen_index_meta( segment_id, field_id.get(), index_build_id, index_version); - auto storage_config = gen_local_storage_config(TestLocalPath); + auto storage_config = get_default_local_storage_config(); auto cm = CreateChunkManager(storage_config); auto fs = storage::InitArrowFileSystem(storage_config); @@ -1297,8 +1297,8 @@ TEST(FMIndex, ExecutorPathMatchRechecksVarchar) { insert_data.SetFieldDataMeta(field_meta); insert_data.SetTimestamps(0, 100); auto serialized_bytes = insert_data.Serialize(storage::Remote); - auto log_path = fmt::format("{}{}/{}/{}/{}/{}", - TestLocalPath, + auto log_path = fmt::format("{}insert_log/fm_index/{}/{}/{}/{}/{}", + storage_config.root_path, collection_id, partition_id, segment_id, @@ -1419,9 +1419,8 @@ struct SealedFMMatch { FieldId varchar_id; FieldId int_id; std::unique_ptr segment; - // Owns the insert-log write and keeps TestLocalPath alive until the - // loaded segment is destroyed. ChunkManagerWrapper::dtor wipes the - // chunk-manager root. + // Owns the insert-log write and cleans the remote test root with this + // fixture. ChunkManagerWrapper::dtor wipes the chunk-manager root. std::unique_ptr cm_w; }; @@ -1488,11 +1487,11 @@ LoadSealedFMMatch(int64_t collection_id, /*max_length=*/65535); auto index_meta = gen_index_meta( segment_id, out.varchar_id.get(), index_build_id, index_build_id); - auto storage_config = gen_local_storage_config(TestLocalPath); + auto storage_config = get_default_local_storage_config(); auto cm = CreateChunkManager(storage_config); - // Same FS handle as ExecutorPathMatchRechecksVarchar. UploadUnified writes - // through StorageV2FSCache; AppendIndexV2 reads through the process Arrow - // FS. Both are rooted at TestLocalPath, so the packed file is visible. + // AppendIndexV2 loads through RemoteChunkManagerSingleton, so build the + // index under that singleton's TestRemotePath root as well. TestLocalPath + // remains the mmap/local-temp root. auto fs = storage::InitArrowFileSystem(storage_config); out.cm_w = std::make_unique(cm); @@ -1544,8 +1543,8 @@ LoadSealedFMMatch(int64_t collection_id, insert_data.SetFieldDataMeta(field_meta); insert_data.SetTimestamps(0, 100); auto serialized_bytes = insert_data.Serialize(storage::Remote); - auto log_path = fmt::format("{}{}/{}/{}/{}/{}", - TestLocalPath, + auto log_path = fmt::format("{}insert_log/fm_index/{}/{}/{}/{}/{}", + storage_config.root_path, collection_id, partition_id, segment_id, diff --git a/internal/core/src/index/Meta.h b/internal/core/src/index/Meta.h index 01d00a6aff..3acbf1f061 100644 --- a/internal/core/src/index/Meta.h +++ b/internal/core/src/index/Meta.h @@ -133,8 +133,4 @@ constexpr const char* DISK_ANN_QUERY_BEAMWIDTH = "beamwidth"; // HNSW query params constexpr const char* HNSW_QUERY_EF = "ef"; -// UT-only: when non-empty, overrides rcm_->GetRootPath() in -// GetRemoteIndexObjectPrefix/GetRemoteTextLogPrefix to avoid -// absolute path duplication with SubTreeFileSystem in tests. -extern std::string kOverrideRootPathForUT; } // namespace milvus::index diff --git a/internal/core/src/index/NgramInvertedIndexTest.cpp b/internal/core/src/index/NgramInvertedIndexTest.cpp index b9be6dd5b9..4030c422d3 100644 --- a/internal/core/src/index/NgramInvertedIndexTest.cpp +++ b/internal/core/src/index/NgramInvertedIndexTest.cpp @@ -147,8 +147,7 @@ test_ngram_with_data(const boost::container::vector& data, auto index_meta = gen_index_meta( segment_id, field_id.get(), index_build_id, index_version); - std::string root_path = TestLocalPath; - auto storage_config = gen_local_storage_config(root_path); + auto storage_config = get_default_local_storage_config(); auto cm = CreateChunkManager(storage_config); auto fs = storage::InitArrowFileSystem(storage_config); @@ -176,8 +175,8 @@ test_ngram_with_data(const boost::container::vector& data, auto serialized_bytes = insert_data.Serialize(storage::Remote); auto get_binlog_path = [=](int64_t log_id) { - return fmt::format("{}{}/{}/{}/{}/{}", - TestLocalPath, + return fmt::format("{}insert_log/ngram_index/{}/{}/{}/{}/{}", + storage_config.root_path, collection_id, partition_id, segment_id, @@ -466,8 +465,7 @@ TEST(NgramIndex, TestNonLikeExpressionsWithNgram) { auto index_meta = gen_index_meta( segment_id, field_id.get(), index_build_id, index_version); - std::string root_path = TestLocalPath; - auto storage_config = gen_local_storage_config(root_path); + auto storage_config = get_default_local_storage_config(); auto cm = CreateChunkManager(storage_config); auto fs = storage::InitArrowFileSystem(storage_config); @@ -495,8 +493,8 @@ TEST(NgramIndex, TestNonLikeExpressionsWithNgram) { auto serialized_bytes = insert_data.Serialize(storage::Remote); auto get_binlog_path = [=](int64_t log_id) { - return fmt::format("{}{}/{}/{}/{}/{}", - TestLocalPath, + return fmt::format("{}insert_log/ngram_index/{}/{}/{}/{}/{}", + storage_config.root_path, collection_id, partition_id, segment_id, diff --git a/internal/core/src/segcore/ChunkedSegmentSealedStorageV2Test.cpp b/internal/core/src/segcore/ChunkedSegmentSealedStorageV2Test.cpp index cd635c5acc..deef6e138c 100644 --- a/internal/core/src/segcore/ChunkedSegmentSealedStorageV2Test.cpp +++ b/internal/core/src/segcore/ChunkedSegmentSealedStorageV2Test.cpp @@ -22,6 +22,8 @@ #include #include #include + +#include "test_utils/Constants.h" #include #include #include @@ -246,7 +248,8 @@ TEST(ChunkedSegmentSealedStorageV2, auto schema = Schema::ParseFrom(schema_proto); auto fs = milvus::segcore::GetDefaultArrowFileSystem(); - const std::string dir = "test_data/storage_v2_direct_warmup"; + const std::string dir = + TestLocalPath + "test_data/storage_v2_direct_warmup"; StorageV2TempDirGuard dir_guard(fs, dir); const std::string path = dir + "/vec.parquet"; ASSERT_TRUE(fs->CreateDir(dir).ok()); @@ -365,9 +368,10 @@ class TestChunkSegmentStorageV2 : public testing::TestWithParam { auto fs = milvus::segcore::GetDefaultArrowFileSystem(); // Prepare paths and column groups - std::vector paths = {"test_data/0/10000.parquet", - "test_data/102/10001.parquet", - "test_data/103/10002.parquet"}; + std::vector paths = { + TestLocalPath + "test_data/0/10000.parquet", + TestLocalPath + "test_data/102/10001.parquet", + TestLocalPath + "test_data/103/10002.parquet"}; // Create directories for the parquet files for (const auto& path : paths) { @@ -496,7 +500,7 @@ class TestChunkSegmentStorageV2 : public testing::TestWithParam { } // Clean up test data directory auto fs = milvus::segcore::GetDefaultArrowFileSystem(); - auto status = fs->DeleteDir("test_data"); + auto status = fs->DeleteDir(TestLocalPath + "test_data"); ASSERT_TRUE(status.ok()); } @@ -909,7 +913,8 @@ TEST(TestChunkSegmentStorageV2Regression, schema->set_primary_field_id(right_fid); auto fs = milvus::segcore::GetDefaultArrowFileSystem(); - const std::string root = "test_compare_expr_misaligned_storage_v2"; + const std::string root = + TestLocalPath + "test_compare_expr_misaligned_storage_v2"; auto cleanup_status = fs->DeleteDir(root); (void)cleanup_status; ASSERT_TRUE(fs->CreateDir(root + "/0").ok()); diff --git a/internal/core/src/segcore/flush_growing_segment_test.cpp b/internal/core/src/segcore/flush_growing_segment_test.cpp index 9e8fbc6406..b01196adf0 100644 --- a/internal/core/src/segcore/flush_growing_segment_test.cpp +++ b/internal/core/src/segcore/flush_growing_segment_test.cpp @@ -14,7 +14,6 @@ #include #include #include -#include #include #include #include @@ -28,6 +27,7 @@ #include "test_utils/c_api_test_utils.h" #include "test_utils/DataGen.h" #include "test_utils/SegcoreConfigUtils.h" +#include "test_utils/TmpPath.h" #include "storage/Util.h" #include "storage/loon_ffi/property_singleton.h" #include "knowhere/index/index_factory.h" @@ -38,8 +38,6 @@ using namespace milvus; using namespace milvus::segcore; -namespace fs = std::filesystem; - class FlushGrowingSegmentTest : public ::testing::Test { protected: struct ParsedBM25Stats { @@ -50,21 +48,13 @@ class FlushGrowingSegmentTest : public ::testing::Test { void SetUp() override { - // create a temporary directory for test output - test_dir_ = "/tmp/flush_growing_test_" + std::to_string(time(nullptr)); - fs::create_directories(test_dir_); - + // Each fixture owns a unique directory under the shard-specific + // TestLocalPath. TmpPath also cleans up only this fixture's files. + test_dir_ = temp_path_.get().string(); // Arrow filesystem is initialized by init_gtest.cpp } - void - TearDown() override { - // cleanup test directory - if (fs::exists(test_dir_)) { - fs::remove_all(test_dir_); - } - } - + milvus::test::TmpPath temp_path_; std::string test_dir_; std::vector diff --git a/internal/core/src/storage/FileManager.h b/internal/core/src/storage/FileManager.h index d6906f50d6..af3c6f35db 100644 --- a/internal/core/src/storage/FileManager.h +++ b/internal/core/src/storage/FileManager.h @@ -309,9 +309,7 @@ class FileManagerImpl : public milvus::FileManager { virtual std::string GetRemoteIndexObjectPrefix() const { - boost::filesystem::path prefix = index::kOverrideRootPathForUT.empty() - ? rcm_->GetRootPath() - : index::kOverrideRootPathForUT; + boost::filesystem::path prefix = rcm_->GetRootPath(); if (index_meta_.index_store_path_version >= ::milvus::proto::index::IndexStorePathVersion:: INDEX_STORE_PATH_VERSION_COLLECTION_ROOTED) { @@ -336,9 +334,7 @@ class FileManagerImpl : public milvus::FileManager { if (!stats_base_path_.empty()) { return stats_base_path_; } - boost::filesystem::path prefix = index::kOverrideRootPathForUT.empty() - ? rcm_->GetRootPath() - : index::kOverrideRootPathForUT; + boost::filesystem::path prefix = rcm_->GetRootPath(); boost::filesystem::path path = std::string(TEXT_LOG_ROOT_PATH); boost::filesystem::path path1 = std::to_string(index_meta_.build_id) + "/" + diff --git a/internal/core/src/storage/StorageV2FSCache.cpp b/internal/core/src/storage/StorageV2FSCache.cpp index 3da8425b61..453ff0036e 100644 --- a/internal/core/src/storage/StorageV2FSCache.cpp +++ b/internal/core/src/storage/StorageV2FSCache.cpp @@ -16,6 +16,7 @@ #include "milvus-storage/filesystem/fs.h" #include "milvus-storage/properties.h" #include "log/Log.h" +#include "storage/loon_ffi/util.h" namespace milvus::storage { @@ -39,7 +40,11 @@ StorageV2FSCache::Get(const Key& key) { props[PROPERTY_FS_BUCKET_NAME] = key.bucket_name; props[PROPERTY_FS_ACCESS_KEY_ID] = key.access_key_id; props[PROPERTY_FS_ACCESS_KEY_VALUE] = key.access_key_value; - props[PROPERTY_FS_ROOT_PATH] = key.root_path; + // Keys handed to this filesystem already carry the storage prefix, so the + // local filesystem is rooted at "/" (remote is rooted at the bucket by + // milvus-storage regardless of root_path). See LoonFSRootPath. + const auto fs_root_path = LoonFSRootPath(key.storage_type, key.root_path); + props[PROPERTY_FS_ROOT_PATH] = fs_root_path; props[PROPERTY_FS_STORAGE_TYPE] = key.storage_type; props[PROPERTY_FS_CLOUD_PROVIDER] = key.cloud_provider; props[PROPERTY_FS_IAM_ENDPOINT] = key.iam_endpoint; @@ -60,11 +65,12 @@ StorageV2FSCache::Get(const Key& key) { props[PROPERTY_FS_USE_CRC32C_CHECKSUM] = key.use_crc32c_checksum; LOG_INFO( - "StorageV2FSCache::Get: address={}, bucket={}, root_path={}, " - "storage_type={}", + "StorageV2FSCache::Get: address={}, bucket={}, key_prefix={}, " + "fs_root_path={}, storage_type={}", key.address, key.bucket_name, key.root_path, + fs_root_path, key.storage_type); auto result = milvus_storage::FilesystemCache::getInstance().get(props, ""); diff --git a/internal/core/src/storage/StorageV2FSCacheLocalRootTest.cpp b/internal/core/src/storage/StorageV2FSCacheLocalRootTest.cpp new file mode 100644 index 0000000000..07d4dc1f10 --- /dev/null +++ b/internal/core/src/storage/StorageV2FSCacheLocalRootTest.cpp @@ -0,0 +1,104 @@ +// Licensed to the LF AI & Data foundation under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include + +#include +#include + +#include "common/type_c.h" +#include "milvus-storage/properties.h" +#include "storage/StorageV2FSCache.h" +#include "storage/loon_ffi/util.h" + +namespace { + +std::string +MakeTempRoot(const std::string& name) { + auto root = + std::filesystem::temp_directory_path() / + ("milvus_local_root_" + name + "_" + std::to_string(::getpid())); + std::filesystem::remove_all(root); + std::filesystem::create_directories(root); + return root.string(); +} + +} // namespace + +// Milvus hands complete filesystem paths (already prefixed with +// localStorage.path) to the local Arrow / loon filesystem, so that filesystem +// must be rooted at "/". If it were rooted at localStorage.path again, every +// write would land under //... (milvus-storage #351, #53051). +TEST(StorageV2FSCacheLocalRootTest, AbsolutePathsAreNotJoinedWithRootTwice) { + const auto root = MakeTempRoot("fs"); + + milvus::storage::StorageV2FSCache::Key key; + key.root_path = root; + key.storage_type = "local"; + auto fs = milvus::storage::StorageV2FSCache::Instance().Get(key); + ASSERT_NE(fs, nullptr); + + const auto dir = root + "/insert_log/1/2/3/_data"; + const auto file = dir + "/0.parquet"; + ASSERT_TRUE(fs->CreateDir(dir, /*recursive=*/true).ok()); + auto out = fs->OpenOutputStream(file); + ASSERT_TRUE(out.ok()) << out.status().ToString(); + ASSERT_TRUE(out.ValueOrDie()->Write("x", 1).ok()); + ASSERT_TRUE(out.ValueOrDie()->Close().ok()); + + // Physical location is exactly the key. + EXPECT_TRUE(std::filesystem::exists(file)); + // And not the double-joined location. + EXPECT_FALSE(std::filesystem::exists(root + root + "/insert_log")); + + // Reading back through the same filesystem with the same key works. + auto in = fs->OpenInputFile(file); + ASSERT_TRUE(in.ok()) << in.status().ToString(); + auto size = in.ValueOrDie()->GetSize(); + ASSERT_TRUE(size.ok()); + EXPECT_EQ(size.ValueOrDie(), 1); + + std::filesystem::remove_all(root); +} + +TEST(StorageV2FSCacheLocalRootTest, LoonFSRootPathRule) { + EXPECT_EQ(LoonFSRootPath("local", "/var/lib/milvus/data"), "/"); + EXPECT_EQ(LoonFSRootPath("local", ""), "/"); + EXPECT_EQ(LoonFSRootPath("remote", "files"), "files"); + EXPECT_EQ(LoonFSRootPath("minio", "files"), "files"); + EXPECT_EQ(LoonFSRootPath("", "files"), "files"); +} + +TEST(StorageV2FSCacheLocalRootTest, InternalPropertiesUseSlashForLocal) { + CStorageConfig c_config{}; + c_config.root_path = "/var/lib/milvus/data"; + c_config.storage_type = "local"; + auto props = MakeInternalPropertiesFromStorageConfig(c_config); + auto root = milvus_storage::api::GetValue( + *props, PROPERTY_FS_ROOT_PATH); + ASSERT_TRUE(root.ok()); + EXPECT_EQ(root.ValueOrDie(), "/"); + + CStorageConfig remote{}; + remote.root_path = "files"; + remote.storage_type = "remote"; + auto remote_props = MakeInternalPropertiesFromStorageConfig(remote); + auto remote_root = milvus_storage::api::GetValue( + *remote_props, PROPERTY_FS_ROOT_PATH); + ASSERT_TRUE(remote_root.ok()); + EXPECT_EQ(remote_root.ValueOrDie(), "files"); +} diff --git a/internal/core/src/storage/loon_ffi/property_singleton.h b/internal/core/src/storage/loon_ffi/property_singleton.h index 9d8e6c20f6..c742546076 100644 --- a/internal/core/src/storage/loon_ffi/property_singleton.h +++ b/internal/core/src/storage/loon_ffi/property_singleton.h @@ -65,16 +65,6 @@ class LoonFFIPropertiesSingleton { } } - void - Init(const char* root_path) { - std::unique_lock lck(mutex_); - - if (properties_ == nullptr) { - properties_ = MakeInternalLocalProperies(root_path); - ApplyArrowReaderConfig(*properties_); - } - } - void SetArrowReaderConfig(int64_t hole_size_limit_bytes, int64_t range_size_limit_bytes) { diff --git a/internal/core/src/storage/loon_ffi/util.cpp b/internal/core/src/storage/loon_ffi/util.cpp index 4c265dddab..35e1abceb7 100644 --- a/internal/core/src/storage/loon_ffi/util.cpp +++ b/internal/core/src/storage/loon_ffi/util.cpp @@ -74,9 +74,17 @@ MakeInternalPropertiesFromStorageConfig(CStorageConfig c_storage_config) { PROPERTY_FS_ACCESS_KEY_VALUE, c_storage_config.access_key_value); } - if (c_storage_config.root_path != nullptr) { + const std::string storage_type = c_storage_config.storage_type != nullptr + ? c_storage_config.storage_type + : ""; + if (c_storage_config.root_path != nullptr || storage_type == "local") { + const std::string root_path = c_storage_config.root_path != nullptr + ? c_storage_config.root_path + : ""; milvus_storage::api::SetValue( - *properties_map, PROPERTY_FS_ROOT_PATH, c_storage_config.root_path); + *properties_map, + PROPERTY_FS_ROOT_PATH, + LoonFSRootPath(storage_type, root_path).c_str()); } if (c_storage_config.storage_type != nullptr) { milvus_storage::api::SetValue(*properties_map, @@ -801,17 +809,12 @@ InjectExternalSpecProperties(milvus_storage::api::Properties& properties, } } -std::shared_ptr -MakeInternalLocalProperies(const char* c_path) { - auto properties_map = std::make_shared(); - - milvus_storage::api::SetValue( - *properties_map, PROPERTY_FS_STORAGE_TYPE, "local"); - - milvus_storage::api::SetValue( - *properties_map, PROPERTY_FS_ROOT_PATH, c_path); - - return properties_map; +std::string +LoonFSRootPath(const std::string& storage_type, const std::string& root_path) { + if (storage_type == "local") { + return kLoonLocalFSRootPath; + } + return root_path; } CStorageConfig diff --git a/internal/core/src/storage/loon_ffi/util.h b/internal/core/src/storage/loon_ffi/util.h index 11c2e0ab05..870a5ec8f6 100644 --- a/internal/core/src/storage/loon_ffi/util.h +++ b/internal/core/src/storage/loon_ffi/util.h @@ -34,16 +34,22 @@ std::shared_ptr MakeInternalPropertiesFromStorageConfig(CStorageConfig c_storage_config); /** - * @brief Create Properties for local filesystem storage + * @brief fs.root_path handed to the loon / Arrow filesystem for a storage + * config. * - * Creates a minimal Properties object configured for local file storage - * with the specified path as the root. - * - * @param c_path Local filesystem path to use as storage root - * @return Shared pointer to Properties configured for local storage + * Milvus keys already carry the storage prefix: localStorage.path for local + * (`/var/lib/milvus/data/insert_log/...`) and minio.rootPath for remote + * (`files/insert_log/...`). The filesystem therefore has to be rooted at the + * namespace root, which is the bucket for remote (milvus-storage never + * applies root_path there) and "/" for local. Rooting the local + * SubTreeFileSystem at localStorage.path joins the prefix twice + * (milvus-storage #351, milvus #53051), making complete keys differ from their + * physical locations. */ -std::shared_ptr -MakeInternalLocalProperies(const char* c_path); +inline constexpr const char* kLoonLocalFSRootPath = "/"; + +std::string +LoonFSRootPath(const std::string& storage_type, const std::string& root_path); /** * @brief Convert StorageConfig to C-style CStorageConfig diff --git a/internal/core/unittest/init_gtest.cpp b/internal/core/unittest/init_gtest.cpp index 4149536929..062a9e865e 100644 --- a/internal/core/unittest/init_gtest.cpp +++ b/internal/core/unittest/init_gtest.cpp @@ -136,6 +136,5 @@ main(int argc, char** argv) { std::chrono::milliseconds(0), std::chrono::milliseconds(-1)); - milvus::index::kOverrideRootPathForUT = "files"; return RUN_ALL_TESTS(); } diff --git a/internal/core/unittest/test_utils/ManifestTestUtil.h b/internal/core/unittest/test_utils/ManifestTestUtil.h index 5485a47cda..01eceb2597 100644 --- a/internal/core/unittest/test_utils/ManifestTestUtil.h +++ b/internal/core/unittest/test_utils/ManifestTestUtil.h @@ -16,6 +16,7 @@ #pragma once +#include "storage/loon_ffi/util.h" #include #include #include @@ -88,9 +89,12 @@ class V3SegmentTestData { int64_t dim, const std::string& root_path, const std::string& base_path) - : schema_(schema), base_path_(base_path), root_path_(root_path) { - std::filesystem::create_directories(std::filesystem::path(root_path) / - std::filesystem::path(base_path)); + : schema_(schema), + // Keys are complete paths: the loon local filesystem is rooted at + // "/" and base_path lives under root_path (see LoonFSRootPath). + base_path_((std::filesystem::path(root_path) / base_path).string()), + root_path_(root_path) { + std::filesystem::create_directories(base_path_); // Convert schema to Arrow schemas auto arrow_schema = schema_->ConvertToArrowSchema(); @@ -118,7 +122,7 @@ class V3SegmentTestData { milvus_storage::api::SetValue( props, PROPERTY_FS_STORAGE_TYPE, LOON_FS_TYPE_LOCAL); milvus_storage::api::SetValue( - props, PROPERTY_FS_ROOT_PATH, root_path.c_str()); + props, PROPERTY_FS_ROOT_PATH, kLoonLocalFSRootPath); milvus_storage::api::SetValue(props, PROPERTY_WRITER_POLICY, LOON_COLUMN_GROUP_POLICY_SCHEMA_BASED); @@ -201,7 +205,7 @@ class V3SegmentTestData { milvus_storage::api::SetValue( reader_props, PROPERTY_FS_STORAGE_TYPE, LOON_FS_TYPE_LOCAL); milvus_storage::api::SetValue( - reader_props, PROPERTY_FS_ROOT_PATH, root_path_.c_str()); + reader_props, PROPERTY_FS_ROOT_PATH, kLoonLocalFSRootPath); auto reader = milvus_storage::api::Reader::create( column_groups_, loon_schema_, needed_columns, reader_props); auto result = reader->get_chunk_reader(cg_index); diff --git a/internal/datacoord/copy_segment_task_test.go b/internal/datacoord/copy_segment_task_test.go index c8fd866074..df7c14b332 100644 --- a/internal/datacoord/copy_segment_task_test.go +++ b/internal/datacoord/copy_segment_task_test.go @@ -1588,7 +1588,7 @@ func (s *CopySegmentTaskSuite) TestSyncCopySegmentTask_IndexWritePlacementMatrix s.True(present) s.Equal(tc.enabled, persistedPlacement) - manifestPath := packed.MarshalManifestPath("files/insert_log/100/10/2001", 3) + manifestPath := packed.MarshalManifestPath("/tmp/test-restart/files/insert_log/100/10/2001", 3) var manifestBuildIDs []int64 if tc.enabled { store.revisions[manifestPath] = []packed.ManifestIndexInfo{{ diff --git a/internal/datacoord/external_collection_refresh_manager.go b/internal/datacoord/external_collection_refresh_manager.go index bae5a220fd..7a4c2b137e 100644 --- a/internal/datacoord/external_collection_refresh_manager.go +++ b/internal/datacoord/external_collection_refresh_manager.go @@ -19,6 +19,7 @@ package datacoord import ( "context" "fmt" + "path" "sync" "time" @@ -71,6 +72,19 @@ func exploreTempDirForAttempt(jobID, attemptID int64) string { return fmt.Sprintf("%s/attempt_%d", exploreTempDirForJob(jobID), attemptID) } +// exploreDirForChunkManager turns an explore temp dir into the complete key +// for the primary chunk manager. On remote storage explore manifests have +// always lived at the bucket root, outside minio.rootPath, so the key is the +// dir itself. On local storage every key is a complete filesystem path, so the +// dir is placed under localStorage.path. Producer (explore) and cleanup must +// agree on this or cleanup silently misses the files. +func (m *externalCollectionRefreshManager) exploreDirForChunkManager(dir string) string { + if _, local := m.chunkManager.(*storage.LocalChunkManager); local { + return path.Join(m.chunkManager.RootPath(), dir) + } + return dir +} + // External Collection Refresh Manager // // The manager is the facade for external collection refresh operations. It encapsulates @@ -317,7 +331,7 @@ func (m *externalCollectionRefreshManager) cleanupExploreTempForJob(jobID int64) if m.chunkManager == nil { return } - exploreBaseDir := exploreTempDirForJob(jobID) + exploreBaseDir := m.exploreDirForChunkManager(exploreTempDirForJob(jobID)) explorePrefix := exploreBaseDir + "/" // Derive from m.ctx so shutdown cancels in-flight cleanup instead of // blocking Stop() on a slow object-store call. @@ -1169,7 +1183,7 @@ func (m *externalCollectionRefreshManager) exploreExternalFiles( if err != nil { return nil, "", merr.Wrap(err, "allocate external refresh Explore attempt ID") } - exploreBaseDir := exploreTempDirForAttempt(job.GetJobId(), attemptID) + exploreBaseDir := m.exploreDirForChunkManager(exploreTempDirForAttempt(job.GetJobId(), attemptID)) fileInfos, manifestPath, err := packed.ExploreFilesReturnManifestPath( columns, spec.Format, diff --git a/internal/datacoord/external_collection_refresh_manager_test.go b/internal/datacoord/external_collection_refresh_manager_test.go index aea773bb38..7ce829b4b9 100644 --- a/internal/datacoord/external_collection_refresh_manager_test.go +++ b/internal/datacoord/external_collection_refresh_manager_test.go @@ -19,6 +19,8 @@ package datacoord import ( "context" "fmt" + "path" + "strings" "sync" "testing" "time" @@ -33,6 +35,7 @@ import ( "github.com/milvus-io/milvus-proto/go-api/v3/schemapb" "github.com/milvus-io/milvus/internal/storage" "github.com/milvus-io/milvus/internal/storagev2/packed" + "github.com/milvus-io/milvus/pkg/v3/objectstorage" "github.com/milvus-io/milvus/pkg/v3/proto/datapb" "github.com/milvus-io/milvus/pkg/v3/proto/indexpb" "github.com/milvus-io/milvus/pkg/v3/util/merr" @@ -1841,6 +1844,8 @@ func TestHandleJobFinished_SourceChangedOnly(t *testing.T) { type recordingChunkManager struct { storage.ChunkManager mu sync.Mutex + rootPath string + objects map[string][]byte prefixCalls []string removeCalls []string prefixErr error @@ -1848,6 +1853,18 @@ type recordingChunkManager struct { prefixBlockUntil <-chan struct{} } +func (r *recordingChunkManager) RootPath() string { return r.rootPath } + +func (r *recordingChunkManager) Write(_ context.Context, key string, value []byte) error { + r.mu.Lock() + defer r.mu.Unlock() + if r.objects == nil { + r.objects = make(map[string][]byte) + } + r.objects[key] = append([]byte(nil), value...) + return nil +} + func (r *recordingChunkManager) RemoveWithPrefix(ctx context.Context, prefix string) error { if r.prefixBlockUntil != nil { select { @@ -1858,6 +1875,13 @@ func (r *recordingChunkManager) RemoveWithPrefix(ctx context.Context, prefix str } r.mu.Lock() r.prefixCalls = append(r.prefixCalls, prefix) + if r.prefixErr == nil { + for key := range r.objects { + if strings.HasPrefix(key, prefix) { + delete(r.objects, key) + } + } + } r.mu.Unlock() return r.prefixErr } @@ -1865,6 +1889,9 @@ func (r *recordingChunkManager) RemoveWithPrefix(ctx context.Context, prefix str func (r *recordingChunkManager) Remove(ctx context.Context, key string) error { r.mu.Lock() r.removeCalls = append(r.removeCalls, key) + if r.removeErr == nil { + delete(r.objects, key) + } r.mu.Unlock() return r.removeErr } @@ -1895,6 +1922,50 @@ func TestCleanupExploreTempForJob_Success(t *testing.T) { assert.Equal(t, []string{"__explore_temp__/coord_42"}, removes) } +func TestCleanupExploreTempForJob_PreservesRemoteProducerNamespace(t *testing.T) { + ctx := context.Background() + cm := &recordingChunkManager{rootPath: "files"} + mgr := newManagerWithChunkManager(t, cm) + // ExploreFilesReturnManifestPath writes baseDir directly through loon's + // bucket-root filesystem. Use that producer's path builder, with a nonempty + // configured root, to verify cleanup deletes the objects actually written. + manifestKey := path.Join(exploreTempDirForAttempt(42, 1), "_metadata/manifest-1.avro") + otherJobKey := path.Join(exploreTempDirForAttempt(420, 1), "_metadata/manifest-1.avro") + prefixedKey := path.Join(cm.RootPath(), manifestKey) + for _, key := range []string{manifestKey, otherJobKey, prefixedKey} { + require.NoError(t, cm.Write(ctx, key, []byte("manifest"))) + } + + mgr.cleanupExploreTempForJob(42) + + prefixes, removes := cm.snapshot() + assert.Equal(t, []string{"__explore_temp__/coord_42/"}, prefixes) + assert.Equal(t, []string{"__explore_temp__/coord_42"}, removes) + assert.NotContains(t, cm.objects, manifestKey) + assert.Contains(t, cm.objects, otherJobKey) + assert.Contains(t, cm.objects, prefixedKey) +} + +func TestCleanupExploreTempForJob_LocalProducerNamespace(t *testing.T) { + ctx := context.Background() + root := t.TempDir() + cm := storage.NewLocalChunkManager(objectstorage.RootPath(root)) + mgr := newManagerWithChunkManager(t, cm) + manifestPath := path.Join(root, exploreTempDirForAttempt(42, 1), "_metadata/manifest-1.avro") + sentinelPath := path.Join(root, exploreTempDirForAttempt(420, 1), "_metadata/manifest-1.avro") + require.NoError(t, cm.Write(ctx, manifestPath, []byte("manifest"))) + require.NoError(t, cm.Write(ctx, sentinelPath, []byte("keep"))) + + mgr.cleanupExploreTempForJob(42) + + exists, err := cm.Exist(ctx, manifestPath) + require.NoError(t, err) + assert.False(t, exists) + sentinel, err := cm.Read(ctx, sentinelPath) + require.NoError(t, err) + assert.Equal(t, []byte("keep"), sentinel) +} + func TestCleanupExploreTempForJob_NilChunkManager(t *testing.T) { mgr := newManagerWithChunkManager(t, nil) diff --git a/internal/datacoord/garbage_collector.go b/internal/datacoord/garbage_collector.go index 0a9008e65f..12aa86887d 100644 --- a/internal/datacoord/garbage_collector.go +++ b/internal/datacoord/garbage_collector.go @@ -20,6 +20,7 @@ import ( "context" "fmt" "path" + "path/filepath" "strconv" "strings" "sync" @@ -611,6 +612,32 @@ func (gc *garbageCollector) recycleUnusedBinlogFiles(ctx context.Context) { segmentIDFromPath: storage.ParseSegmentIDByBinlog, label: metrics.StatFileLabel, }, + { + prefix: path.Join(gc.option.cli.RootPath(), common.SegmentBm25LogPath), + checker: func(objectInfo *storage.ChunkObjectInfo, segment *SegmentInfo) bool { + if segment == nil { + return false + } + for _, fieldLogs := range segment.GetBm25Statslogs() { + for _, statsLog := range fieldLogs.GetBinlogs() { + key := statsLog.GetLogPath() + if key == "" { + // Catalog binlogs carry IDs, not paths. Match the field too: + // compound stats reuse the same log ID across fields. + key = metautil.BuildBm25LogPath(gc.option.cli.RootPath(), + segment.GetCollectionID(), segment.GetPartitionID(), segment.GetID(), + fieldLogs.GetFieldID(), statsLog.GetLogID()) + } + if key == objectInfo.FilePath { + return true + } + } + } + return false + }, + segmentIDFromPath: storage.ParseSegmentIDByBinlog, + label: common.SegmentBm25LogPath, + }, { prefix: path.Join(gc.option.cli.RootPath(), common.SegmentDeltaLogPath), checker: func(objectInfo *storage.ChunkObjectInfo, segment *SegmentInfo) bool { @@ -1159,6 +1186,32 @@ func (gc *garbageCollector) getAllSegmentIndexesForDroppedSegment(segmentID int6 return gc.meta.indexMeta.GetAllSegmentIndexes(segmentID) } +// v3SegmentGCPrefix accepts canonical and legacy namespaces beneath the root, +// but only for this segment. Remote keys retain their literal spelling. +func (gc *garbageCollector) v3SegmentGCPrefix(basePath string, segment *SegmentInfo) (string, error) { + root := path.Clean(gc.option.cli.RootPath()) + _, local := gc.option.cli.(*storage.LocalChunkManager) + if path.IsAbs(basePath) != local { + return "", merr.WrapErrDataIntegrityMsg("GC V3 segment %d has an invalid storage path %q", segment.GetID(), basePath) + } + basePath = strings.TrimRight(basePath, "/") + if local { + basePath = path.Clean(basePath) + rel, err := filepath.Rel(root, basePath) + if err != nil || rel == "." || !filepath.IsLocal(rel) { + return "", merr.WrapErrDataIntegrityMsg("GC V3 segment %d path %q is outside storage root %q", segment.GetID(), basePath, root) + } + } else if root != "." && (basePath == root || !strings.HasPrefix(basePath, root+"/")) { + return "", merr.WrapErrDataIntegrityMsg("GC V3 segment %d path %q is outside storage root %q", segment.GetID(), basePath, root) + } + + if !segmentBaseMatches(basePath, segment.GetCollectionID(), segment.GetPartitionID(), segment.GetID()) { + return "", merr.WrapErrDataIntegrityMsg("GC V3 segment %d manifest base path does not match segment identity", segment.GetID()) + } + // Keep segment 2001 from matching sibling 20010 during prefix deletion. + return basePath + "/", nil +} + func (gc *garbageCollector) removeDroppedSegmentFiles(ctx context.Context, cloned *SegmentInfo, indexFiles map[string]struct{}) error { log := mlog.With(mlog.Int64("segmentID", cloned.GetID())) @@ -1176,7 +1229,11 @@ func (gc *garbageCollector) removeDroppedSegmentFiles(ctx context.Context, clone mlog.Err(err)) return err } - log.Info(ctx, "GC V3 segment start...", + basePath, err = gc.v3SegmentGCPrefix(basePath, cloned) + if err != nil { + return err + } + log.Info(ctx, "GC V3 segment start, removing basePath...", mlog.String("basePath", basePath), mlog.Int("indexFiles", len(indexFiles))) if err := gc.removeObjectFiles(ctx, indexFiles); err != nil { diff --git a/internal/datacoord/garbage_collector_bm25_test.go b/internal/datacoord/garbage_collector_bm25_test.go new file mode 100644 index 0000000000..69d9043c34 --- /dev/null +++ b/internal/datacoord/garbage_collector_bm25_test.go @@ -0,0 +1,163 @@ +// Licensed to the LF AI & Data foundation under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package datacoord + +import ( + "context" + "os" + "path" + "strings" + "testing" + "time" + + "github.com/bytedance/mockey" + "github.com/cockroachdb/errors" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + + "github.com/milvus-io/milvus-proto/go-api/v3/commonpb" + "github.com/milvus-io/milvus/internal/mocks" + "github.com/milvus-io/milvus/internal/storage" + "github.com/milvus-io/milvus/pkg/v3/objectstorage" + "github.com/milvus-io/milvus/pkg/v3/proto/datapb" + "github.com/milvus-io/milvus/pkg/v3/util/conc" +) + +func TestGarbageCollectorBM25OrphanFiles(t *testing.T) { + for _, local := range []bool{true, false} { + name := "remote complete keys" + if local { + name = "local filesystem" + } + t.Run(name, func(t *testing.T) { + ctx := context.Background() + root := "files" + if local { + root = t.TempDir() + } + key := func(suffix string) string { return path.Join(root, "bm25_stats", suffix) } + cases := []struct { + suffix string + keep bool + fresh bool + }{ + {"100/10/200/101/1000", true, false}, // compressed metadata + {"100/10/200/101/1", true, false}, // compound stats + {"100/10/200/102/1", false, false}, // same LogID, different field + {"100/10/200/101/999", false, false}, // superseded stats + {"100/10/200/101/777", true, false}, // explicit LogPath overrides LogID + {"100/10/200/101/1002", false, false}, + {"101/10/200/101/1000", false, false}, // same segment/log IDs, wrong collection + {"100/11/200/101/1000", false, false}, // same segment/log IDs, wrong partition + {"100/10/201/101/1000", false, false}, // absent segment + {"100/10/202/101/1000", true, true}, // fresh orphan + {"100/10/205/101/1000", true, false}, // snapshot-protected absent segment + {"100/10/300/101/999", true, false}, // registered V3 skipped + } + segment := &SegmentInfo{SegmentInfo: &datapb.SegmentInfo{ + ID: 200, CollectionID: 100, PartitionID: 10, + State: commonpb.SegmentState_Flushed, + Bm25Statslogs: []*datapb.FieldBinlog{{FieldID: 101, Binlogs: []*datapb.Binlog{ + {LogID: 1000}, {LogID: 1}, {LogID: 1002, LogPath: key("100/10/200/101/777")}, + }}}, + }} + mt := &meta{segments: &SegmentsInfo{segments: map[int64]*SegmentInfo{ + 200: segment, + 300: {SegmentInfo: &datapb.SegmentInfo{ID: 300, CollectionID: 100, PartitionID: 10, StorageVersion: storage.StorageV3}}, + }}, snapshotMeta: &snapshotMeta{}} + blocked := mockey.Mock((*snapshotMeta).IsSegmentGCBlocked).To( + func(_ *snapshotMeta, _ int64, segmentID int64) bool { return segmentID == 205 }, + ).Build() + defer blocked.UnPatch() + pool := conc.NewPool[struct{}](1) + t.Cleanup(pool.Release) + gc := &garbageCollector{meta: mt, option: GcOption{missingTolerance: time.Hour, removeObjectPool: pool}} + old := time.Now().Add(-2 * time.Hour) + if local { + cm := storage.NewLocalChunkManager(objectstorage.RootPath(root)) + gc.option.cli = cm + for _, tc := range cases { + file := key(tc.suffix) + require.NoError(t, cm.Write(ctx, file, []byte(tc.suffix))) + if !tc.fresh { + require.NoError(t, os.Chtimes(file, old, old)) + } + } + gc.recycleUnusedBinlogFiles(ctx) + for _, tc := range cases { + exists, err := cm.Exist(ctx, key(tc.suffix)) + require.NoError(t, err) + assert.Equal(t, tc.keep, exists, tc.suffix) + } + // GC must not decompress or otherwise rewrite authoritative metadata. + assert.Empty(t, segment.GetBm25Statslogs()[0].GetBinlogs()[0].GetLogPath()) + return + } + + cm := mocks.NewChunkManager(t) + gc.option.cli = cm + cm.EXPECT().RootPath().Return(root) + files := make(map[string]*storage.ChunkObjectInfo) + for _, tc := range cases { + modified := old + if tc.fresh { + modified = time.Now() + } + files[key(tc.suffix)] = &storage.ChunkObjectInfo{FilePath: key(tc.suffix), ModifyTime: modified} + } + cm.EXPECT().WalkWithPrefix(mock.Anything, mock.Anything, true, mock.Anything).RunAndReturn( + func(_ context.Context, prefix string, _ bool, walk storage.ChunkObjectWalkFunc) error { + // Snapshot the listing: the deletion worker may run concurrently. + listed := make([]*storage.ChunkObjectInfo, 0) + for file, info := range files { + if strings.HasPrefix(file, strings.TrimSuffix(prefix, "/")+"/") { + listed = append(listed, info) + } + } + for _, info := range listed { + if !walk(info) { + break + } + } + return nil + }) + retryFile := key("100/10/201/101/1000") + cm.EXPECT().Remove(mock.Anything, retryFile).Return(errors.New("temporary delete failure")).Once() + for _, tc := range cases { + if tc.keep || key(tc.suffix) == retryFile { + continue + } + cm.EXPECT().Remove(mock.Anything, key(tc.suffix)).RunAndReturn(func(_ context.Context, file string) error { + delete(files, file) + return nil + }).Once() + } + gc.recycleUnusedBinlogFiles(ctx) + require.Contains(t, files, retryFile) + cm.EXPECT().Remove(mock.Anything, retryFile).RunAndReturn(func(_ context.Context, file string) error { + delete(files, file) + return nil + }).Once() + gc.recycleUnusedBinlogFiles(ctx) + for _, tc := range cases { + _, exists := files[key(tc.suffix)] + assert.Equal(t, tc.keep, exists, tc.suffix) + } + }) + } +} diff --git a/internal/datacoord/garbage_collector_lob.go b/internal/datacoord/garbage_collector_lob.go index fb40ca1714..b03ba716c2 100644 --- a/internal/datacoord/garbage_collector_lob.go +++ b/internal/datacoord/garbage_collector_lob.go @@ -33,7 +33,6 @@ import ( "github.com/milvus-io/milvus/pkg/v3/proto/indexpb" "github.com/milvus-io/milvus/pkg/v3/util/conc" "github.com/milvus-io/milvus/pkg/v3/util/merr" - "github.com/milvus-io/milvus/pkg/v3/util/paramtable" "github.com/milvus-io/milvus/pkg/v3/util/typeutil" ) @@ -154,14 +153,10 @@ func (gc *garbageCollector) recycleUnusedLOBFiles(ctx context.Context) { logger.Info(ctx, "recycleUnusedLOBFiles done", mlog.Duration("timeCost", time.Since(start))) }() - storageConfig := gc.getStorageConfig() - if storageConfig == nil { - logger.Warn(ctx, "failed to get storage config, skip LOB GC") - return - } - lobCtx := newLOBGCContext(gc) - lobCtx.storageConfig = storageConfig + // Same builder as every other primary-storage caller: under local storage + // the key prefix is localStorage.path, not minio.rootPath. + lobCtx.storageConfig = createStorageConfig() // Step 1: Collect all used LOB files from active segments. // If any manifest read fails, abort the entire GC run to avoid @@ -187,30 +182,6 @@ func (gc *garbageCollector) recycleUnusedLOBFiles(ctx context.Context) { lobCtx.cache.Cleanup() } -// getStorageConfig returns the storage configuration for FFI calls -func (gc *garbageCollector) getStorageConfig() *indexpb.StorageConfig { - params := paramtable.Get() - - return &indexpb.StorageConfig{ - Address: params.MinioCfg.Address.GetValue(), - AccessKeyID: params.MinioCfg.AccessKeyID.GetValue(), - SecretAccessKey: params.MinioCfg.SecretAccessKey.GetValue(), - UseSSL: params.MinioCfg.UseSSL.GetAsBool(), - SslCACert: params.MinioCfg.SslCACert.GetValue(), - BucketName: params.MinioCfg.BucketName.GetValue(), - RootPath: params.MinioCfg.RootPath.GetValue(), - UseIAM: params.MinioCfg.UseIAM.GetAsBool(), - IAMEndpoint: params.MinioCfg.IAMEndpoint.GetValue(), - StorageType: params.CommonCfg.StorageType.GetValue(), - Region: params.MinioCfg.Region.GetValue(), - UseVirtualHost: params.MinioCfg.UseVirtualHost.GetAsBool(), - CloudProvider: params.MinioCfg.CloudProvider.GetValue(), - RequestTimeoutMs: params.MinioCfg.RequestTimeoutMs.GetAsInt64(), - MaxConnections: uint32(params.MinioCfg.MaxConnections.GetAsInt()), - GcpCredentialJSON: params.MinioCfg.GcpCredentialJSON.GetValue(), - } -} - // collectUsedLOBFiles collects all LOB file paths that are referenced by active segments // and by dropped segments that are protected by snapshots. // Returns a set of LOB file paths (relative paths within LOB directory). @@ -290,6 +261,10 @@ func (lobCtx *lobGCContext) scanOrphanLOBFiles(ctx context.Context, usedFiles ty safetyWindow := Params.DataCoordCfg.GCLOBSafetyWindow.GetAsDuration(time.Second) // LOB files are stored at: {root_path}/insert_log/{coll}/{part}/lobs/{field_id}/_data/{file_id}.vx + // TODO: Cover the legacy local namespace when upgrading from 3.0.0/3.0.1 + // with LOB enabled. Files under localStorage.path/minio.rootPath/insert_log + // remain readable in place, but this scan misses their partition-level LOBs, + // so unreferenced LOB files there are not reclaimed. lobBasePath := path.Join(lobCtx.gc.option.cli.RootPath(), common.SegmentInsertLogPath) // Walk through all files under insert_log to find LOB files diff --git a/internal/datacoord/garbage_collector_lob_test.go b/internal/datacoord/garbage_collector_lob_test.go index 26d3085aca..2dca3679c9 100644 --- a/internal/datacoord/garbage_collector_lob_test.go +++ b/internal/datacoord/garbage_collector_lob_test.go @@ -26,7 +26,6 @@ import ( "github.com/milvus-io/milvus/internal/storagev2/packed" "github.com/milvus-io/milvus/pkg/v3/proto/datapb" - "github.com/milvus-io/milvus/pkg/v3/proto/indexpb" "github.com/milvus-io/milvus/pkg/v3/util/typeutil" ) @@ -271,15 +270,30 @@ func TestNewLOBGCContext(t *testing.T) { assert.Equal(t, gc, lobCtx.gc) } -func TestGetStorageConfig(t *testing.T) { - // this test verifies the getStorageConfig function returns a valid config - // based on paramtable values - gc := &garbageCollector{} - config := gc.getStorageConfig() +// LOB GC reads manifests through the primary storage config, so its key prefix +// must be localStorage.path under local storage. Deriving it from minio.rootPath +// would address a namespace that holds no LOB files (#53051). +func TestLOBGCStorageConfigUsesPrimaryStorageRoot(t *testing.T) { + params := Params + localRoot := t.TempDir() + require.NoError(t, params.Save(params.MinioCfg.RootPath.Key, "files")) + require.NoError(t, params.Save(params.LocalStorageCfg.Path.Key, localRoot)) + t.Cleanup(func() { + _ = params.Reset(params.CommonCfg.StorageType.Key) + _ = params.Reset(params.MinioCfg.RootPath.Key) + _ = params.Reset(params.LocalStorageCfg.Path.Key) + }) + require.NoError(t, params.Save(params.CommonCfg.StorageType.Key, "local")) + config := createStorageConfig() require.NotNil(t, config) - // verify it's a valid StorageConfig struct - assert.IsType(t, &indexpb.StorageConfig{}, config) + assert.Equal(t, "local", config.GetStorageType()) + assert.Equal(t, localRoot, config.GetRootPath()) + + require.NoError(t, params.Save(params.CommonCfg.StorageType.Key, "remote")) + config = createStorageConfig() + require.NotNil(t, config) + assert.Equal(t, "files", config.GetRootPath()) } func TestCollectLOBFilesFromSegment(t *testing.T) { diff --git a/internal/datacoord/garbage_collector_test.go b/internal/datacoord/garbage_collector_test.go index e0678bf5d5..220da5ea87 100644 --- a/internal/datacoord/garbage_collector_test.go +++ b/internal/datacoord/garbage_collector_test.go @@ -23,6 +23,7 @@ import ( "math/rand" "os" "path" + "path/filepath" "strconv" "strings" "sync" @@ -3089,7 +3090,7 @@ func TestGarbageCollector_getDroppedSegmentIndexFiles_ManifestUnreadable(t *test func TestGarbageCollector_removeDroppedSegmentFilesV3(t *testing.T) { ctx := context.Background() - basePath := "/tmp/test-gc-v3/insert_log/100/10/2001" + basePath := "root/insert_log/100/10/2001" segment := &SegmentInfo{ SegmentInfo: &datapb.SegmentInfo{ ID: 2001, @@ -3103,6 +3104,7 @@ func TestGarbageCollector_removeDroppedSegmentFilesV3(t *testing.T) { t.Run("success with index files", func(t *testing.T) { cm := mocks.NewChunkManager(t) + cm.EXPECT().RootPath().Return("root").Maybe() cm.EXPECT().RemoveWithPrefix(mock.Anything, basePath+"/").Return(nil).Once() cm.EXPECT().Remove(mock.Anything, "root/index_files/40/1/10/2001/idx-file").Return(nil).Once() gc := newGarbageCollector(nil, nil, GcOption{cli: cm}) @@ -3124,14 +3126,25 @@ func TestGarbageCollector_removeDroppedSegmentFilesV3(t *testing.T) { t.Run("remove base path failed", func(t *testing.T) { cm := mocks.NewChunkManager(t) + cm.EXPECT().RootPath().Return("root").Maybe() cm.EXPECT().RemoveWithPrefix(mock.Anything, basePath+"/").Return(errors.New("remove failed")).Once() gc := newGarbageCollector(nil, nil, GcOption{cli: cm}) assert.Error(t, gc.removeDroppedSegmentFiles(ctx, segment, nil)) }) + t.Run("remote not found semantics unchanged", func(t *testing.T) { + cm := mocks.NewChunkManager(t) + cm.EXPECT().RootPath().Return("root").Maybe() + cm.EXPECT().RemoveWithPrefix(mock.Anything, basePath+"/").Return(os.ErrNotExist).Once() + gc := newGarbageCollector(nil, nil, GcOption{cli: cm}) + + assert.ErrorIs(t, gc.removeDroppedSegmentFiles(ctx, segment, nil), os.ErrNotExist) + }) + t.Run("remove index file failed", func(t *testing.T) { cm := mocks.NewChunkManager(t) + cm.EXPECT().RootPath().Return("root").Maybe() // No RemoveWithPrefix expectation: a failed index-file delete must // leave basePath - and the manifest inside it, the only thing naming // the surviving index files - untouched for the next cycle. @@ -3151,6 +3164,7 @@ func TestGarbageCollector_removeDroppedSegmentFilesV3(t *testing.T) { var mu sync.Mutex var calls []string cm := mocks.NewChunkManager(t) + cm.EXPECT().RootPath().Return("root").Maybe() cm.EXPECT().Remove(mock.Anything, mock.Anything).RunAndReturn(func(_ context.Context, file string) error { mu.Lock() defer mu.Unlock() @@ -4408,7 +4422,7 @@ func Test_parseV3SegmentID(t *testing.T) { func TestGarbageCollector_recycleDroppedSegments_V3(t *testing.T) { ctx := context.Background() - cli := storage.NewLocalChunkManager(objectstorage.RootPath("/tmp/test-gc-v3")) + cli := storage.NewLocalChunkManager(objectstorage.RootPath(t.TempDir())) catalog := &datacoord.Catalog{} smMeta := &snapshotMeta{} @@ -4422,7 +4436,9 @@ func TestGarbageCollector_recycleDroppedSegments_V3(t *testing.T) { channelCPs: newChannelCps(), } - basePath := "/tmp/test-gc-v3/insert_log/100/10/2001" + basePath := path.Join(cli.RootPath(), "insert_log/100/10/2001") + require.NoError(t, os.MkdirAll(basePath, 0o755)) + completeBasePath := basePath + "/" manifestPath := packed.MarshalManifestPath(basePath, 1) // V3 dropped segment with ManifestPath and StorageVersion=3 @@ -4509,7 +4525,7 @@ func TestGarbageCollector_recycleDroppedSegments_V3(t *testing.T) { // V3 segment should use RemoveWithPrefix with basePath assert.True(t, removeWithPrefixCalled, "V3 segment should use RemoveWithPrefix") - assert.Equal(t, basePath+"/", removeWithPrefixArg, "RemoveWithPrefix should name the segment directory") + assert.Equal(t, completeBasePath, removeWithPrefixArg, "RemoveWithPrefix should be called with a complete local path") // V1 segment should use removeObjectFiles assert.True(t, removeObjectFilesCalled, "V1 segment should use removeObjectFiles") @@ -4521,6 +4537,304 @@ func TestGarbageCollector_recycleDroppedSegments_V3(t *testing.T) { assert.Nil(t, m.GetSegment(ctx, 2002)) } +func TestGarbageCollectorRemoveDroppedSegmentFilesRejectsUnsafeLocalManifestRoot(t *testing.T) { + ctx := context.Background() + root := t.TempDir() + cli := storage.NewLocalChunkManager(objectstorage.RootPath(root)) + gc := &garbageCollector{option: GcOption{cli: cli}} + sentinelPath := path.Join(root, "sentinel") + require.NoError(t, cli.Write(ctx, sentinelPath, []byte("keep"))) + + unsafeBasePaths := []string{ + "", + ".", + "/", + "../outside", + root, + "s3://bucket/files/insert_log/100/10/2001", + "insert_log", + "insert_log/100/10/20010", + } + for _, basePath := range unsafeBasePaths { + t.Run(fmt.Sprintf("base_%q", basePath), func(t *testing.T) { + segment := &SegmentInfo{SegmentInfo: &datapb.SegmentInfo{ + ID: 2001, + StorageVersion: storage.StorageV3, + ManifestPath: packed.MarshalManifestPath(basePath, 1), + }} + err := gc.removeDroppedSegmentFiles(ctx, segment, nil) + require.Error(t, err) + content, readErr := cli.Read(ctx, sentinelPath) + require.NoError(t, readErr) + assert.Equal(t, []byte("keep"), content) + }) + } +} + +func TestGarbageCollectorRemoveDroppedSegmentFilesUsesSegmentBoundary(t *testing.T) { + ctx := context.Background() + root := t.TempDir() + cli := storage.NewLocalChunkManager(objectstorage.RootPath(root)) + gc := &garbageCollector{option: GcOption{cli: cli}} + segmentFile := path.Join(root, "files/insert_log/100/10/2001/data") + siblingFile := path.Join(root, "files/insert_log/100/10/20010/data") + require.NoError(t, cli.Write(ctx, segmentFile, []byte("remove"))) + require.NoError(t, cli.Write(ctx, siblingFile, []byte("keep"))) + + segment := &SegmentInfo{SegmentInfo: &datapb.SegmentInfo{ + ID: 2001, + CollectionID: 100, + PartitionID: 10, + StorageVersion: storage.StorageV3, + ManifestPath: packed.MarshalManifestPath(path.Join(root, "files/insert_log/100/10/2001"), 1), + }} + require.NoError(t, gc.removeDroppedSegmentFiles(ctx, segment, nil)) + + segmentExists, err := cli.Exist(ctx, segmentFile) + require.NoError(t, err) + assert.False(t, segmentExists) + siblingContent, err := cli.Read(ctx, siblingFile) + require.NoError(t, err) + assert.Equal(t, []byte("keep"), siblingContent) +} + +func TestGarbageCollectorRemoveDroppedSegmentFilesRejectsForeignLocalRoot(t *testing.T) { + ctx := context.Background() + testRoot := t.TempDir() + root := path.Join(testRoot, "owned") + cli := storage.NewLocalChunkManager(objectstorage.RootPath(root)) + gc := &garbageCollector{option: GcOption{cli: cli}} + for _, otherRoot := range []string{path.Join(testRoot, "other"), root + "-other"} { + t.Run(path.Base(otherRoot), func(t *testing.T) { + basePath := path.Join(otherRoot, "insert_log/100/10/2001") + sentinelPath := path.Join(basePath, "data") + require.NoError(t, cli.Write(ctx, sentinelPath, []byte("keep foreign data"))) + segment := &SegmentInfo{SegmentInfo: &datapb.SegmentInfo{ + ID: 2001, CollectionID: 100, PartitionID: 10, + StorageVersion: storage.StorageV3, + ManifestPath: packed.MarshalManifestPath(basePath, 1), + }} + require.ErrorIs(t, gc.removeDroppedSegmentFiles(ctx, segment, nil), merr.ErrDataIntegrity) + content, err := cli.Read(ctx, sentinelPath) + require.NoError(t, err) + assert.Equal(t, "keep foreign data", string(content)) + }) + } +} + +func TestGarbageCollectorRemoveDroppedSegmentFilesLocalFilesystemRoot(t *testing.T) { + ctx := context.Background() + // Even with a filesystem-wide configured root, only test-owned files are + // created and deleted; the manifest's segment suffix remains mandatory. + testRoot, err := filepath.EvalSymlinks(t.TempDir()) + require.NoError(t, err) + basePath := path.Join(testRoot, "insert_log/100/10/2001") + cli := storage.NewLocalChunkManager(objectstorage.RootPath("/")) + gc := &garbageCollector{option: GcOption{cli: cli}} + filePath := path.Join(basePath, "data") + require.NoError(t, cli.Write(ctx, filePath, []byte("remove"))) + segment := &SegmentInfo{SegmentInfo: &datapb.SegmentInfo{ + ID: 2001, CollectionID: 100, PartitionID: 10, + StorageVersion: storage.StorageV3, + ManifestPath: packed.MarshalManifestPath(basePath, 1), + }} + require.NoError(t, gc.removeDroppedSegmentFiles(ctx, segment, nil)) + assert.NoFileExists(t, filePath) +} + +func TestGarbageCollectorRemoveDroppedSegmentFilesAllowsMissingLocalSuffix(t *testing.T) { + for _, existingSuffix := range []string{"", "insert_log/100/10"} { + t.Run(fmt.Sprintf("existing_%q", existingSuffix), func(t *testing.T) { + ctx := context.Background() + root := t.TempDir() + cli := storage.NewLocalChunkManager(objectstorage.RootPath(root)) + gc := newGarbageCollector(nil, nil, GcOption{cli: cli}) + require.NoError(t, os.MkdirAll(path.Join(root, existingSuffix), 0o755)) + basePath := path.Join(root, "insert_log/100/10/2001") + segment := &SegmentInfo{SegmentInfo: &datapb.SegmentInfo{ + ID: 2001, CollectionID: 100, PartitionID: 10, + StorageVersion: storage.StorageV3, + ManifestPath: packed.MarshalManifestPath(basePath, 1), + }} + indexFile := path.Join(root, "index_files/40/1/10/2001/idx-file") + require.NoError(t, cli.Write(ctx, indexFile, []byte("remove index"))) + segment.ID = 20010 + require.ErrorIs(t, gc.removeDroppedSegmentFiles(ctx, segment, map[string]struct{}{indexFile: {}}), merr.ErrDataIntegrity) + require.FileExists(t, indexFile, "a missing base must not bypass segment identity validation") + segment.ID = 2001 + for range 2 { + // Exercise the real local prefix walker: a missing base returns + // ENOENT unless GC recognizes that data removal is already done. + require.NoError(t, gc.removeDroppedSegmentFiles(ctx, segment, map[string]struct{}{indexFile: {}})) + assert.NoFileExists(t, indexFile) + assert.NoDirExists(t, basePath) + } + }) + } +} + +func TestGarbageCollector_recycleDroppedSegment_MissingLocalV3DataStillRemovesMeta(t *testing.T) { + ctx := context.Background() + m, segment, _, _ := setupDroppedSegmentWithIndexForGC(t) + root := t.TempDir() + cli := storage.NewLocalChunkManager(objectstorage.RootPath(root)) + basePath := path.Join(root, "files/insert_log/100/10/1001") + segment.StorageVersion = storage.StorageV3 + segment.ManifestPath = packed.MarshalManifestPath(basePath, 1) + gc := newGarbageCollector(m, newMockHandler(), GcOption{cli: cli}) + _, indexFiles, blocked := gc.getDroppedSegmentIndexFiles(ctx, segment.ID) + require.Equal(t, gcNotBlocked, blocked) + require.NotEmpty(t, indexFiles) + for file := range indexFiles { + require.NoError(t, cli.Write(ctx, file, []byte("remaining index"))) + } + sibling := path.Join(root, "files/insert_log/100/10/10010/data") + require.NoError(t, cli.Write(ctx, sibling, []byte("keep sibling"))) + require.NoDirExists(t, basePath) + + gc.recycleDroppedSegment(ctx, segment.ID, segment) + + assert.Nil(t, m.GetSegment(ctx, segment.ID)) + assert.Empty(t, m.indexMeta.GetAllSegmentIndexes(segment.ID)) + for file := range indexFiles { + assert.NoFileExists(t, file) + } + contents, err := cli.Read(ctx, sibling) + require.NoError(t, err) + assert.Equal(t, "keep sibling", string(contents)) +} + +func TestGarbageCollectorRemoveDroppedSegmentFilesRejectsInvalidConfiguredRoot(t *testing.T) { + segment := &SegmentInfo{SegmentInfo: &datapb.SegmentInfo{ + ID: 2001, CollectionID: 100, PartitionID: 10, + StorageVersion: storage.StorageV3, + }} + t.Run("relative local root", func(t *testing.T) { + cli := storage.NewLocalChunkManager(objectstorage.RootPath("data")) + gc := &garbageCollector{option: GcOption{cli: cli}} + segment.ManifestPath = packed.MarshalManifestPath(path.Join(t.TempDir(), "insert_log/100/10/2001"), 1) + require.ErrorIs(t, gc.removeDroppedSegmentFiles(context.Background(), segment, nil), merr.ErrDataIntegrity) + }) + for _, root := range []string{"/", "/files", "s3://bucket/files"} { + t.Run(fmt.Sprintf("remote_root_%q", root), func(t *testing.T) { + cli := &gcRootedChunkManager{root: root} + gc := &garbageCollector{option: GcOption{cli: cli}} + segment.ManifestPath = packed.MarshalManifestPath("files/insert_log/100/10/2001", 1) + require.ErrorIs(t, gc.removeDroppedSegmentFiles(context.Background(), segment, nil), merr.ErrDataIntegrity) + assert.False(t, cli.removeCalled) + }) + } +} + +type gcRootedChunkManager struct { + storage.ChunkManager + root string + removeCalled bool + removedPrefix string +} + +func (cm *gcRootedChunkManager) RootPath() string { + return cm.root +} + +func (cm *gcRootedChunkManager) RemoveWithPrefix(_ context.Context, prefix string) error { + cm.removeCalled = true + cm.removedPrefix = prefix + return nil +} + +// A manifest must identify both the configured namespace and this segment; +// matching IDs alone do not authorize deletion in another instance's prefix. +func TestGarbageCollectorRemoveDroppedSegmentFilesRejectsMalformedRemoteManifestBase(t *testing.T) { + unsafeBasePaths := []string{ + "files/insert_log", + "files/insert_log/100/10/20010", + "/files/insert_log/100/10/2001", + "s3://bucket/files/insert_log/100/10/2001", + "../insert_log/100/10/2001", + "other/insert_log/100/10/2001", + "files-other/insert_log/100/10/2001", + "insert_log/100/10/2001", + } + for _, basePath := range unsafeBasePaths { + t.Run(fmt.Sprintf("base_%q", basePath), func(t *testing.T) { + cli := &gcRootedChunkManager{root: "files"} + gc := &garbageCollector{option: GcOption{cli: cli}} + segment := &SegmentInfo{SegmentInfo: &datapb.SegmentInfo{ + ID: 2001, + CollectionID: 100, + PartitionID: 10, + StorageVersion: storage.StorageV3, + ManifestPath: packed.MarshalManifestPath(basePath, 1), + }} + err := gc.removeDroppedSegmentFiles(context.Background(), segment, nil) + require.ErrorIs(t, err, merr.ErrDataIntegrity) + assert.False(t, cli.removeCalled) + }) + } +} + +func TestGarbageCollectorRemoveDroppedSegmentFilesUsesCompleteRemoteKey(t *testing.T) { + for _, tc := range []struct { + name string + root string + base string + }{ + {"configured prefix", "files", "files/insert_log/100/10/2001"}, + {"nested prefix", "tenant/files", "tenant/files/insert_log/100/10/2001"}, + {"layout prefix within root", "files", "files/legacy/insert_log/100/10/2001"}, + {"bucket root complete key", "", "files/insert_log/100/10/2001"}, + {"dot bucket root complete key", ".", "files/insert_log/100/10/2001"}, + {"parent component is part of object key", "files", "files/tmp/../insert_log/100/10/2001"}, + {"dot component is part of object key", "files", "files/./insert_log/100/10/2001"}, + {"repeated separator is part of object key", "files", "files//insert_log/100/10/2001"}, + {"trailing directory separator", "files", "files/insert_log/100/10/2001/"}, + {"parent component at bucket root", "", "../insert_log/100/10/2001"}, + {"parent component within bucket", ".", "tmp/../insert_log/100/10/2001"}, + } { + t.Run(tc.name, func(t *testing.T) { + cli := &gcRootedChunkManager{root: tc.root} + gc := &garbageCollector{option: GcOption{cli: cli}} + segment := &SegmentInfo{SegmentInfo: &datapb.SegmentInfo{ + ID: 2001, + CollectionID: 100, + PartitionID: 10, + StorageVersion: storage.StorageV3, + ManifestPath: packed.MarshalManifestPath(tc.base, 1), + }} + + require.NoError(t, gc.removeDroppedSegmentFiles(context.Background(), segment, nil)) + assert.True(t, cli.removeCalled) + assert.Equal(t, strings.TrimRight(tc.base, "/")+"/", cli.removedPrefix) + }) + } +} + +func TestGarbageCollectorRemoveDroppedSegmentFilesRemoteBucketRoot(t *testing.T) { + for _, root := range []string{"", "."} { + t.Run(fmt.Sprintf("root_%q", root), func(t *testing.T) { + cli := &gcRootedChunkManager{root: root} + gc := &garbageCollector{option: GcOption{cli: cli}} + segment := &SegmentInfo{SegmentInfo: &datapb.SegmentInfo{ + ID: 2001, + CollectionID: 100, + PartitionID: 10, + StorageVersion: storage.StorageV3, + ManifestPath: packed.MarshalManifestPath(path.Join(root, "insert_log/100/10/2001"), 1), + }} + require.NoError(t, gc.removeDroppedSegmentFiles(context.Background(), segment, nil)) + assert.Equal(t, "insert_log/100/10/2001/", cli.removedPrefix) + + for _, unsafeBase := range []string{"", ".", "..", "insert_log/100/10/20010"} { + cli.removeCalled = false + segment.ManifestPath = packed.MarshalManifestPath(unsafeBase, 1) + require.Error(t, gc.removeDroppedSegmentFiles(context.Background(), segment, nil)) + assert.False(t, cli.removeCalled) + } + }) + } +} + func TestGarbageCollector_recycleUnusedBinlogFiles_SkipV3(t *testing.T) { ctx := context.Background() diff --git a/internal/datacoord/manifest_index.go b/internal/datacoord/manifest_index.go index 182bc23fa3..9f3811ce47 100644 --- a/internal/datacoord/manifest_index.go +++ b/internal/datacoord/manifest_index.go @@ -312,8 +312,12 @@ func (m *meta) reloadSegmentIndexesFromManifests(ctx context.Context) error { return nil, nil } } - return nil, merr.Wrapf(err, "recover segment %d indexes from manifest %s", - segment.GetID(), segment.GetManifestPath()) + // Do not let initMeta replay the entire metastore load after the + // per-segment retries are exhausted. A second newMeta attempt would + // repeat every catalog read and can expose a partially initialized + // state; the caller can retry the startup as a whole instead. + return nil, retry.Unrecoverable(merr.Wrapf(err, + "recover segment %d indexes from manifest %s", segment.GetID(), segment.GetManifestPath())) } for _, entry := range entries { if _, ok := manifestIndexFilePathInfoForSegment(rootPath, segment.SegmentInfo, entry); !ok { diff --git a/internal/datacoord/manifest_index_review_test.go b/internal/datacoord/manifest_index_review_test.go index 9ca76b9c8d..e0283328c0 100644 --- a/internal/datacoord/manifest_index_review_test.go +++ b/internal/datacoord/manifest_index_review_test.go @@ -104,12 +104,13 @@ func TestInitMetaDoesNotReplayManifestScan(t *testing.T) { require.NoError(t, m.AddSegment(ctx, NewSegmentInfo(&datapb.SegmentInfo{ CollectionID: 100, PartitionID: 10, ID: id, State: commonpb.SegmentState_Flushed, StorageVersion: storage.StorageV3, ManifestHasIndex: true, - ManifestPath: packed.MarshalManifestPath(metautil.JoinIDPath(id), 1), + ManifestPath: packed.MarshalManifestPath(path.Join("/tmp/test-restart", "insert_log", metautil.JoinIDPath(100, 10, id)), 1), }))) } var mu sync.Mutex reads := make(map[string]int) - bad := packed.MarshalManifestPath("5002", 1) + good := packed.MarshalManifestPath(path.Join("/tmp/test-restart", "insert_log", metautil.JoinIDPath(100, 10, 5001)), 1) + bad := packed.MarshalManifestPath(path.Join("/tmp/test-restart", "insert_log", metautil.JoinIDPath(100, 10, 5002)), 1) reader := mockey.Mock(packed.GetManifestIndexInfos).To(func(pointer string, _ *indexpb.StorageConfig) ([]packed.ManifestIndexInfo, error) { mu.Lock() defer mu.Unlock() @@ -133,7 +134,7 @@ func TestInitMetaDoesNotReplayManifestScan(t *testing.T) { b.EXPECT().ListDatabases(mock.Anything).Run(func(context.Context) { close(collectionsLoaded) }).Return(nil, nil).Maybe() server := &Server{ctx: ctx, kv: kv, broker: b} err := server.initMeta(cm) - require.Equal(t, 1, reads[packed.MarshalManifestPath("5001", 1)]) + require.Equal(t, 1, reads[good]) if mode == "transient" { require.NoError(t, err) <-collectionsLoaded diff --git a/internal/datacoord/meta.go b/internal/datacoord/meta.go index bf32f62a33..10f0386e7d 100644 --- a/internal/datacoord/meta.go +++ b/internal/datacoord/meta.go @@ -413,9 +413,23 @@ func (m *meta) reloadFromKV(ctx context.Context, collectionIDs []int64) error { metrics.DataCoordNumSegments.Reset() numStoredRows := int64(0) numSegments := 0 + local, localStorage := m.chunkManager.(*storage.LocalChunkManager) for _, segments := range collectionSegments { numSegments += len(segments) for _, segment := range segments { + if localStorage && segment.GetStorageVersion() == storage.StorageV3 { + manifestPath, err := normalizeLocalManifestPath(segment.GetManifestPath(), local.RootPath(), + Params.MinioCfg.RootPath.GetValue(), segment.GetCollectionID(), segment.GetPartitionID(), segment.GetID()) + if err != nil { + return err + } + if manifestPath != segment.GetManifestPath() { + // Resolve only the loaded view. Startup does not write the + // catalog; a later ordinary update can persist the new path. + segment = proto.Clone(segment).(*datapb.SegmentInfo) + segment.ManifestPath = manifestPath + } + } // segments from catalog.ListSegments will not have logPath m.segments.SetSegment(segment.ID, NewSegmentInfo(segment)) metrics.DataCoordNumSegments.WithLabelValues(segmentMetricLabelValues(NewSegmentInfo(segment))...).Inc() diff --git a/internal/datacoord/meta_local_manifest.go b/internal/datacoord/meta_local_manifest.go new file mode 100644 index 0000000000..d9e1b96122 --- /dev/null +++ b/internal/datacoord/meta_local_manifest.go @@ -0,0 +1,58 @@ +// Licensed to the LF AI & Data foundation under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package datacoord + +import ( + "path" + "path/filepath" + + "github.com/milvus-io/milvus/internal/storagev2/packed" + "github.com/milvus-io/milvus/pkg/v3/common" + "github.com/milvus-io/milvus/pkg/v3/util/merr" + "github.com/milvus-io/milvus/pkg/v3/util/metautil" +) + +// normalizeLocalManifestPath resolves legacy minio-root-relative manifests to +// root/legacyPrefix/insert_log//

/, preserving their existing location. +// Absolute paths are unchanged. This only updates the loaded view, without IO. +// Callers must restrict it to locally owned segments, not foreign snapshots. +func normalizeLocalManifestPath(manifestPath, root, legacyPrefix string, collectionID, partitionID, segmentID int64) (string, error) { + if manifestPath == "" { + return manifestPath, nil + } + base, version, err := packed.UnmarshalManifestPath(manifestPath) + if err != nil { + return "", merr.WrapErrDataIntegrity(err, "segment %d has an invalid local manifest path %q", segmentID, manifestPath) + } + if path.IsAbs(base) { + return manifestPath, nil + } + if !filepath.IsAbs(root) { + return "", merr.WrapErrParameterInvalidMsg("localStorage.path %q must be an absolute path", root) + } + // Validate the relative prefix once, before joining it to the storage root. + // Empty and dot prefixes both address the root directly. + if !filepath.IsLocal(filepath.FromSlash(path.Clean(legacyPrefix))) { + return "", merr.WrapErrDataIntegrityMsg("invalid legacy local manifest prefix %q", legacyPrefix) + } + expectedBase := path.Join(legacyPrefix, common.SegmentInsertLogPath, metautil.JoinIDPath(collectionID, partitionID, segmentID)) + if path.Clean(base) != expectedBase { + return "", merr.WrapErrDataIntegrityMsg("segment %d legacy local manifest base %q does not match its storage prefix and segment identity", segmentID, base) + } + newBase := filepath.Join(root, filepath.FromSlash(expectedBase)) + return packed.MarshalManifestPath(filepath.ToSlash(newBase), version), nil +} diff --git a/internal/datacoord/meta_local_manifest_integration_test.go b/internal/datacoord/meta_local_manifest_integration_test.go new file mode 100644 index 0000000000..6e5ebf42cc --- /dev/null +++ b/internal/datacoord/meta_local_manifest_integration_test.go @@ -0,0 +1,438 @@ +// Licensed to the LF AI & Data foundation under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package datacoord + +import ( + "context" + "io" + "io/fs" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/apache/arrow/go/v17/arrow" + "github.com/apache/arrow/go/v17/arrow/array" + "github.com/apache/arrow/go/v17/arrow/memory" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/proto" + + "github.com/milvus-io/milvus-proto/go-api/v3/commonpb" + "github.com/milvus-io/milvus-proto/go-api/v3/schemapb" + "github.com/milvus-io/milvus/internal/metastore" + catalogmocks "github.com/milvus-io/milvus/internal/metastore/mocks" + "github.com/milvus-io/milvus/internal/storage" + "github.com/milvus-io/milvus/internal/storage/localmigrate" + "github.com/milvus-io/milvus/internal/storagev2/packed" + "github.com/milvus-io/milvus/pkg/v3/objectstorage" + "github.com/milvus-io/milvus/pkg/v3/proto/datapb" + "github.com/milvus-io/milvus/pkg/v3/proto/indexpb" + "github.com/milvus-io/milvus/pkg/v3/util/conc" + "github.com/milvus-io/milvus/pkg/v3/util/merr" + "github.com/milvus-io/milvus/pkg/v3/util/paramtable" + "github.com/milvus-io/milvus/pkg/v3/util/typeutil" +) + +type legacyLocalPackedFixture struct { + root string + legacyRoot string + physicalManifest string + segment *datapb.SegmentInfo + cm *storage.LocalChunkManager + cfg *indexpb.StorageConfig + pks []int64 + texts []string +} + +// Write a real manifest at the physical location used by the old rooted +// filesystem. Its writer-generated internal references are serialized relative +// to the segment/partition, while the catalog retains the old relative base. +// Compatibility must resolve that base to this original location without copying. +// This does not fabricate a parquet, manifest, stats blob, or LOB marker file. +func newLegacyLocalPackedFixture(t *testing.T) *legacyLocalPackedFixture { + t.Helper() + return newLocalPackedFixture(t, false) +} + +// The displaced variant models complete keys passed to the old double-rooted +// filesystem. Both variants use real writers and unchanged relative manifest +// references; only the physical namespace and catalog base differ. +func newLocalPackedFixture(t *testing.T, displaced bool) *legacyLocalPackedFixture { + t.Helper() + setLocalManifestLegacyPrefix(t, "files") + root := t.TempDir() + params := paramtable.Get() + for _, setting := range []struct{ key, value, original string }{ + {params.CommonCfg.StorageType.Key, "local", params.CommonCfg.StorageType.GetValue()}, + {params.LocalStorageCfg.Path.Key, root, params.LocalStorageCfg.Path.GetValue()}, + } { + require.NoError(t, params.Save(setting.key, setting.value)) + t.Cleanup(func() { require.NoError(t, params.Save(setting.key, setting.original)) }) + } + cfg := &indexpb.StorageConfig{StorageType: "local", RootPath: root} + legacyRoot := filepath.Join(root, "files") + catalogBase := "files/insert_log/1/2/3" + if displaced { + legacyRoot = filepath.Join(root, strings.TrimPrefix(root, string(filepath.Separator))) + catalogBase = filepath.Join(root, "insert_log/1/2/3") + } + base := filepath.Join(legacyRoot, "insert_log/1/2/3") + pks := []int64{11, 22, 33} + texts := []string{strings.Repeat("first lob ", 32), strings.Repeat("second lob ", 32), strings.Repeat("third lob ", 32)} + schema := arrow.NewSchema([]arrow.Field{ + {Name: "100", Type: arrow.PrimitiveTypes.Int64, Metadata: arrow.NewMetadata([]string{packed.ArrowFieldIdMetadataKey}, []string{"100"})}, + {Name: "101", Type: arrow.BinaryTypes.String, Metadata: arrow.NewMetadata([]string{packed.ArrowFieldIdMetadataKey}, []string{"101"})}, + }, nil) + builder := array.NewRecordBuilder(memory.DefaultAllocator, schema) + defer builder.Release() + builder.Field(0).(*array.Int64Builder).AppendValues(pks, nil) + builder.Field(1).(*array.StringBuilder).AppendValues(texts, nil) + record := builder.NewRecord() + defer record.Release() + writer, err := packed.NewFFISegmentWriter(schema, &packed.SegmentWriterConfig{ + SegmentPath: base, + WriterFormat: "parquet", + TextColumns: []packed.TextColumnConfig{{ + FieldID: 101, LobBasePath: filepath.Join(legacyRoot, "insert_log/1/2/lobs/101"), + InlineThreshold: 1, MaxLobFileBytes: 1 << 20, FlushThresholdBytes: 1 << 20, + }}, + }, cfg) + require.NoError(t, err) + closed := false + t.Cleanup(func() { + if !closed { + out, closeErr := writer.Close() + if out != nil { + out.Destroy() + } + require.NoError(t, closeErr) + } + }) + require.NoError(t, writer.Write(record)) + output, err := writer.Close() + closed = true + require.NoError(t, err) + defer output.Destroy() + statsWriter := &storage.StatsWriter{} + require.NoError(t, statsWriter.GenerateByData(100, schemapb.DataType_Int64, &storage.Int64FieldData{Data: pks})) + statsPath := filepath.Join(base, "_stats/bloom_filter.100/42") + require.NoError(t, packed.WriteFile(cfg, statsPath, statsWriter.GetBuffer())) + physicalManifest, err := packed.CommitManifestUpdates(base, packed.ManifestEarliest, cfg, &packed.ManifestUpdates{ + NewFiles: output, + Stats: []packed.StatEntry{{Key: "bloom_filter.100", Files: []string{statsPath}, Metadata: map[string]string{"memory_size": "24"}}}, + }) + require.NoError(t, err) + _, version, err := packed.UnmarshalManifestPath(physicalManifest) + require.NoError(t, err) + return &legacyLocalPackedFixture{ + root: root, legacyRoot: legacyRoot, physicalManifest: physicalManifest, + segment: &datapb.SegmentInfo{ + ID: 3, CollectionID: 1, PartitionID: 2, State: commonpb.SegmentState_Flushed, + StorageVersion: storage.StorageV3, NumOfRows: int64(len(pks)), + ManifestPath: packed.MarshalManifestPath(catalogBase, version), + }, + cm: storage.NewLocalChunkManager(objectstorage.RootPath(root)), cfg: cfg, pks: pks, texts: texts, + } +} + +func (f *legacyLocalPackedFixture) assertNoCanonicalLayout(t *testing.T) { + t.Helper() + _, err := os.Stat(filepath.Join(f.root, "insert_log")) + require.ErrorIs(t, err, os.ErrNotExist, "legacy V3 compatibility must not create a canonical insert_log tree") +} + +// Read the old segment's real data, stats and LOB payload in place. This does +// not exercise compaction between the legacy and canonical storage namespaces. +func (f *legacyLocalPackedFixture) assertReadable(t *testing.T, manifest, expectedRoot string) { + t.Helper() + base, _, err := packed.UnmarshalManifestPath(manifest) + require.NoError(t, err) + fragments, err := packed.ReadFragmentsFromManifest(manifest, f.cfg, []string{"100"}) + require.NoError(t, err) + require.NotEmpty(t, fragments) + for _, fragment := range fragments { + require.True(t, strings.HasPrefix(fragment.FilePath, filepath.Join(base, "_data")+"/"), fragment.FilePath) + } + pkSchema := arrow.NewSchema([]arrow.Field{{Name: "100", Type: arrow.PrimitiveTypes.Int64}}, nil) + reader, err := packed.NewFFIPackedReader(manifest, pkSchema, []string{"100"}, 1<<20, f.cfg, nil, packed.ExternalReaderContext{}) + require.NoError(t, err) + defer func() { require.NoError(t, reader.Close()) }() + var gotPKs []int64 + for { + record, readErr := reader.ReadNext() + if readErr == io.EOF { + break + } + require.NoError(t, readErr) + gotPKs = append(gotPKs, record.Column(0).(*array.Int64).Int64Values()...) + } + require.Equal(t, f.pks, gotPKs) + + statsPaths, err := packed.NewStatsResolver(manifest, f.cfg).BloomFilterPaths(100) + require.NoError(t, err) + require.Equal(t, []string{filepath.Join(base, "_stats/bloom_filter.100/42")}, statsPaths) + statsBytes, err := f.cm.Read(context.Background(), statsPaths[0]) + require.NoError(t, err) + statsReader := &storage.StatsReader{} + statsReader.SetBuffer(statsBytes) + stats, err := statsReader.GetPrimaryKeyStats() + require.NoError(t, err) + assert.Equal(t, int64(100), stats.FieldID) + assert.True(t, stats.MinPk.EQ(storage.NewInt64PrimaryKey(f.pks[0]))) + assert.True(t, stats.MaxPk.EQ(storage.NewInt64PrimaryKey(f.pks[len(f.pks)-1]))) + + lobs, err := packed.GetManifestLobFiles(manifest, f.cfg) + require.NoError(t, err) + require.Len(t, lobs, 1, "the TEXT values must be stored out of line in a real Vortex file") + lob := lobs[0] + require.True(t, strings.HasPrefix(lob.Path, filepath.Join(expectedRoot, "insert_log/1/2/lobs/101/_data")+"/"), lob.Path) + require.Equal(t, int64(101), lob.FieldID) + require.Equal(t, int64(len(f.texts)), lob.TotalRows) + lobSchema := arrow.NewSchema([]arrow.Field{{Name: "text_data", Type: arrow.BinaryTypes.String}}, nil) + lobReader, err := packed.NewFFIPackedReaderWithFragments([]string{"text_data"}, "vortex", []packed.Fragment{{ + FilePath: lob.Path, StartRow: 0, EndRow: lob.TotalRows, RowCount: lob.TotalRows, + }}, lobSchema, []string{"text_data"}, 1<<20, f.cfg, nil, packed.ExternalReaderContext{}) + require.NoError(t, err) + defer func() { require.NoError(t, lobReader.Close()) }() + var gotTexts []string + for { + record, readErr := lobReader.ReadNext() + if readErr == io.EOF { + break + } + require.NoError(t, readErr) + column := record.Column(0).(*array.String) + for i := 0; i < column.Len(); i++ { + // ReadNext releases the preceding Arrow batch. Copy strings while + // its backing buffer is still alive, including before the EOF read. + gotTexts = append(gotTexts, strings.Clone(column.Value(i))) + } + } + require.Equal(t, f.texts, gotTexts) +} + +// Keep every fixture read beneath an opened root. The names returned by +// WalkDir are root-relative; they are never reopened as unchecked absolute +// filesystem paths. +func readLocalPackedFixtureFiles(t *testing.T, directory string) map[string][]byte { + t.Helper() + root, err := os.OpenRoot(directory) + require.NoError(t, err) + defer func() { require.NoError(t, root.Close()) }() + rootFS := root.FS() + files := make(map[string][]byte) + require.NoError(t, fs.WalkDir(rootFS, ".", func(name string, entry fs.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if entry.IsDir() { + return nil + } + contents, err := fs.ReadFile(rootFS, name) + if err == nil { + files[name] = contents + } + return err + })) + return files +} + +func TestReloadLocalManifestPreservesPackedReferences(t *testing.T) { + f := newLegacyLocalPackedFixture(t) + t.Chdir(f.root) + f.assertReadable(t, f.physicalManifest, f.legacyRoot) + f.assertNoCanonicalLayout(t) + originalFiles := readLocalPackedFixtureFiles(t, f.legacyRoot) + catalog := catalogmocks.NewDataCoordCatalog(t) + catalog.EXPECT().ListSegments(mock.Anything, int64(1)).Return([]*datapb.SegmentInfo{f.segment}, nil).Twice() + catalog.EXPECT().ListChannelCheckpoint(mock.Anything).Return(nil, nil).Twice() + mt := newLocalManifestTestMeta(catalog, f.cm) + originalSegment := proto.Clone(f.segment) + for i := 0; i < 2; i++ { + // The R/M namespace remains live compatibility data, not a rename source + // candidate, even when startup also checks for the double-root layout. + report, err := localmigrate.Migrate(t.Context(), f.root, localmigrate.Options{LegacyPrefix: "files"}) + require.NoError(t, err) + require.Zero(t, report.Renamed) + require.NoError(t, mt.reloadFromKV(context.Background(), []int64{1})) + loaded := mt.segments.GetSegment(3) + require.Equal(t, f.physicalManifest, loaded.GetManifestPath()) + f.assertReadable(t, loaded.GetManifestPath(), f.legacyRoot) + require.True(t, proto.Equal(originalSegment, f.segment)) + assertLocalManifestCatalogReadOnly(t, catalog) + f.assertNoCanonicalLayout(t) + } + // Neither metadata reload nor real reads rewrite the original manifest, + // data, stats or partition-level LOB files under root/minio.rootPath. + require.Equal(t, originalFiles, readLocalPackedFixtureFiles(t, f.legacyRoot)) + f.assertReadable(t, f.physicalManifest, f.legacyRoot) +} + +func TestMigrateDoubleRootPackedReferencesAndGCRetry(t *testing.T) { + f := newLocalPackedFixture(t, true) + t.Chdir(f.root) + f.assertNoCanonicalLayout(t) + f.assertReadable(t, f.physicalManifest, f.legacyRoot) + sources := readLocalPackedFixtureFiles(t, f.legacyRoot) + // CWD is the canonical root, leaving only the double-root rename source. + opts := localmigrate.Options{LegacyPrefix: "files"} + report, err := localmigrate.Migrate(t.Context(), f.root, opts) + require.NoError(t, err) + // The displaced insert_log directory is moved atomically as one unit; no + // file copy or backup tree is created. + require.Equal(t, 1, report.Renamed) + f.assertReadable(t, f.segment.GetManifestPath(), f.root) + canonicalRoot, err := os.OpenRoot(f.root) + require.NoError(t, err) + defer func() { require.NoError(t, canonicalRoot.Close()) }() + for name, original := range sources { + require.NoFileExists(t, filepath.Join(f.legacyRoot, name)) + contents, err := fs.ReadFile(canonicalRoot.FS(), name) + require.NoError(t, err) + require.Equal(t, original, contents, name) + } + + catalog := catalogmocks.NewDataCoordCatalog(t) + catalog.EXPECT().ListSegments(mock.Anything, int64(1)).Return([]*datapb.SegmentInfo{f.segment}, nil).Twice() + catalog.EXPECT().ListChannelCheckpoint(mock.Anything).Return(nil, nil).Twice() + mt := newLocalManifestTestMeta(catalog, f.cm) + for range 2 { + report, err = localmigrate.Migrate(t.Context(), f.root, opts) + require.NoError(t, err) + require.Zero(t, report.Renamed) + require.NoError(t, mt.reloadFromKV(context.Background(), []int64{1})) + require.Equal(t, f.segment.GetManifestPath(), mt.segments.GetSegment(3).GetManifestPath()) + assertLocalManifestCatalogReadOnly(t, catalog) + f.assertReadable(t, mt.segments.GetSegment(3).GetManifestPath(), f.root) + } + + // Segment GC and partition-level LOB GC may reclaim the canonical files. + removePool := conc.NewPool[struct{}](1) + t.Cleanup(removePool.Release) + gc := &garbageCollector{option: GcOption{cli: f.cm, removeObjectPool: removePool}} + dropped := proto.Clone(f.segment).(*datapb.SegmentInfo) + dropped.State = commonpb.SegmentState_Dropped + for range 2 { + require.NoError(t, gc.removeDroppedSegmentFiles(context.Background(), &SegmentInfo{SegmentInfo: dropped}, nil)) + } + canonicalBase, _, err := packed.UnmarshalManifestPath(f.segment.GetManifestPath()) + require.NoError(t, err) + require.NoFileExists(t, filepath.Join(canonicalBase, "_stats/bloom_filter.100/42")) + params := paramtable.Get() + safetyWindow := params.DataCoordCfg.GCLOBSafetyWindow.GetValue() + require.NoError(t, params.Save(params.DataCoordCfg.GCLOBSafetyWindow.Key, "0")) + t.Cleanup(func() { require.NoError(t, params.Save(params.DataCoordCfg.GCLOBSafetyWindow.Key, safetyWindow)) }) + lobGC := newLOBGCContext(gc) + orphans := lobGC.scanOrphanLOBFiles(t.Context(), typeutil.NewSet[string]()) + require.Len(t, orphans, 1, "the canonical orphan LOB is offered to the deletion callback") + require.True(t, strings.HasPrefix(orphans[0].FilePath, filepath.Join(f.root, "insert_log/1/2/lobs")+"/")) + lobGC.removeOrphanLOBFiles(t.Context(), orphans) + require.Empty(t, lobGC.scanOrphanLOBFiles(t.Context(), typeutil.NewSet[string]())) + // A fresh startup never treats completed renames as sources. GC must not be + // undone by restoring old immutable manifests, stats, data or LOB files. + for range 2 { + report, err = localmigrate.Migrate(t.Context(), f.root, opts) + require.NoError(t, err) + require.Zero(t, report.Renamed) + for name := range sources { + _, err := canonicalRoot.Stat(name) + require.ErrorIs(t, err, fs.ErrNotExist, name) + require.NoFileExists(t, filepath.Join(f.legacyRoot, name)) + } + } +} + +func TestReloadLocalManifestCommitSegmentManifestCAS(t *testing.T) { + for _, mutationType := range []ManifestMutationType{ManifestMutationNoop, ManifestMutationCommitUpdates} { + name := "noop CAS" + if mutationType == ManifestMutationCommitUpdates { + name = "structured mutation" + } + t.Run(name, func(t *testing.T) { + f := newLegacyLocalPackedFixture(t) + f.assertNoCanonicalLayout(t) + stored := proto.Clone(f.segment).(*datapb.SegmentInfo) + catalog := catalogmocks.NewDataCoordCatalog(t) + catalog.EXPECT().ListSegments(mock.Anything, int64(1)).RunAndReturn(func(context.Context, int64) ([]*datapb.SegmentInfo, error) { + return []*datapb.SegmentInfo{stored}, nil + }).Twice() + catalog.EXPECT().ListChannelCheckpoint(mock.Anything).Return(nil, nil).Twice() + mt := newLocalManifestTestMeta(catalog, f.cm) + ctx := context.Background() + require.NoError(t, mt.reloadFromKV(ctx, []int64{1})) + assertLocalManifestCatalogReadOnly(t, catalog) + base, version, err := packed.UnmarshalManifestPath(f.physicalManifest) + require.NoError(t, err) + prepared := packed.MarshalManifestPath(base, version+1) + // CAS compares the authoritative normalized in-memory pointer, not the + // still-relative catalog value. Old workers and old versions are stale. + for _, expected := range []string{f.segment.GetManifestPath(), packed.MarshalManifestPath(base, version-1)} { + err := mt.CommitSegmentManifest(ctx, SegmentManifestCommit{ + SegmentID: 3, ExpectedManifest: expected, + Mutation: ManifestMutation{Type: ManifestMutationNoop, ManifestPath: prepared}, + }) + require.ErrorIs(t, err, errSegmentManifestStale) + require.ErrorIs(t, err, merr.ErrServiceUnavailable) + } + catalog.AssertNotCalled(t, "Update", mock.Anything, mock.Anything) + require.Equal(t, f.segment.GetManifestPath(), stored.GetManifestPath()) + updates := &packed.ManifestUpdates{Stats: []packed.StatEntry{{ + Key: "bloom_filter.100", Files: []string{filepath.Join(base, "_stats/bloom_filter.100/42")}, + Metadata: map[string]string{"memory_size": "48"}, + }}} + commit := SegmentManifestCommit{ + SegmentID: 3, StorageConfig: f.cfg, + Mutation: ManifestMutation{Type: mutationType, Updates: updates}, + } + if mutationType == ManifestMutationNoop { + actualPrepared, err := packed.CommitManifestUpdates(base, version, f.cfg, updates) + require.NoError(t, err) + require.Equal(t, prepared, actualPrepared) + commit.ExpectedManifest = f.physicalManifest + commit.Mutation.ManifestPath = actualPrepared + } + catalog.EXPECT().Update(mock.Anything, mock.Anything).RunAndReturn(func(_ context.Context, actions ...metastore.UpdateAction) error { + require.Equal(t, f.segment.GetManifestPath(), stored.GetManifestPath(), "no startup write may precede the ordinary commit") + require.Equal(t, f.physicalManifest, mt.segments.GetSegment(3).GetManifestPath(), "publish memory only after catalog success") + require.Len(t, actions, 1) + entry, ok := actions[0].Entry.(metastore.SegmentEntry) + require.True(t, ok) + require.Equal(t, prepared, entry.Segment.GetManifestPath()) + stored = proto.Clone(entry.Segment).(*datapb.SegmentInfo) + return nil + }).Once() + require.NoError(t, mt.CommitSegmentManifest(ctx, commit)) + require.Equal(t, prepared, mt.segments.GetSegment(3).GetManifestPath()) + f.assertReadable(t, prepared, f.legacyRoot) + f.assertNoCanonicalLayout(t) + // A result prepared on the previously normalized pointer must not be + // accepted after this publication, even with a higher prepared version. + err = mt.CommitSegmentManifest(ctx, SegmentManifestCommit{ + SegmentID: 3, ExpectedManifest: f.physicalManifest, + Mutation: ManifestMutation{Type: ManifestMutationNoop, ManifestPath: packed.MarshalManifestPath(base, version+2)}, + }) + require.ErrorIs(t, err, errSegmentManifestStale) + require.Equal(t, prepared, stored.GetManifestPath()) + catalog.AssertNumberOfCalls(t, "Update", 1) + require.NoError(t, mt.reloadFromKV(ctx, []int64{1})) + require.Equal(t, prepared, mt.segments.GetSegment(3).GetManifestPath()) + f.assertNoCanonicalLayout(t) + }) + } +} diff --git a/internal/datacoord/meta_local_manifest_test.go b/internal/datacoord/meta_local_manifest_test.go new file mode 100644 index 0000000000..c3776e4185 --- /dev/null +++ b/internal/datacoord/meta_local_manifest_test.go @@ -0,0 +1,306 @@ +// Licensed to the LF AI & Data foundation under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package datacoord + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/proto" + + "github.com/milvus-io/milvus-proto/go-api/v3/commonpb" + "github.com/milvus-io/milvus/internal/metastore" + catalogmocks "github.com/milvus-io/milvus/internal/metastore/mocks" + "github.com/milvus-io/milvus/internal/mocks" + "github.com/milvus-io/milvus/internal/storage" + "github.com/milvus-io/milvus/internal/storagev2/packed" + "github.com/milvus-io/milvus/pkg/v3/objectstorage" + "github.com/milvus-io/milvus/pkg/v3/proto/datapb" + "github.com/milvus-io/milvus/pkg/v3/util/merr" + "github.com/milvus-io/milvus/pkg/v3/util/paramtable" +) + +func TestNormalizeLocalManifestPath(t *testing.T) { + const root = "/var/lib/milvus/data" + const suffix = "insert_log/1/2/3" + const version = int64(17) + for _, tc := range []struct { + name string + base string + prefix string + }{ + {"legacy prefix", "files/" + suffix, "files"}, + {"nested prefix", "legacy/files/" + suffix, "legacy/files"}, + {"empty prefix", suffix, ""}, + {"bucket root prefix", suffix, "."}, + {"empty component", "files//" + suffix, "files"}, + {"trailing separator", "files/" + suffix + "/", "files"}, + {"dot component", "files/./" + suffix, "files"}, + {"parent component within prefix", "files/tmp/../" + suffix, "files"}, + {"dot prefix component", "files/" + suffix, "./files"}, + {"empty prefix component", "legacy/files/" + suffix, "legacy//files"}, + } { + t.Run(tc.name, func(t *testing.T) { + got, err := normalizeLocalManifestPath(packed.MarshalManifestPath(tc.base, version), root, tc.prefix, 1, 2, 3) + require.NoError(t, err) + assert.Equal(t, packed.MarshalManifestPath(filepath.Join(root, tc.base), version), got) + // A persisted absolute path needs no further transformation. + again, err := normalizeLocalManifestPath(got, root, tc.prefix, 1, 2, 3) + require.NoError(t, err) + assert.Equal(t, got, again) + }) + } + + t.Run("absolute manifest preserved byte for byte", func(t *testing.T) { + manifest := ` { "ver": 23, "base_path": "/var/lib/milvus/data/insert_log/1/2/3" } ` + got, err := normalizeLocalManifestPath(manifest, root, "files", 1, 2, 3) + require.NoError(t, err) + assert.Equal(t, manifest, got) + }) + t.Run("no manifest yet", func(t *testing.T) { + got, err := normalizeLocalManifestPath("", root, "files", 1, 2, 3) + require.NoError(t, err) + assert.Empty(t, got) + }) + t.Run("filesystem root", func(t *testing.T) { + got, err := normalizeLocalManifestPath(packed.MarshalManifestPath("files/"+suffix, version), "/", "files", 1, 2, 3) + require.NoError(t, err) + assert.Equal(t, packed.MarshalManifestPath("/files/"+suffix, version), got) + }) + t.Run("relative storage root rejected", func(t *testing.T) { + got, err := normalizeLocalManifestPath(packed.MarshalManifestPath("files/"+suffix, version), "data", "files", 1, 2, 3) + require.ErrorIs(t, err, merr.ErrParameterInvalid) + assert.Empty(t, got) + }) + + for _, tc := range []struct { + name string + base string + prefix string + }{ + {"empty base", "", "files"}, + {"dot base", ".", "files"}, + {"parent base", "..", "files"}, + {"parent component", "files/../" + suffix, "files"}, + {"URI base", "s3://files/" + suffix, "files"}, + {"missing configured prefix", suffix, "files"}, + {"wrong prefix", "other/" + suffix, "files"}, + {"wrong collection", "files/insert_log/9/2/3", "files"}, + {"wrong partition", "files/insert_log/1/9/3", "files"}, + {"wrong segment", "files/insert_log/1/2/9", "files"}, + {"noncanonical ID", "files/insert_log/1/2/03", "files"}, + {"unexpected suffix", "files/" + suffix + "/_manifest", "files"}, + {"absolute prefix", "files/" + suffix, "/files"}, + {"URI prefix", "files/" + suffix, "s3://files"}, + {"parent prefix", "files/" + suffix, "../files"}, + {"outside storage root", "../files/" + suffix, "../files"}, + } { + t.Run(tc.name, func(t *testing.T) { + got, err := normalizeLocalManifestPath(packed.MarshalManifestPath(tc.base, version), root, tc.prefix, 1, 2, 3) + require.ErrorIs(t, err, merr.ErrDataIntegrity) + assert.Empty(t, got) + }) + } + t.Run("malformed JSON", func(t *testing.T) { + got, err := normalizeLocalManifestPath("not-json", root, "files", 1, 2, 3) + require.ErrorIs(t, err, merr.ErrDataIntegrity) + assert.Empty(t, got) + }) +} + +func setLocalManifestLegacyPrefix(t *testing.T, prefix string) { + t.Helper() + paramtable.Init() + params := paramtable.Get() + key := params.MinioCfg.RootPath.Key + original := params.MinioCfg.RootPath.GetValue() + require.NoError(t, params.Save(key, prefix)) + t.Cleanup(func() { require.NoError(t, params.Save(key, original)) }) +} + +func newLocalManifestTestMeta(catalog *catalogmocks.DataCoordCatalog, cm storage.ChunkManager) *meta { + return &meta{ + ctx: context.Background(), + catalog: catalog, + chunkManager: cm, + segments: NewSegmentsInfo(), + channelCPs: newChannelCps(), + } +} + +func assertLocalManifestCatalogReadOnly(t *testing.T, catalog *catalogmocks.DataCoordCatalog) { + t.Helper() + catalog.AssertNotCalled(t, "AlterSegments", mock.Anything, mock.Anything) + for _, call := range catalog.Calls { + assert.Contains(t, []string{"ListSegments", "ListChannelCheckpoint"}, call.Method, + "loading local manifests must not call a catalog mutation") + } +} + +func TestReloadLocalManifestPathsReadOnlyAndLazyPersistence(t *testing.T) { + setLocalManifestLegacyPrefix(t, "files") + ctx := context.Background() + root := t.TempDir() + cm := storage.NewLocalChunkManager(objectstorage.RootPath(root)) + legacyBase := "files/insert_log/1/2/3" + legacyFile := filepath.Join(root, legacyBase, "_data", "test-marker.parquet") + require.NoError(t, os.MkdirAll(filepath.Dir(legacyFile), 0o755)) + require.NoError(t, os.WriteFile(legacyFile, []byte("legacy-segment-data"), 0o600)) + // No layout migration: the marker exists only under the legacy prefix. + require.NoDirExists(t, filepath.Join(root, "insert_log")) + + raw := &datapb.SegmentInfo{ + ID: 3, + CollectionID: 1, + PartitionID: 2, + State: commonpb.SegmentState_Flushed, + StorageVersion: storage.StorageV3, + ManifestPath: packed.MarshalManifestPath(legacyBase, 17), + } + original := proto.Clone(raw).(*datapb.SegmentInfo) + stored := raw + catalog := catalogmocks.NewDataCoordCatalog(t) + catalog.EXPECT().ListSegments(mock.Anything, int64(1)).RunAndReturn(func(context.Context, int64) ([]*datapb.SegmentInfo, error) { + return []*datapb.SegmentInfo{stored}, nil + }).Times(3) + catalog.EXPECT().ListChannelCheckpoint(mock.Anything).Return(nil, nil).Times(3) + mt := newLocalManifestTestMeta(catalog, cm) + expected := packed.MarshalManifestPath(filepath.ToSlash(filepath.Join(root, legacyBase)), 17) + + // The same legacy catalog value is resolved on every reload without writes. + // Stats is deliberately nil: NewSegmentInfo fills it, so the compatibility + // layer must clone before constructing the in-memory segment as well. + for i := 0; i < 2; i++ { + require.NoError(t, mt.reloadFromKV(ctx, []int64{1})) + loaded := mt.segments.GetSegment(3) + require.NotNil(t, loaded) + assert.Equal(t, expected, loaded.GetManifestPath()) + assert.NotSame(t, raw, loaded.SegmentInfo) + assert.True(t, proto.Equal(original, raw), "reload must not mutate the catalog-owned proto") + assertLocalManifestCatalogReadOnly(t, catalog) + + base, version, err := packed.UnmarshalManifestPath(loaded.GetManifestPath()) + require.NoError(t, err) + assert.Equal(t, int64(17), version) + contents, err := cm.Read(ctx, filepath.Join(base, "_data", "test-marker.parquet")) + require.NoError(t, err) + assert.Equal(t, "legacy-segment-data", string(contents)) + assert.NoDirExists(t, filepath.Join(root, "insert_log"), "reload must not copy legacy files into the new layout") + } + + // A failed ordinary write does not undo the resolved in-memory view or + // commit the attempted state change. + catalog.EXPECT().AlterSegments(mock.Anything, mock.Anything).Return(merr.ErrIoTooManyRequests).Once() + require.ErrorIs(t, mt.UpdateSegmentsInfo(ctx, UpdateStatusOperator(3, commonpb.SegmentState_Dropped)), merr.ErrIoTooManyRequests) + assert.Equal(t, expected, mt.segments.GetSegment(3).GetManifestPath()) + assert.Equal(t, commonpb.SegmentState_Flushed, mt.segments.GetSegment(3).GetState()) + assert.True(t, proto.Equal(original, raw)) + assert.Same(t, raw, stored) + + // A successful normal segment update, not startup, persists the resolved path. + catalog.EXPECT().AlterSegments(mock.Anything, mock.Anything).RunAndReturn(func(_ context.Context, segments []*datapb.SegmentInfo, increments ...metastore.BinlogsIncrement) error { + require.Len(t, segments, 1) + assert.Empty(t, increments) + assert.Equal(t, expected, segments[0].GetManifestPath()) + assert.Equal(t, commonpb.SegmentState_Dropped, segments[0].GetState()) + stored = proto.Clone(segments[0]).(*datapb.SegmentInfo) + return nil + }).Once() + require.NoError(t, mt.UpdateSegmentsInfo(ctx, UpdateStatusOperator(3, commonpb.SegmentState_Dropped))) + assert.True(t, proto.Equal(original, raw)) + assert.Equal(t, expected, stored.GetManifestPath()) + require.NoError(t, mt.reloadFromKV(ctx, []int64{1})) + assert.Equal(t, expected, mt.segments.GetSegment(3).GetManifestPath()) + catalog.AssertNumberOfCalls(t, "AlterSegments", 2) + legacyContents, err := os.ReadFile(legacyFile) + require.NoError(t, err) + assert.Equal(t, "legacy-segment-data", string(legacyContents)) + assert.NoDirExists(t, filepath.Join(root, "insert_log")) +} + +func TestReloadLocalManifestPathsRejectInvalid(t *testing.T) { + setLocalManifestLegacyPrefix(t, "files") + for _, tc := range []struct { + name string + manifest string + }{ + {"malformed JSON", "not-json"}, + {"wrong segment", packed.MarshalManifestPath("files/insert_log/1/2/99", 1)}, + {"traversal", packed.MarshalManifestPath("files/../insert_log/1/2/3", 1)}, + {"missing configured prefix", packed.MarshalManifestPath("insert_log/1/2/3", 1)}, + } { + t.Run(tc.name, func(t *testing.T) { + raw := &datapb.SegmentInfo{ + ID: 3, CollectionID: 1, PartitionID: 2, + State: commonpb.SegmentState_Flushed, StorageVersion: storage.StorageV3, + ManifestPath: tc.manifest, + } + original := proto.Clone(raw) + catalog := catalogmocks.NewDataCoordCatalog(t) + catalog.EXPECT().ListSegments(mock.Anything, int64(1)).Return([]*datapb.SegmentInfo{raw}, nil).Once() + catalog.EXPECT().ListChannelCheckpoint(mock.Anything).Return(nil, nil).Maybe() + mt := newLocalManifestTestMeta(catalog, storage.NewLocalChunkManager(objectstorage.RootPath(t.TempDir()))) + require.ErrorIs(t, mt.reloadFromKV(context.Background(), []int64{1}), merr.ErrDataIntegrity) + assert.Nil(t, mt.segments.GetSegment(3), "invalid metadata must never enter the segment cache") + assert.True(t, proto.Equal(original, raw)) + assertLocalManifestCatalogReadOnly(t, catalog) + }) + } +} + +func TestReloadLocalManifestPathsBypass(t *testing.T) { + setLocalManifestLegacyPrefix(t, "files") + for _, tc := range []struct { + name string + remote bool + version int64 + manifest string + }{ + {"remote V3 remains relative", true, storage.StorageV3, packed.MarshalManifestPath("files/insert_log/1/2/3", 7)}, + {"local V2 does not parse manifest", false, storage.StorageV2, "not-a-v3-manifest"}, + {"local V3 absolute unchanged", false, storage.StorageV3, ` { "ver": 9, "base_path": "/var/lib/milvus/data/insert_log/1/2/3" } `}, + {"local V3 no manifest yet", false, storage.StorageV3, ""}, + } { + t.Run(tc.name, func(t *testing.T) { + raw := &datapb.SegmentInfo{ + ID: 3, CollectionID: 1, PartitionID: 2, + State: commonpb.SegmentState_Flushed, StorageVersion: tc.version, + ManifestPath: tc.manifest, Stats: &datapb.Statistics{}, + } + original := proto.Clone(raw) + catalog := catalogmocks.NewDataCoordCatalog(t) + catalog.EXPECT().ListSegments(mock.Anything, int64(1)).Return([]*datapb.SegmentInfo{raw}, nil).Once() + catalog.EXPECT().ListChannelCheckpoint(mock.Anything).Return(nil, nil).Once() + var cm storage.ChunkManager = storage.NewLocalChunkManager(objectstorage.RootPath(t.TempDir())) + if tc.remote { + cm = mocks.NewChunkManager(t) + } + mt := newLocalManifestTestMeta(catalog, cm) + require.NoError(t, mt.reloadFromKV(context.Background(), []int64{1})) + loaded := mt.segments.GetSegment(3) + require.NotNil(t, loaded) + assert.Equal(t, tc.manifest, loaded.GetManifestPath()) + assert.True(t, proto.Equal(original, raw)) + assertLocalManifestCatalogReadOnly(t, catalog) + }) + } +} diff --git a/internal/datacoord/segment_manager.go b/internal/datacoord/segment_manager.go index fc4bbd31f8..0a2b6a0a0b 100644 --- a/internal/datacoord/segment_manager.go +++ b/internal/datacoord/segment_manager.go @@ -19,7 +19,6 @@ package datacoord import ( "context" "fmt" - "path" "sync" "time" @@ -30,13 +29,10 @@ import ( "github.com/milvus-io/milvus/internal/datacoord/allocator" "github.com/milvus-io/milvus/internal/storage" "github.com/milvus-io/milvus/internal/storagev2/packed" - "github.com/milvus-io/milvus/pkg/v3/common" "github.com/milvus-io/milvus/pkg/v3/mlog" "github.com/milvus-io/milvus/pkg/v3/proto/datapb" "github.com/milvus-io/milvus/pkg/v3/util/lock" "github.com/milvus-io/milvus/pkg/v3/util/merr" - "github.com/milvus-io/milvus/pkg/v3/util/metautil" - "github.com/milvus-io/milvus/pkg/v3/util/paramtable" "github.com/milvus-io/milvus/pkg/v3/util/retry" "github.com/milvus-io/milvus/pkg/v3/util/tsoutil" "github.com/milvus-io/milvus/pkg/v3/util/typeutil" @@ -417,8 +413,7 @@ func (s *SegmentManager) openNewSegmentWithGivenSegmentID(ctx context.Context, r var manifestPath string if req.StorageVersion == storage.StorageV3 { - k := metautil.JoinIDPath(req.CollectionID, req.PartitionID, req.SegmentID) - basePath := path.Join(paramtable.Get().MinioCfg.RootPath.GetValue(), common.SegmentInsertLogPath, k) + basePath := storage.SegmentManifestBasePath(createStorageConfig().GetRootPath(), req.CollectionID, req.PartitionID, req.SegmentID) manifestPath = packed.MarshalManifestPath(basePath, packed.ManifestEarliest) } diff --git a/internal/core/src/index/Meta.cpp b/internal/datacoord/segment_path.go similarity index 61% rename from internal/core/src/index/Meta.cpp rename to internal/datacoord/segment_path.go index 96f1ecb9c6..c213f1dd61 100644 --- a/internal/core/src/index/Meta.cpp +++ b/internal/datacoord/segment_path.go @@ -14,8 +14,18 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "index/Meta.h" +package datacoord -namespace milvus::index { -std::string kOverrideRootPathForUT; -} // namespace milvus::index +import ( + "fmt" + "strings" + + "github.com/milvus-io/milvus/pkg/v3/common" +) + +// segmentBaseMatches checks only the segment identity, allowing legacy prefixes. +// Callers own backend-specific normalization and root containment. +func segmentBaseMatches(base string, collectionID, partitionID, segmentID int64) bool { + suffix := fmt.Sprintf("%s/%d/%d/%d", common.SegmentInsertLogPath, collectionID, partitionID, segmentID) + return base == suffix || strings.HasSuffix(base, "/"+suffix) +} diff --git a/internal/datacoord/segment_path_test.go b/internal/datacoord/segment_path_test.go new file mode 100644 index 0000000000..f69e0bf71e --- /dev/null +++ b/internal/datacoord/segment_path_test.go @@ -0,0 +1,44 @@ +// Licensed to the LF AI & Data foundation under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package datacoord + +import "testing" + +func TestSegmentBaseMatches(t *testing.T) { + for _, tc := range []struct { + base string + want bool + }{ + {"insert_log/1/2/3", true}, + {"files/insert_log/1/2/3", true}, + {"/var/lib/milvus/data/files/insert_log/1/2/3", true}, + {"files//insert_log/1/2/3", true}, + {"files/tmp/../insert_log/1/2/3", true}, + {"insert_log/1/2/30", false}, + {"insert_log/1/9/3", false}, + {"insert_log/9/2/3", false}, + {"not_insert_log/1/2/3", false}, + {"insert_log/1/2/3/_stats", false}, + {"", false}, + } { + t.Run(tc.base, func(t *testing.T) { + if got := segmentBaseMatches(tc.base, 1, 2, 3); got != tc.want { + t.Fatalf("segmentBaseMatches(%q) = %v, want %v", tc.base, got, tc.want) + } + }) + } +} diff --git a/internal/datacoord/server.go b/internal/datacoord/server.go index 3316223ab3..82cee51599 100644 --- a/internal/datacoord/server.go +++ b/internal/datacoord/server.go @@ -641,7 +641,6 @@ func (s *Server) initMeta(chunkManager storage.ChunkManager) error { if err != nil { return err } - // Load collection information asynchronously // HINT: please make sure this is the last step in the `reloadEtcdFn` function !!! go func() { diff --git a/internal/datacoord/snapshot_local_manifest_test.go b/internal/datacoord/snapshot_local_manifest_test.go new file mode 100644 index 0000000000..8de8b67de7 --- /dev/null +++ b/internal/datacoord/snapshot_local_manifest_test.go @@ -0,0 +1,197 @@ +// Licensed to the LF AI & Data foundation under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package datacoord + +import ( + "context" + "path" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/milvus-io/milvus-proto/go-api/v3/schemapb" + snapshotstorage "github.com/milvus-io/milvus/internal/snapshotio/storage" + "github.com/milvus-io/milvus/internal/storage" + "github.com/milvus-io/milvus/internal/storagev2/packed" + "github.com/milvus-io/milvus/pkg/v3/objectstorage" + "github.com/milvus-io/milvus/pkg/v3/proto/datapb" + "github.com/milvus-io/milvus/pkg/v3/util/merr" + "github.com/milvus-io/milvus/pkg/v3/util/paramtable" +) + +func newLocalManifestSnapshot(t *testing.T, layout datapb.SnapshotLayout, manifestPath string) (*snapshotMeta, *datapb.SnapshotInfo, map[string][]byte) { + t.Helper() + paramtable.Init() + oldPrefix := Params.MinioCfg.RootPath.GetValue() + require.NoError(t, Params.Save(Params.MinioCfg.RootPath.Key, "files")) + t.Cleanup(func() { require.NoError(t, Params.Save(Params.MinioCfg.RootPath.Key, oldPrefix)) }) + + ctx := context.Background() + cm := storage.NewLocalChunkManager(objectstorage.RootPath(t.TempDir())) + sm := createTestSnapshotMeta(t) + t.Cleanup(sm.loaderCancel) + // A read must not persist to the metadata catalog. + sm.catalog = nil + sm.chunkManager = cm + sm.reader = snapshotstorage.NewSnapshotReader(cm) + sm.writer = snapshotstorage.NewSnapshotWriter(cm) + info := &datapb.SnapshotInfo{ + Id: 10, + CollectionId: 100, + Name: "legacy-local", + State: datapb.SnapshotState_SnapshotStateCommitted, + } + manifestDir, metadataPath := snapshotstorage.GetSnapshotPaths(cm.RootPath(), info.GetCollectionId(), info.GetId()) + info.S3Location = metadataPath + snapshot := &snapshotstorage.SnapshotData{ + SnapshotInfo: info, + Collection: &datapb.CollectionDescription{Schema: &schemapb.CollectionSchema{Name: "local"}}, + Segments: []*datapb.SegmentDescription{{ + SegmentId: 1001, + PartitionId: 1, + StorageVersion: storage.StorageV3, + ManifestPath: manifestPath, + }}, + SegmentIDs: []int64{1001}, + } + _, _, err := sm.writer.SaveToRootWithSize(ctx, snapshot, cm.RootPath(), layout) + require.NoError(t, err) + sm.snapshotID2Info.Insert(info.GetId(), info) + sm.addToSecondaryIndexes(info) + + stored := make(map[string][]byte) + for _, file := range []string{metadataPath, snapshotstorage.GetSegmentManifestPath(manifestDir, 1001)} { + data, err := cm.Read(ctx, file) + require.NoError(t, err) + stored[file] = data + } + return sm, info, stored +} + +func assertLocalSnapshotFilesUnchanged(t *testing.T, sm *snapshotMeta, stored map[string][]byte) { + t.Helper() + for file, original := range stored { + data, err := sm.chunkManager.Read(context.Background(), file) + require.NoError(t, err) + assert.Equal(t, original, data, file) + } +} + +func TestReadSnapshotDataNormalizesLegacyLocalManifestWithoutWrites(t *testing.T) { + ctx := context.Background() + legacy := packed.MarshalManifestPath("files/insert_log/100/1/1001", 7) + sm, info, stored := newLocalManifestSnapshot(t, datapb.SnapshotLayout_SnapshotLayoutReferenced, legacy) + want := packed.MarshalManifestPath(path.Join(sm.chunkManager.RootPath(), "files/insert_log/100/1/1001"), 7) + + var firstFingerprint string + for read := range 2 { + loaded, err := sm.ReadSnapshotData(ctx, info.GetCollectionId(), info.GetName(), true) + require.NoError(t, err) + require.Len(t, loaded.Segments, 1) + assert.Equal(t, want, loaded.Segments[0].GetManifestPath()) + assertLocalSnapshotFilesUnchanged(t, sm, stored) + fingerprint, err := snapshotstorage.SnapshotFingerprint(loaded) + require.NoError(t, err) + if read == 0 { + firstFingerprint = fingerprint + } else { + assert.Equal(t, firstFingerprint, fingerprint) + } + // Neither a prior read nor an in-memory consumer can alter the source. + loaded.Segments[0].ManifestPath = "modified-by-consumer" + } + storedSnapshot, err := sm.reader.ReadSnapshot(ctx, info.GetS3Location(), true) + require.NoError(t, err) + assert.Equal(t, legacy, storedSnapshot.Segments[0].GetManifestPath()) + metadataOnly, err := sm.ReadSnapshotData(ctx, info.GetCollectionId(), info.GetName(), false) + require.NoError(t, err) + assert.Empty(t, metadataOnly.Segments) + assertLocalSnapshotFilesUnchanged(t, sm, stored) +} + +func TestReadSnapshotDataRejectsMismatchedLegacyManifestIdentity(t *testing.T) { + legacy := packed.MarshalManifestPath("files/insert_log/100/1/9999", 7) + sm, info, stored := newLocalManifestSnapshot(t, datapb.SnapshotLayout_SnapshotLayoutReferenced, legacy) + loaded, err := sm.ReadSnapshotData(context.Background(), info.GetCollectionId(), info.GetName(), true) + require.ErrorIs(t, err, merr.ErrDataIntegrity) + assert.Nil(t, loaded) + assertLocalSnapshotFilesUnchanged(t, sm, stored) +} + +func TestReadSnapshotDataPreservesOtherManifestNamespaces(t *testing.T) { + ctx := context.Background() + legacy := packed.MarshalManifestPath("files/insert_log/100/1/1001", 7) + t.Run("self-contained", func(t *testing.T) { + sm, info, stored := newLocalManifestSnapshot(t, datapb.SnapshotLayout_SnapshotLayoutReferenced, legacy) + snapshot, err := sm.reader.ReadSnapshot(ctx, info.GetS3Location(), true) + require.NoError(t, err) + bundleManifest := packed.MarshalManifestPath(path.Join(sm.chunkManager.RootPath(), + snapshotstorage.ExportedSnapshotFilesPath, "insert_log/100/1/1001"), 7) + snapshot.Segments[0].ManifestPath = bundleManifest + _, _, err = sm.writer.SaveToRootWithSize(ctx, snapshot, sm.chunkManager.RootPath(), datapb.SnapshotLayout_SnapshotLayoutSelfContained) + require.NoError(t, err) + for file := range stored { + stored[file], err = sm.chunkManager.Read(ctx, file) + require.NoError(t, err) + } + loaded, err := sm.ReadSnapshotData(ctx, info.GetCollectionId(), info.GetName(), true) + require.NoError(t, err) + assert.Equal(t, bundleManifest, loaded.Segments[0].GetManifestPath()) + assertLocalSnapshotFilesUnchanged(t, sm, stored) + }) + t.Run("self-contained-rejects-outside-bundle", func(t *testing.T) { + sm, info, stored := newLocalManifestSnapshot(t, datapb.SnapshotLayout_SnapshotLayoutSelfContained, legacy) + // A self-contained bundle's paths are governed by the bundle root, + // not by the instance's legacy prefix compatibility rule. + _, directErr := sm.reader.ReadSnapshot(ctx, info.GetS3Location(), true) + require.ErrorIs(t, directErr, merr.ErrDataIntegrity) + loaded, err := sm.ReadSnapshotData(ctx, info.GetCollectionId(), info.GetName(), true) + require.ErrorIs(t, err, merr.ErrDataIntegrity) + assert.Equal(t, directErr.Error(), err.Error()) + assert.Nil(t, loaded) + assertLocalSnapshotFilesUnchanged(t, sm, stored) + }) + t.Run("absolute", func(t *testing.T) { + absolute := packed.MarshalManifestPath("/existing/insert_log/100/1/1001", 9) + sm, info, stored := newLocalManifestSnapshot(t, datapb.SnapshotLayout_SnapshotLayoutReferenced, absolute) + loaded, err := sm.ReadSnapshotData(ctx, info.GetCollectionId(), info.GetName(), true) + require.NoError(t, err) + assert.Equal(t, absolute, loaded.Segments[0].GetManifestPath()) + assertLocalSnapshotFilesUnchanged(t, sm, stored) + }) + t.Run("foreign", func(t *testing.T) { + sm, info, stored := newLocalManifestSnapshot(t, datapb.SnapshotLayout_SnapshotLayoutReferenced, legacy) + loaded, err := sm.ReadExternalSnapshotDataWithChunkManager(ctx, sm.chunkManager, info.GetS3Location(), true) + require.NoError(t, err) + assert.Equal(t, legacy, loaded.Segments[0].GetManifestPath()) + assertLocalSnapshotFilesUnchanged(t, sm, stored) + }) + t.Run("remote", func(t *testing.T) { + sm, info, stored := newLocalManifestSnapshot(t, datapb.SnapshotLayout_SnapshotLayoutReferenced, legacy) + // Keep real snapshot reads on disk while exercising the non-local backend gate. + sm.chunkManager = &snapshotManifestNonLocalChunkManager{ChunkManager: sm.chunkManager} + loaded, err := sm.ReadSnapshotData(ctx, info.GetCollectionId(), info.GetName(), true) + require.NoError(t, err) + assert.Equal(t, legacy, loaded.Segments[0].GetManifestPath()) + assertLocalSnapshotFilesUnchanged(t, sm, stored) + }) +} + +type snapshotManifestNonLocalChunkManager struct { + storage.ChunkManager +} diff --git a/internal/datacoord/snapshot_meta.go b/internal/datacoord/snapshot_meta.go index 83c42f5fb1..fc30dcc9ce 100644 --- a/internal/datacoord/snapshot_meta.go +++ b/internal/datacoord/snapshot_meta.go @@ -939,6 +939,23 @@ func (sm *snapshotMeta) ReadSnapshotData(ctx context.Context, collectionID int64 mlog.Error(context.TODO(), "failed to read snapshot data from S3", mlog.Err(err)) return nil, err } + // Named, referenced snapshots point at this instance's storage. Older + // snapshots retain relative V3 manifest bases whose files remain under the + // legacy local prefix. Resolve only the returned view to their absolute paths; + // the stored snapshot is immutable, and foreign/self-contained layouts use other roots. + if local, ok := sm.chunkManager.(*storage.LocalChunkManager); ok && snapshotData.Layout == datapb.SnapshotLayout_SnapshotLayoutReferenced { + for _, segment := range snapshotData.Segments { + if segment.GetStorageVersion() != storage.StorageV3 || segment.GetManifestPath() == "" { + continue + } + manifestPath, err := normalizeLocalManifestPath(segment.GetManifestPath(), local.RootPath(), + Params.MinioCfg.RootPath.GetValue(), collectionID, segment.GetPartitionId(), segment.GetSegmentId()) + if err != nil { + return nil, merr.Wrapf(err, "normalize local snapshot %d segment %d manifest", snapshotInfo.GetId(), segment.GetSegmentId()) + } + segment.ManifestPath = manifestPath + } + } // Step 3: Merge S3 location from memory into returned data snapshotData.SnapshotInfo.S3Location = snapshotInfo.S3Location diff --git a/internal/datacoord/task_refresh_external_collection_test.go b/internal/datacoord/task_refresh_external_collection_test.go index 082b2759b4..1bf7c01418 100644 --- a/internal/datacoord/task_refresh_external_collection_test.go +++ b/internal/datacoord/task_refresh_external_collection_test.go @@ -913,6 +913,7 @@ func TestRefreshExternalCollectionTask_QueryTaskOnWorker(t *testing.T) { }) t.Run("task_failed_on_worker", func(t *testing.T) { + paramtable.Init() catalog := &stubCatalog{} refreshMeta, err := newExternalCollectionRefreshMeta(context.Background(), catalog) assert.NoError(t, err) @@ -924,6 +925,7 @@ func TestRefreshExternalCollectionTask_QueryTaskOnWorker(t *testing.T) { State: indexpb.JobState_JobStateInProgress, ExternalSource: "s3://bucket/path", ExternalSpec: "iceberg", + TaskIds: []int64{1001, 1002}, } err = refreshMeta.AddJob(job) assert.NoError(t, err) @@ -939,16 +941,26 @@ func TestRefreshExternalCollectionTask_QueryTaskOnWorker(t *testing.T) { } err = refreshMeta.AddTask(protoTask) assert.NoError(t, err) + // A failed task must terminate the job without waiting for siblings. + sibling := proto.Clone(protoTask).(*datapb.ExternalCollectionRefreshTask) + sibling.TaskId = 1002 + assert.NoError(t, refreshMeta.AddTask(sibling)) alloc := &stubAllocator{nextID: 99999} task := newRefreshExternalCollectionTask(protoTask, refreshMeta, nil, alloc) + var failedJobs []int64 + checker := newRefreshChecker(t.Context(), nil, refreshMeta, make(chan struct{}), nil, nil, + func(jobID int64) { failedJobs = append(failedJobs, jobID) }, nil, nil) + task.processFinishedJob = checker.processJobByID cluster := &stubCluster{} - // Mock QueryRefreshExternalCollectionTask to return failed state + // Missing explore manifests surface as a worker failure, even when + // the native reader classifies the underlying error as transient. + failReason := "failed to read explore manifest: File not found: loon FFI transient error" mockQuery := mockey.Mock((*stubCluster).QueryRefreshExternalCollectionTask).Return(&datapb.RefreshExternalCollectionTaskResponse{ State: indexpb.JobState_JobStateFailed, - FailReason: "worker error", + FailReason: failReason, }, nil).Build() defer mockQuery.UnPatch() @@ -957,7 +969,11 @@ func TestRefreshExternalCollectionTask_QueryTaskOnWorker(t *testing.T) { // Task should be marked as failed metaTask := refreshMeta.GetTask(1001) assert.Equal(t, indexpb.JobState_JobStateFailed, metaTask.GetState()) - assert.Contains(t, metaTask.GetFailReason(), "worker error") + assert.Equal(t, failReason, metaTask.GetFailReason()) + assert.Equal(t, indexpb.JobState_JobStateFailed, refreshMeta.GetJob(1).GetState()) + assert.Equal(t, failReason, refreshMeta.GetJob(1).GetFailReason()) + assert.Equal(t, []int64{1}, failedJobs) + assert.Equal(t, indexpb.JobState_JobStateInProgress, refreshMeta.GetTask(1002).GetState()) }) t.Run("task_finished_success", func(t *testing.T) { diff --git a/internal/datacoord/task_stats.go b/internal/datacoord/task_stats.go index 6718b73c2a..ca4a38a314 100644 --- a/internal/datacoord/task_stats.go +++ b/internal/datacoord/task_stats.go @@ -19,6 +19,8 @@ package datacoord import ( "context" "fmt" + "path" + "strings" "time" "github.com/cockroachdb/errors" @@ -353,16 +355,43 @@ func (st *statsTask) cleanupRejectedStatsResultFiles(ctx context.Context, result return } - files, err := collectRejectedStatsResultFiles(result) + expectedSegmentID := st.GetSegmentID() + if st.GetTargetSegmentID() != 0 { + expectedSegmentID = st.GetTargetSegmentID() + } + if result.GetCollectionID() != st.GetCollectionID() || + result.GetPartitionID() != st.GetPartitionID() || + result.GetSegmentID() != expectedSegmentID { + mlog.Warn(ctx, "refuse to cleanup rejected stats files for a mismatched segment", + mlog.FieldTaskID(st.GetTaskID()), + mlog.Int64("resultCollectionID", result.GetCollectionID()), + mlog.Int64("resultPartitionID", result.GetPartitionID()), + mlog.Int64("resultSegmentID", result.GetSegmentID())) + return + } + + basePath, files, err := collectRejectedStatsResultFiles( + result, st.meta.chunkManager.RootPath(), + st.GetCollectionID(), st.GetPartitionID(), expectedSegmentID) if err != nil { mlog.Warn(ctx, "failed to collect rejected stats result files", mlog.FieldTaskID(st.GetTaskID()), mlog.FieldSegmentID(st.GetSegmentID()), mlog.Err(err)) + return } if len(files) == 0 { return } + // Complete keys are absolute filesystem paths locally and bucket-relative + // object keys remotely. Never resolve a local base against the process CWD + // or pass a filesystem path to remote storage. + if _, local := st.meta.chunkManager.(*storage.LocalChunkManager); local != path.IsAbs(basePath) { + mlog.Warn(ctx, "refuse to cleanup rejected stats files with a manifest base for another storage namespace", + mlog.FieldTaskID(st.GetTaskID()), + mlog.String("basePath", basePath)) + return + } if err := st.meta.chunkManager.MultiRemove(ctx, files); err != nil { mlog.Warn(ctx, "failed to cleanup rejected stats result files", mlog.FieldTaskID(st.GetTaskID()), @@ -372,49 +401,131 @@ func (st *statsTask) cleanupRejectedStatsResultFiles(ctx context.Context, result } } -func collectRejectedStatsResultFiles(result *workerpb.StatsResult) ([]string, error) { +func collectRejectedStatsResultFiles( + result *workerpb.StatsResult, + rootPath string, + collectionID, partitionID, segmentID int64, +) (string, []string, error) { files := make([]string, 0) seen := make(map[string]struct{}) - addFile := func(file string) { - if file == "" { - return + addStatsFiles := func(statsBasePath string, statFiles []string, relative bool) error { + local := path.IsAbs(statsBasePath) + statsPrefix := statsBasePath + "/" + if local { + statsPrefix = path.Clean(statsBasePath) + "/" } - if _, ok := seen[file]; ok { - return + for _, file := range statFiles { + if strings.TrimSpace(file) == "" { + continue + } + // JSON reports names relative to the field's stats directory; + // TEXT reports complete keys. Decode before the shared check. + if relative { + if path.IsAbs(file) { + return merr.WrapErrDataIntegrityMsg("rejected JSON stats file must be relative to its stats directory") + } + file = statsBasePath + "/" + file + } + checkedFile := file + if local { + checkedFile = path.Clean(file) + } + // Workers report complete TEXT stats keys, including an absolute + // filesystem path on local storage. Do not guess a prefix for a + // relative filename or a key outside this field's stats directory. + if !strings.HasPrefix(checkedFile, statsPrefix) { + return merr.WrapErrDataIntegrityMsg("rejected stats file escapes its stats directory") + } + // Clean only for the local directory check. Remote object keys must be + // deleted exactly as reported, not rewritten as filesystem paths. + if _, ok := seen[file]; ok { + continue + } + seen[file] = struct{}{} + files = append(files, file) } - seen[file] = struct{}{} - files = append(files, file) + return nil } - - for _, stats := range result.GetTextStatsLogs() { - for _, file := range stats.GetFiles() { - addFile(file) - } - } - - jsonStats := result.GetJsonKeyStatsLogs() - if len(jsonStats) == 0 { - return files, nil - } - manifest := result.GetBaseManifest() if manifest == "" { manifest = result.GetManifest() } - if manifest == "" { - return files, merr.WrapErrServiceInternalMsg("manifest is empty for rejected json stats result") - } - basePath, _, err := packed.UnmarshalManifestPath(manifest) - if err != nil { - return files, err - } - for fieldID, stats := range jsonStats { - statsBasePath := fmt.Sprintf("%s/_stats/json_stats.%d", basePath, fieldID) - for _, file := range metautil.BuildStatsFilePaths(statsBasePath, stats.GetFiles()) { - addFile(file) + basePath := rootPath + if manifest != "" { + var err error + basePath, _, err = packed.UnmarshalManifestPath(manifest) + if err != nil { + return "", nil, err + } + if err := validateStatsManifestBase(basePath, collectionID, partitionID, segmentID); err != nil { + return "", nil, err + } + if result.GetManifest() != "" && result.GetManifest() != manifest { + resultBase, _, err := packed.UnmarshalManifestPath(result.GetManifest()) + if err != nil { + return "", nil, err + } + sameBase := resultBase == basePath + if path.IsAbs(basePath) { + sameBase = path.Clean(resultBase) == path.Clean(basePath) + } + if !sameBase { + return "", nil, merr.WrapErrDataIntegrityMsg("rejected stats result changes manifest base path") + } } } - return files, nil + + // Manifest-less V2 results use build/version directories; V3 uses _stats. + validateLegacyLocation := func(fieldID, buildID, version int64) error { + if buildID <= 0 || version <= 0 { + return merr.WrapErrDataIntegrityMsg( + "rejected stats result for field %d carries no build ID or version to locate its stats directory", fieldID) + } + return nil + } + for fieldID, stats := range result.GetTextStatsLogs() { + statsBase := basePath + fmt.Sprintf("/_stats/text_index.%d", fieldID) + if manifest == "" { + if err := validateLegacyLocation(fieldID, stats.GetBuildID(), stats.GetVersion()); err != nil { + return "", nil, err + } + statsBase = metautil.BuildTextIndexPrefix(rootPath, stats.GetBuildID(), stats.GetVersion(), + collectionID, partitionID, segmentID, fieldID) + } + if err := addStatsFiles(statsBase, stats.GetFiles(), false); err != nil { + return "", nil, err + } + } + for fieldID, stats := range result.GetJsonKeyStatsLogs() { + statsBase := metautil.BuildJSONKeyStatsV3Prefix(basePath, fieldID) + if manifest == "" { + if err := validateLegacyLocation(fieldID, stats.GetBuildID(), stats.GetVersion()); err != nil { + return "", nil, err + } + statsBase = metautil.BuildJSONKeyStatsPrefix(rootPath, + stats.GetJsonKeyStatsDataFormat(), stats.GetBuildID(), stats.GetVersion(), + collectionID, partitionID, segmentID, fieldID) + } + if err := addStatsFiles(statsBase, stats.GetFiles(), true); err != nil { + return "", nil, err + } + } + return basePath, files, nil +} + +func validateStatsManifestBase(basePath string, collectionID, partitionID, segmentID int64) error { + // Manifests carry complete storage keys, not provider URIs. A URI would + // be passed verbatim to ChunkManager cleanup and is not a valid object key. + if strings.Contains(basePath, "://") { + return merr.WrapErrDataIntegrityMsg("stats manifest base must be a storage key") + } + if path.IsAbs(basePath) { + basePath = path.Clean(basePath) + } + if !segmentBaseMatches(basePath, collectionID, partitionID, segmentID) { + return merr.WrapErrDataIntegrityMsg("stats manifest base does not match task segment identity") + } + return nil } func (st *statsTask) DropTaskOnWorker(cluster session.Cluster) { diff --git a/internal/datacoord/task_stats_test.go b/internal/datacoord/task_stats_test.go index 140f90e17b..9d0a9419da 100644 --- a/internal/datacoord/task_stats_test.go +++ b/internal/datacoord/task_stats_test.go @@ -18,12 +18,17 @@ package datacoord import ( "context" + "fmt" + "os" + "path" + "path/filepath" "testing" "time" "github.com/bytedance/mockey" "github.com/cockroachdb/errors" "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" "github.com/milvus-io/milvus-proto/go-api/v3/commonpb" @@ -35,15 +40,498 @@ import ( "github.com/milvus-io/milvus/internal/storage" "github.com/milvus-io/milvus/internal/storagev2/packed" "github.com/milvus-io/milvus/pkg/v3/common" + "github.com/milvus-io/milvus/pkg/v3/objectstorage" "github.com/milvus-io/milvus/pkg/v3/proto/datapb" "github.com/milvus-io/milvus/pkg/v3/proto/indexpb" "github.com/milvus-io/milvus/pkg/v3/proto/workerpb" "github.com/milvus-io/milvus/pkg/v3/taskcommon" "github.com/milvus-io/milvus/pkg/v3/util/lock" "github.com/milvus-io/milvus/pkg/v3/util/merr" + "github.com/milvus-io/milvus/pkg/v3/util/metautil" "github.com/milvus-io/milvus/pkg/v3/util/typeutil" ) +// Manifest bases on local storage are complete filesystem paths; cleanup +// deletes the reported files below /_stats/... directly, without +// joining any root. +func TestCleanupRejectedStatsResultFiles_LocalUsesCompletePaths(t *testing.T) { + root := t.TempDir() + cm := storage.NewLocalChunkManager(objectstorage.RootPath(root)) + basePath := filepath.Join(root, "insert_log/1/2/3") + statsFile := filepath.Join(basePath, "_stats/text_index.100/1178/2/index") + require.NoError(t, cm.Write(context.Background(), statsFile, []byte("stale"))) + + task := &statsTask{ + StatsTask: &indexpb.StatsTask{CollectionID: 1, PartitionID: 2, SegmentID: 3}, + meta: &meta{chunkManager: cm}, + } + task.cleanupRejectedStatsResultFiles(context.Background(), &workerpb.StatsResult{ + CollectionID: 1, + PartitionID: 2, + SegmentID: 3, + BaseManifest: packed.MarshalManifestPath(basePath, 1), + TextStatsLogs: map[int64]*datapb.TextIndexStats{ + 100: {Files: []string{statsFile}}, + }, + }) + + _, err := os.Stat(statsFile) + require.ErrorIs(t, err, os.ErrNotExist) +} + +func TestCleanupRejectedStatsResultFiles_LocalCompleteTextAndRelativeJSON(t *testing.T) { + ctx := context.Background() + root := t.TempDir() + cm := storage.NewLocalChunkManager(objectstorage.RootPath(root)) + basePath := filepath.Join(root, "insert_log/1/2/3") + textFile := filepath.Join(basePath, "_stats/text_index.100/tokenizer.json") + jsonFile := filepath.Join(basePath, "_stats/json_stats.101/shared_key_index/.managed.json_0") + untouchedFile := filepath.Join(basePath, "_stats/json_stats.101/shared_key_index/keep") + for _, file := range []string{textFile, jsonFile, untouchedFile} { + require.NoError(t, cm.Write(ctx, file, []byte("stats"))) + } + + task := &statsTask{ + StatsTask: &indexpb.StatsTask{CollectionID: 1, PartitionID: 2, SegmentID: 3}, + meta: &meta{chunkManager: cm}, + } + task.cleanupRejectedStatsResultFiles(ctx, &workerpb.StatsResult{ + CollectionID: 1, + PartitionID: 2, + SegmentID: 3, + BaseManifest: packed.MarshalManifestPath(basePath, 1), + TextStatsLogs: map[int64]*datapb.TextIndexStats{ + 100: {Files: []string{textFile}}, + }, + JsonKeyStatsLogs: map[int64]*datapb.JsonKeyStats{ + 101: {Files: []string{"shared_key_index/.managed.json_0"}}, + }, + }) + + require.NoFileExists(t, textFile) + require.NoFileExists(t, jsonFile) + content, err := cm.Read(ctx, untouchedFile) + require.NoError(t, err) + require.Equal(t, []byte("stats"), content) +} + +func TestCleanupRejectedStatsResultFiles_LocalRejectsRelativeTextFiles(t *testing.T) { + for _, relativeFile := range []string{"index", "./index", "task/index"} { + t.Run(relativeFile, func(t *testing.T) { + ctx := context.Background() + root := t.TempDir() + cm := storage.NewLocalChunkManager(objectstorage.RootPath(root)) + basePath := filepath.Join(root, "insert_log/1/2/3") + statsDir := filepath.Join(basePath, "_stats/text_index.100") + completeFile := filepath.Join(statsDir, "complete") + guessedFile := filepath.Join(statsDir, relativeFile) + require.NoError(t, cm.Write(ctx, completeFile, []byte("keep-complete"))) + require.NoError(t, cm.Write(ctx, guessedFile, []byte("keep-relative"))) + + task := &statsTask{ + StatsTask: &indexpb.StatsTask{CollectionID: 1, PartitionID: 2, SegmentID: 3}, + meta: &meta{chunkManager: cm}, + } + task.cleanupRejectedStatsResultFiles(ctx, &workerpb.StatsResult{ + CollectionID: 1, + PartitionID: 2, + SegmentID: 3, + BaseManifest: packed.MarshalManifestPath(basePath, 1), + TextStatsLogs: map[int64]*datapb.TextIndexStats{ + 100: {Files: []string{completeFile, relativeFile}}, + }, + }) + + // Reject the whole result, including its otherwise valid entry; no + // relative entry may be guessed into a complete deletion path. + content, err := cm.Read(ctx, completeFile) + require.NoError(t, err) + require.Equal(t, []byte("keep-complete"), content) + content, err = cm.Read(ctx, guessedFile) + require.NoError(t, err) + require.Equal(t, []byte("keep-relative"), content) + }) + } +} + +// A relative manifest base on local storage is a pre-3.0.2 manifest that +// DataCoord has not rewritten; cleanup must neither resolve it against the +// process CWD nor guess a root. +func TestCleanupRejectedStatsResultFiles_LocalRefusesRelativeManifestBase(t *testing.T) { + root := t.TempDir() + cm := storage.NewLocalChunkManager(objectstorage.RootPath(root)) + inRoot := filepath.Join(root, "insert_log/1/2/3/_stats/text_index.100/index") + require.NoError(t, cm.Write(context.Background(), inRoot, []byte("keep"))) + + task := &statsTask{ + StatsTask: &indexpb.StatsTask{CollectionID: 1, PartitionID: 2, SegmentID: 3}, + meta: &meta{chunkManager: cm}, + } + task.cleanupRejectedStatsResultFiles(context.Background(), &workerpb.StatsResult{ + CollectionID: 1, + PartitionID: 2, + SegmentID: 3, + BaseManifest: packed.MarshalManifestPath("insert_log/1/2/3", 1), + TextStatsLogs: map[int64]*datapb.TextIndexStats{ + 100: {Files: []string{"insert_log/1/2/3/_stats/text_index.100/index"}}, + }, + }) + require.FileExists(t, inRoot) +} + +func TestCleanupRejectedStatsResultFiles_LocalRejectsUnsafePaths(t *testing.T) { + ctx := context.Background() + outer := t.TempDir() + root := filepath.Join(outer, "storage") + require.NoError(t, os.MkdirAll(root, 0o755)) + cm := storage.NewLocalChunkManager(objectstorage.RootPath(root)) + task := &statsTask{ + StatsTask: &indexpb.StatsTask{CollectionID: 1, PartitionID: 2, SegmentID: 3}, + meta: &meta{chunkManager: cm}, + } + basePath := filepath.Join(root, "insert_log/1/2/3") + baseManifest := packed.MarshalManifestPath(basePath, 1) + resultForTextPath := func(filePath string) *workerpb.StatsResult { + return &workerpb.StatsResult{ + CollectionID: 1, + PartitionID: 2, + SegmentID: 3, + BaseManifest: baseManifest, + TextStatsLogs: map[int64]*datapb.TextIndexStats{ + 100: {Files: []string{filePath}}, + }, + } + } + + // Traversal and complete paths outside the stats directory are refused. + outsidePath := filepath.Join(outer, "outside") + require.NoError(t, os.WriteFile(outsidePath, []byte("keep"), 0o600)) + task.cleanupRejectedStatsResultFiles(ctx, resultForTextPath("../../../../outside")) + require.FileExists(t, outsidePath) + task.cleanupRejectedStatsResultFiles(ctx, resultForTextPath(outsidePath)) + require.FileExists(t, outsidePath) + + // A file outside /_stats/. is refused even when it is + // below the segment base. + arbitraryPath := filepath.Join(basePath, "not-stats") + require.NoError(t, os.MkdirAll(basePath, 0o755)) + require.NoError(t, os.WriteFile(arbitraryPath, []byte("keep"), 0o600)) + task.cleanupRejectedStatsResultFiles(ctx, resultForTextPath("../../not-stats")) + require.FileExists(t, arbitraryPath) + task.cleanupRejectedStatsResultFiles(ctx, resultForTextPath(arbitraryPath)) + require.FileExists(t, arbitraryPath) + + safePath := filepath.Join(basePath, "_stats/text_index.100/index") + require.NoError(t, os.MkdirAll(filepath.Dir(safePath), 0o755)) + require.NoError(t, os.WriteFile(safePath, []byte("keep"), 0o600)) + for _, unsafePath := range []string{ + "file://" + safePath, + filepath.Dir(safePath), + filepath.Dir(safePath) + "/../../not-stats", + } { + task.cleanupRejectedStatsResultFiles(ctx, resultForTextPath(unsafePath)) + require.FileExists(t, safePath, unsafePath) + } + + // A result for another segment is refused even when it reports a + // complete path in this task's stats directory. + task.cleanupRejectedStatsResultFiles(ctx, &workerpb.StatsResult{ + CollectionID: 1, + PartitionID: 2, + SegmentID: 4, + BaseManifest: packed.MarshalManifestPath(filepath.Join(root, "insert_log/1/2/4"), 1), + TextStatsLogs: map[int64]*datapb.TextIndexStats{ + 100: {Files: []string{safePath}}, + }, + }) + require.FileExists(t, safePath) + + // An unparsable or empty manifest is refused. + task.cleanupRejectedStatsResultFiles(ctx, &workerpb.StatsResult{ + CollectionID: 1, + PartitionID: 2, + SegmentID: 3, + BaseManifest: "not-a-manifest", + TextStatsLogs: map[int64]*datapb.TextIndexStats{ + 100: {Files: []string{safePath}}, + }, + }) + require.FileExists(t, safePath) + task.cleanupRejectedStatsResultFiles(ctx, &workerpb.StatsResult{ + CollectionID: 1, + PartitionID: 2, + SegmentID: 3, + BaseManifest: packed.MarshalManifestPath("", 1), + TextStatsLogs: map[int64]*datapb.TextIndexStats{ + 100: {Files: []string{safePath}}, + }, + }) + require.FileExists(t, safePath) +} + +type rootedStatsChunkManager struct { + storage.ChunkManager + rootPath string + removed []string +} + +func (m *rootedStatsChunkManager) RootPath() string { return m.rootPath } + +func (m *rootedStatsChunkManager) MultiRemove(_ context.Context, filePaths []string) error { + m.removed = append(m.removed, filePaths...) + return nil +} + +// The manifest base is a complete object key; cleanup deletes exactly the keys +// the worker reported under it and does not second-guess them against the +// configured root (a manifest written under an earlier minio.rootPath is still +// authoritative for where its files are). +func TestCleanupRejectedStatsResultFiles_RemoteUsesManifestKeysAsIs(t *testing.T) { + cm := &rootedStatsChunkManager{rootPath: "files"} + task := &statsTask{ + StatsTask: &indexpb.StatsTask{CollectionID: 1, PartitionID: 2, SegmentID: 3}, + meta: &meta{chunkManager: cm}, + } + result := func(basePath string) *workerpb.StatsResult { + return &workerpb.StatsResult{ + CollectionID: 1, + PartitionID: 2, + SegmentID: 3, + BaseManifest: packed.MarshalManifestPath(basePath, 1), + TextStatsLogs: map[int64]*datapb.TextIndexStats{ + 100: {Files: []string{basePath + "/_stats/text_index.100/index"}}, + }, + } + } + + task.cleanupRejectedStatsResultFiles(context.Background(), result("other/insert_log/1/2/3")) + require.Equal(t, []string{"other/insert_log/1/2/3/_stats/text_index.100/index"}, cm.removed) + + cm.removed = nil + task.cleanupRejectedStatsResultFiles(context.Background(), result("files/insert_log/1/2/3")) + require.Equal(t, []string{"files/insert_log/1/2/3/_stats/text_index.100/index"}, cm.removed) + + cm.rootPath = "" + cm.removed = nil + task.cleanupRejectedStatsResultFiles(context.Background(), result("insert_log/1/2/3")) + require.Equal(t, []string{"insert_log/1/2/3/_stats/text_index.100/index"}, cm.removed) + for _, base := range []string{"files/./insert_log/1/2/3", "files//insert_log/1/2/3", "files/tmp/../insert_log/1/2/3"} { + cm.removed = nil + task.cleanupRejectedStatsResultFiles(context.Background(), result(base)) + require.Equal(t, []string{base + "/_stats/text_index.100/index"}, cm.removed) + } + + // Wrong segment identity and URI bases are still refused. + for _, base := range []string{"files/insert_log/1/2/4", "s3://bucket/files/insert_log/1/2/3"} { + cm.removed = nil + task.cleanupRejectedStatsResultFiles(context.Background(), result(base)) + require.Empty(t, cm.removed, base) + } +} + +func TestCleanupRejectedStatsResultFiles_RemoteRejectsIncompleteOrUnsafeTextPaths(t *testing.T) { + for _, basePath := range []string{"files/insert_log/1/2/3", "insert_log/1/2/3"} { + t.Run(basePath, func(t *testing.T) { + statsDir := path.Join(basePath, "_stats/text_index.100") + completeFile := path.Join(statsDir, "index") + for _, file := range []string{ + "index", + "./index", + "task/index", + "../index", + "/" + completeFile, + "s3://bucket/" + completeFile, + path.Join(basePath, "not-stats"), + path.Join(basePath, "_stats/text_index.1000/index"), + statsDir, + path.Join(statsDir, "../text_index.101/index"), + } { + t.Run(file, func(t *testing.T) { + cm := &rootedStatsChunkManager{} + task := &statsTask{ + StatsTask: &indexpb.StatsTask{CollectionID: 1, PartitionID: 2, SegmentID: 3}, + meta: &meta{chunkManager: cm}, + } + result := &workerpb.StatsResult{ + CollectionID: 1, + PartitionID: 2, + SegmentID: 3, + BaseManifest: packed.MarshalManifestPath(basePath, 1), + TextStatsLogs: map[int64]*datapb.TextIndexStats{ + 100: {Files: []string{completeFile, file}}, + }, + } + + _, files, err := collectRejectedStatsResultFiles(result, "files", 1, 2, 3) + require.Error(t, err) + require.Empty(t, files) + task.cleanupRejectedStatsResultFiles(context.Background(), result) + require.Empty(t, cm.removed) + }) + } + }) + } +} + +func TestCleanupRejectedStatsResultFiles_RemoteRejectsAbsoluteManifestBase(t *testing.T) { + cm := &rootedStatsChunkManager{rootPath: "files"} + task := &statsTask{ + StatsTask: &indexpb.StatsTask{CollectionID: 1, PartitionID: 2, SegmentID: 3}, + meta: &meta{chunkManager: cm}, + } + basePath := "/files/insert_log/1/2/3" + task.cleanupRejectedStatsResultFiles(context.Background(), &workerpb.StatsResult{ + CollectionID: 1, + PartitionID: 2, + SegmentID: 3, + BaseManifest: packed.MarshalManifestPath(basePath, 1), + TextStatsLogs: map[int64]*datapb.TextIndexStats{ + 100: {Files: []string{path.Join(basePath, "_stats/text_index.100/index")}}, + }, + }) + require.Empty(t, cm.removed) +} + +func TestCleanupRejectedStatsResultFiles_RejectsInvalidJSONDescriptors(t *testing.T) { + basePath := "files/insert_log/1/2/3" + for _, descriptor := range []string{ + "/tmp/index", + } { + t.Run(descriptor, func(t *testing.T) { + cm := &rootedStatsChunkManager{rootPath: "files"} + task := &statsTask{ + StatsTask: &indexpb.StatsTask{CollectionID: 1, PartitionID: 2, SegmentID: 3}, + meta: &meta{chunkManager: cm}, + } + result := &workerpb.StatsResult{ + CollectionID: 1, + PartitionID: 2, + SegmentID: 3, + BaseManifest: packed.MarshalManifestPath(basePath, 1), + TextStatsLogs: map[int64]*datapb.TextIndexStats{ + 100: {Files: []string{path.Join(basePath, "_stats/text_index.100/index")}}, + }, + JsonKeyStatsLogs: map[int64]*datapb.JsonKeyStats{ + 101: {Files: []string{"shared_key_index/index", descriptor}}, + }, + } + _, files, err := collectRejectedStatsResultFiles(result, "files", 1, 2, 3) + require.Error(t, err) + require.Empty(t, files) + task.cleanupRejectedStatsResultFiles(context.Background(), result) + require.Empty(t, cm.removed) + }) + } +} + +func TestCollectRejectedStatsResultFiles_LocalJSONDirectoryBoundary(t *testing.T) { + basePath := filepath.Join(t.TempDir(), "insert_log/1/2/3") + for _, descriptor := range []string{"../index", "nested/../../index", ".", ".."} { + t.Run(descriptor, func(t *testing.T) { + result := &workerpb.StatsResult{ + BaseManifest: packed.MarshalManifestPath(basePath, 1), + JsonKeyStatsLogs: map[int64]*datapb.JsonKeyStats{ + 101: {Files: []string{descriptor}}, + }, + } + _, files, err := collectRejectedStatsResultFiles(result, "", 1, 2, 3) + require.Error(t, err) + require.Empty(t, files) + }) + } +} + +func TestCollectRejectedStatsResultFiles_ManifestBaseIdentity(t *testing.T) { + for _, root := range []string{"files", t.TempDir()} { + t.Run(root, func(t *testing.T) { + base := root + "/./insert_log/1/2/3" + result := &workerpb.StatsResult{ + BaseManifest: packed.MarshalManifestPath(base, 1), + Manifest: packed.MarshalManifestPath(path.Clean(base), 2), + JsonKeyStatsLogs: map[int64]*datapb.JsonKeyStats{ + 101: {Files: []string{"index"}}, + }, + } + _, files, err := collectRejectedStatsResultFiles(result, root, 1, 2, 3) + if path.IsAbs(root) { + require.NoError(t, err) + require.Equal(t, []string{base + "/_stats/json_stats.101/index"}, files) + } else { + // Equivalent filesystem spellings are distinct remote objects. + require.Error(t, err) + require.Empty(t, files) + } + }) + } +} + +func TestCollectRejectedStatsResultFiles_PreservesFileKeys(t *testing.T) { + for _, root := range []string{"files", "", t.TempDir()} { + for _, manifestLayout := range []bool{false, true} { + t.Run(fmt.Sprintf("root=%s/manifest=%t", root, manifestLayout), func(t *testing.T) { + basePath := path.Join(root, "insert_log/1/2/3") + textDir := metautil.BuildTextIndexPrefix(root, 10, 1, 1, 2, 3, 100) + jsonDir := metautil.BuildJSONKeyStatsPrefix(root, 2, 10, 1, 1, 2, 3, 101) + manifest := "" + if manifestLayout { + manifest = packed.MarshalManifestPath(basePath, 1) + textDir = basePath + "/_stats/text_index.100" + jsonDir = basePath + "/_stats/json_stats.101" + } + names := []string{"./index", "nested//index", "nested/../index", "s3://bucket/index"} + if !path.IsAbs(root) { + // These are literal object names, not filesystem traversal. + names = append(names, "../index", ".", "..") + } + textFiles := make([]string, 0, len(names)) + want := make([]string, 0, 2*len(names)) + for _, name := range names { + textFiles = append(textFiles, textDir+"/"+name) + want = append(want, textDir+"/"+name, jsonDir+"/"+name) + } + result := &workerpb.StatsResult{ + BaseManifest: manifest, + TextStatsLogs: map[int64]*datapb.TextIndexStats{ + 100: {BuildID: 10, Version: 1, Files: append(textFiles, textFiles[0])}, + }, + JsonKeyStatsLogs: map[int64]*datapb.JsonKeyStats{ + 101: {BuildID: 10, Version: 1, JsonKeyStatsDataFormat: 2, Files: names}, + }, + } + _, files, err := collectRejectedStatsResultFiles(result, root, 1, 2, 3) + require.NoError(t, err) + require.ElementsMatch(t, want, files) + require.Equal(t, names, result.GetJsonKeyStatsLogs()[101].GetFiles()) + }) + } + } +} + +func TestCleanupRejectedStatsResultFiles_LocalAcceptsEquivalentPaths(t *testing.T) { + ctx := context.Background() + root := t.TempDir() + cm := storage.NewLocalChunkManager(objectstorage.RootPath(root)) + basePath := filepath.Join(root, "insert_log/1/2/3") + statsDir := basePath + "/_stats/text_index.100" + require.NoError(t, os.MkdirAll(statsDir+"/nested", 0o755)) + task := &statsTask{ + StatsTask: &indexpb.StatsTask{CollectionID: 1, PartitionID: 2, SegmentID: 3}, + meta: &meta{chunkManager: cm}, + } + for _, name := range []string{"./index", "nested//index", "nested/../index"} { + file := statsDir + "/" + name + require.NoError(t, cm.Write(ctx, file, []byte("stale"))) + task.cleanupRejectedStatsResultFiles(ctx, &workerpb.StatsResult{ + CollectionID: 1, + PartitionID: 2, + SegmentID: 3, + BaseManifest: packed.MarshalManifestPath(basePath, 1), + TextStatsLogs: map[int64]*datapb.TextIndexStats{100: {Files: []string{file}}}, + }) + require.NoFileExists(t, file) + } +} + type statsTaskSuite struct { suite.Suite mt *meta @@ -67,6 +555,8 @@ type mockeyChunkManager struct { storage.ChunkManager } +func (m *mockeyChunkManager) RootPath() string { return "" } + func Test_statsTaskSuite(t *testing.T) { suite.Run(t, new(statsTaskSuite)) } @@ -897,7 +1387,7 @@ func (s *statsTaskSuite) TestClassifyStatsManifestCommitError() { func (s *statsTaskSuite) TestCollectRejectedStatsResultFiles() { baseManifest := `{"base_path":"files/insert_log/1/2/1179","ver":2}` s.Run("collect text and json stats files", func() { - files, err := collectRejectedStatsResultFiles(&workerpb.StatsResult{ + basePath, files, err := collectRejectedStatsResultFiles(&workerpb.StatsResult{ BaseManifest: baseManifest, TextStatsLogs: map[int64]*datapb.TextIndexStats{ 101: { @@ -909,9 +1399,10 @@ func (s *statsTaskSuite) TestCollectRejectedStatsResultFiles() { Files: []string{"shared_key_index/.managed.json_0"}, }, }, - }) + }, "files", 1, 2, 1179) s.NoError(err) + s.Equal("files/insert_log/1/2/1179", basePath) s.ElementsMatch([]string{ "files/insert_log/1/2/1179/_stats/text_index.101/tokenizer.json", "files/insert_log/1/2/1179/_stats/json_stats.102/shared_key_index/.managed.json_0", @@ -919,7 +1410,8 @@ func (s *statsTaskSuite) TestCollectRejectedStatsResultFiles() { }) s.Run("deduplicate text stats files without json stats", func() { - files, err := collectRejectedStatsResultFiles(&workerpb.StatsResult{ + basePath, files, err := collectRejectedStatsResultFiles(&workerpb.StatsResult{ + BaseManifest: baseManifest, TextStatsLogs: map[int64]*datapb.TextIndexStats{ 101: { Files: []string{ @@ -929,39 +1421,123 @@ func (s *statsTaskSuite) TestCollectRejectedStatsResultFiles() { }, }, }, - }) + }, "files", 1, 2, 1179) s.NoError(err) + s.Equal("files/insert_log/1/2/1179", basePath) s.Equal([]string{"files/insert_log/1/2/1179/_stats/text_index.101/tokenizer.json"}, files) }) - s.Run("json stats without manifest returns typed error", func() { - files, err := collectRejectedStatsResultFiles(&workerpb.StatsResult{ + // Without a manifest the legacy prefix is rebuilt from the build ID and task + // version, so a result that reports neither does not say where its files + // live. Refuse rather than addressing build 0 / version 0. + s.Run("stats without manifest or build coordinates are refused", func() { + basePath, files, err := collectRejectedStatsResultFiles(&workerpb.StatsResult{ JsonKeyStatsLogs: map[int64]*datapb.JsonKeyStats{ 102: { Files: []string{"shared_key_index/.managed.json_0"}, }, }, - }) + }, "files", 1, 2, 1179) + s.Empty(basePath) s.Empty(files) - s.ErrorIs(err, merr.ErrServiceInternal) - s.Contains(err.Error(), "manifest is empty for rejected json stats result") + s.ErrorIs(err, merr.ErrDataIntegrity) + s.Contains(err.Error(), "carries no build ID or version") }) s.Run("json stats with invalid manifest returns error", func() { - files, err := collectRejectedStatsResultFiles(&workerpb.StatsResult{ + basePath, files, err := collectRejectedStatsResultFiles(&workerpb.StatsResult{ BaseManifest: "invalid", JsonKeyStatsLogs: map[int64]*datapb.JsonKeyStats{ 102: { Files: []string{"shared_key_index/.managed.json_0"}, }, }, - }) + }, "files", 1, 2, 1179) + s.Empty(basePath) s.Empty(files) s.Error(err) }) + + // A StorageV2 stats result has no manifest at all: the worker writes TEXT and + // JSON stats to the traditional prefixes under the storage root. Requiring a + // manifest would skip cleanup and leak those files. + s.Run("legacy text stats without manifest are collected", func() { + basePath, files, err := collectRejectedStatsResultFiles(&workerpb.StatsResult{ + TextStatsLogs: map[int64]*datapb.TextIndexStats{ + 101: { + BuildID: 77, + Version: 3, + Files: []string{ + "files/text_log/77/3/1/2/1179/101/tokenizer.json", + "files/text_log/77/3/1/2/1179/101/meta.json", + }, + }, + }, + }, "files", 1, 2, 1179) + + s.NoError(err) + s.Equal("files", basePath) + s.ElementsMatch([]string{ + "files/text_log/77/3/1/2/1179/101/tokenizer.json", + "files/text_log/77/3/1/2/1179/101/meta.json", + }, files) + }) + + s.Run("legacy json stats without manifest resolve relative names", func() { + basePath, files, err := collectRejectedStatsResultFiles(&workerpb.StatsResult{ + JsonKeyStatsLogs: map[int64]*datapb.JsonKeyStats{ + 102: { + BuildID: 77, + Version: 3, + JsonKeyStatsDataFormat: 2, + Files: []string{"shared_key_index/.managed.json_0"}, + }, + }, + }, "files", 1, 2, 1179) + + s.NoError(err) + s.Equal("files", basePath) + s.Equal([]string{"files/json_stats/2/77/3/1/2/1179/102/shared_key_index/.managed.json_0"}, files) + }) + + // Local storage keys are absolute, and the legacy prefix is rebuilt from the + // storage root rather than guessed against the process working directory. + s.Run("legacy stats under an absolute local root", func() { + basePath, files, err := collectRejectedStatsResultFiles(&workerpb.StatsResult{ + TextStatsLogs: map[int64]*datapb.TextIndexStats{ + 101: { + BuildID: 77, + Version: 3, + Files: []string{"/var/lib/milvus/data/text_log/77/3/1/2/1179/101/tokenizer.json"}, + }, + }, + }, "/var/lib/milvus/data", 1, 2, 1179) + + s.NoError(err) + s.Equal("/var/lib/milvus/data", basePath) + s.Equal([]string{"/var/lib/milvus/data/text_log/77/3/1/2/1179/101/tokenizer.json"}, files) + }) + + // The segment identity still comes from the task, so a file written under + // another segment's legacy prefix is refused rather than deleted. + s.Run("legacy stats outside the task segment prefix are refused", func() { + basePath, files, err := collectRejectedStatsResultFiles(&workerpb.StatsResult{ + TextStatsLogs: map[int64]*datapb.TextIndexStats{ + 101: { + BuildID: 77, + Version: 3, + Files: []string{"files/text_log/77/3/1/2/9999/101/tokenizer.json"}, + }, + }, + }, "files", 1, 2, 1179) + + s.Empty(basePath) + s.Empty(files) + s.ErrorIs(err, merr.ErrDataIntegrity) + }) } func (s *statsTaskSuite) TestQueryTaskOnWorkerDiscardsStaleStatsResult() { diff --git a/internal/datanode/compactor/bump_schema_version_compactor.go b/internal/datanode/compactor/bump_schema_version_compactor.go index 480e68a5c0..b6eb3ae930 100644 --- a/internal/datanode/compactor/bump_schema_version_compactor.go +++ b/internal/datanode/compactor/bump_schema_version_compactor.go @@ -631,12 +631,36 @@ func (t *bumpSchemaVersionCompactionTask) runFullSchemaRewrite(existingFields ma _, ttlFieldPhysicallyPresent := existingFields[ttlFieldID] sourceHasTTLField := ttlFieldID >= common.StartOfUserFieldID && ttlFieldPhysicallyPresent preMaterializeFilter := len(delta) > 0 || t.plan.GetCollectionTtl() > 0 || sourceHasTTLField - reader, _, err := newCompactionSegmentRecordReaderWithFields(t.ctx, segment, t.plan.GetSchema(), t.compactionParams.StorageConfig, existingFields, - storage.WithCollectionID(collectionID), - storage.WithVersion(segment.GetStorageVersion()), - storage.WithDownloader(t.chunkManager.MultiRead), - storage.WithStorageConfig(t.compactionParams.StorageConfig), - ) + + // Prepare LOB (TEXT) handling before opening the source reader: a legacy + // partition namespace needs a TEXT-decoding reader rooted at the source. + if err := t.initLOBCompactionContext(t.ctx); err != nil { + return nil, err + } + textDecodeConfigs, err := t.lobContext.GetSourceTextColumnConfigs(segment.GetManifest()) + if err != nil { + return nil, err + } + var reader storage.RecordReader + if len(textDecodeConfigs) > 0 { + reader, err = storage.NewTextDecodedManifestRecordReader( + t.ctx, + segment.GetManifest(), + compactionReadSchema(t.plan.GetSchema(), existingFields), + textDecodeConfigs, + storage.WithPresentFields(existingFields), + storage.WithCollectionID(collectionID), + storage.WithVersion(segment.GetStorageVersion()), + storage.WithStorageConfig(t.compactionParams.StorageConfig), + ) + } else { + reader, _, err = newCompactionSegmentRecordReaderWithFields(t.ctx, segment, t.plan.GetSchema(), t.compactionParams.StorageConfig, existingFields, + storage.WithCollectionID(collectionID), + storage.WithVersion(segment.GetStorageVersion()), + storage.WithDownloader(t.chunkManager.MultiRead), + storage.WithStorageConfig(t.compactionParams.StorageConfig), + ) + } if err != nil { return nil, err } @@ -648,14 +672,6 @@ func (t *bumpSchemaVersionCompactionTask) runFullSchemaRewrite(existingFields ma } defer materializer.Close() - // Prepare LOB (TEXT) handling: a schema bump keeps existing TEXT LOB data - // unchanged, so REUSE_ALL — the writer preserves the encoded LOB reference - // bytes (WithTextRefsAsBinary) and the source LOB files are merged into the - // output manifest afterwards (applyLOBCompaction). Mirrors sort compaction. - if err := t.initLOBCompactionContext(t.ctx); err != nil { - return nil, err - } - alloc := allocator.NewLocalAllocator(t.plan.GetPreAllocatedLogIDs().GetBegin(), t.plan.GetPreAllocatedLogIDs().GetEnd()) writerOpts := []storage.RwOption{ storage.WithUploader(func(ctx context.Context, kvs map[string][]byte) error { @@ -666,6 +682,19 @@ func (t *bumpSchemaVersionCompactionTask) runFullSchemaRewrite(existingFields ma storage.WithCollectionID(collectionID), storage.WithUseLoonFFI(t.compactionParams.UseLoonFFI), } + if t.lobContext != nil && t.lobContext.ShouldRewriteAnyField() { + lobBasePath := storage.SegmentPartitionBasePath( + t.compactionParams.StorageConfig.GetRootPath(), collectionID, segment.GetPartitionID()) + textColumnConfigs := t.lobContext.GetTextColumnConfigs( + lobBasePath, + t.compactionParams.TextInlineThreshold, + t.compactionParams.TextMaxLobFileBytes, + t.compactionParams.TextFlushThresholdBytes, + ) + if len(textColumnConfigs) > 0 { + writerOpts = append(writerOpts, storage.WithTextColumnConfigs(textColumnConfigs)) + } + } if t.lobContext != nil && t.lobContext.HasReuseAllFields() { // Existing TEXT columns arrive from the reader as encoded binary LOB refs; // tell the writer to write them via the physical binary schema unchanged. @@ -838,11 +867,9 @@ func (t *bumpSchemaVersionCompactionTask) runFullSchemaRewrite(existingFields ma }, nil } -// initLOBCompactionContext prepares REUSE_ALL LOB handling for the (single) input -// segment's TEXT columns. A schema bump is 1->1 and never changes existing TEXT LOB -// data, so all TEXT fields are forced REUSE_ALL (GetForcedStrategy): the source LOB -// files stay in place and only their references are merged into the output manifest. -// No-op when the schema has no TEXT fields. Mirrors sort compaction. +// initLOBCompactionContext prepares LOB handling for the full-rewrite path. A +// schema bump normally reuses references, unless its new output segment has a +// different partition base from the source manifest. func (t *bumpSchemaVersionCompactionTask) initLOBCompactionContext(ctx context.Context) error { textFieldIDs := compaction.GetTEXTFieldIDsFromSchema(t.plan.GetSchema()) if len(textFieldIDs) == 0 { @@ -865,9 +892,20 @@ func (t *bumpSchemaVersionCompactionTask) initLOBCompactionContext(ctx context.C // schema-bump is always 1 source -> 1 output; forced REUSE_ALL. t.lobContext.SetCompactionType(datapb.CompactionType_BumpSchemaVersionCompaction, 1, 1) t.lobContext.ComputeStrategies(textFieldIDs, t.compactionParams.LOBHoleRatioThreshold) - mlog.Info(ctx, "schema bump: initialized LOB compaction context (REUSE_ALL)", + outputPartitionBase := storage.SegmentPartitionBasePath( + t.compactionParams.StorageConfig.GetRootPath(), segment.GetCollectionID(), segment.GetPartitionID()) + partitionBaseMismatch, err := compaction.LOBSourcePartitionBaseMismatch(sourceManifests, outputPartitionBase) + if err != nil { + return err + } + if partitionBaseMismatch { + t.lobContext.ForceRewriteAllAcrossPartitionBases(textFieldIDs) + } + mlog.Info(ctx, "schema bump: initialized LOB compaction context", mlog.Int64("planID", t.GetPlanID()), mlog.FieldSegmentID(segment.GetSegmentID()), + mlog.Bool("partitionBaseMismatch", partitionBaseMismatch), + mlog.String("outputPartitionBase", outputPartitionBase), mlog.Int64s("textFieldIDs", textFieldIDs), ) return nil diff --git a/internal/datanode/compactor/bump_schema_version_compactor_test.go b/internal/datanode/compactor/bump_schema_version_compactor_test.go index 80f5f029f8..3d2f399e1b 100644 --- a/internal/datanode/compactor/bump_schema_version_compactor_test.go +++ b/internal/datanode/compactor/bump_schema_version_compactor_test.go @@ -954,6 +954,9 @@ func (s *BumpSchemaVersionCompactionTaskSuite) buildTextLOBTask(withTextField bo OutputFieldNames: []string{"sparse"}, OutputFieldIds: []int64{102}, }} } + params := compaction.GenParams() + manifest := packed.MarshalManifestPath( + storage.SegmentManifestBasePath(params.StorageConfig.GetRootPath(), 1, 1, 100), 1) plan := &datapb.CompactionPlan{ PlanID: 999, Type: datapb.CompactionType_BumpSchemaVersionCompaction, @@ -961,10 +964,9 @@ func (s *BumpSchemaVersionCompactionTaskSuite) buildTextLOBTask(withTextField bo TotalRows: 3, SegmentBinlogs: []*datapb.CompactionSegmentBinlogs{{ CollectionID: 1, PartitionID: 1, SegmentID: 100, - StorageVersion: storage.StorageV3, Manifest: "manifest", + StorageVersion: storage.StorageV3, Manifest: manifest, }}, } - params := compaction.GenParams() params.StorageVersion = storage.StorageV3 return &bumpSchemaVersionCompactionTask{ ctx: context.Background(), @@ -973,6 +975,23 @@ func (s *BumpSchemaVersionCompactionTaskSuite) buildTextLOBTask(withTextField bo } } +func (s *BumpSchemaVersionCompactionTaskSuite) TestInitLOBCompactionContextRewritesAcrossPartitionBases() { + task := s.buildTextLOBTask(true) + task.plan.SegmentBinlogs[0].Manifest = packed.MarshalManifestPath( + task.compactionParams.StorageConfig.GetRootPath()+"/files/insert_log/1/1/100", 1) + collectPatch := mockey.Mock(compaction.CollectLobFilesFromManifests).Return( + map[int64][]packed.LobFileInfo{100: {{FieldID: 101, TotalRows: 3, ValidRows: 3}}}, nil, + ).Build() + defer collectPatch.UnPatch() + + err := task.initLOBCompactionContext(context.Background()) + s.NoError(err) + s.Require().NotNil(task.lobContext) + s.False(task.lobContext.HasReuseAllFields()) + s.True(task.lobContext.ShouldRewriteAnyField()) + s.Equal(compaction.LOBStrategyRewriteAll, task.lobContext.GetStrategy(101)) +} + // TestInitLOBCompactionContextTextFieldReuseAll: a schema-bump on a segment with a // TEXT field forces REUSE_ALL for that field (never REWRITE_ALL), so its existing // LOB data is carried by reference. diff --git a/internal/datanode/compactor/bump_schema_version_ut_test.go b/internal/datanode/compactor/bump_schema_version_ut_test.go index 6a1243d896..57b25f3573 100644 --- a/internal/datanode/compactor/bump_schema_version_ut_test.go +++ b/internal/datanode/compactor/bump_schema_version_ut_test.go @@ -25,6 +25,7 @@ import ( "os" "path" "path/filepath" + "strconv" "strings" "testing" "time" @@ -117,6 +118,7 @@ type fixtureSpec struct { commitTs uint64 fillTextAnalyzer bool // decorate the base text field with multi_analyzer_params (by_field=lang) textLOBFieldID int64 // >0: write this source TEXT field through the LOB-aware writer + legacyNamespace bool // write the source under localStorage.path/files instead of the canonical root } type fixtureOpt func(*fixtureSpec) @@ -167,6 +169,10 @@ func withTextLOBSource(fieldID int64) fixtureOpt { return func(s *fixtureSpec) { s.textLOBFieldID = fieldID } } +func withLegacySourceNamespace() fixtureOpt { + return func(s *fixtureSpec) { s.legacyNamespace = true } +} + type bumpFixture struct { t *testing.T task *bumpSchemaVersionCompactionTask @@ -270,14 +276,20 @@ func buildBumpFixture(t *testing.T, opts ...fixtureOpt) *bumpFixture { compAlloc := NewCompactionAllocator(segIDAlloc, logIDAlloc) params := compaction.GenParams() params.StorageVersion = storage.StorageV3 + sourceStorageConfig := params.StorageConfig + if spec.legacyNamespace { + sourceStorageConfig = proto.Clone(params.StorageConfig).(*indexpb.StorageConfig) + sourceStorageConfig.RootPath = path.Join(params.StorageConfig.GetRootPath(), "files") + } fixtureWriterOpts := []storage.RwOption{ - storage.WithStorageConfig(params.StorageConfig), + storage.WithStorageConfig(sourceStorageConfig), storage.WithVersion(storage.StorageV3), } if spec.textLOBFieldID > 0 { - lobBasePath := path.Join(params.StorageConfig.GetRootPath(), - common.SegmentInsertLogPath, metautil.JoinIDPath(CollectionID, PartitionID)) + lobBasePath := path.Join( + storage.SegmentPartitionBasePath(sourceStorageConfig.GetRootPath(), CollectionID, PartitionID), + "lobs", strconv.FormatInt(spec.textLOBFieldID, 10)) fixtureWriterOpts = append(fixtureWriterOpts, storage.WithTextColumnConfigs([]packed.TextColumnConfig{{ FieldID: spec.textLOBFieldID, LobBasePath: lobBasePath, @@ -2111,6 +2123,44 @@ func readTextRefs(t *testing.T, fix *bumpFixture, manifest string, textID int64) return refs } +func readLOBPayloads(t *testing.T, fix *bumpFixture, manifest string, textID int64) []string { + lobFiles, err := packed.GetManifestLobFiles(manifest, fix.cfg) + require.NoError(t, err) + var fragments []packed.Fragment + for _, lobFile := range lobFiles { + if lobFile.FieldID != textID { + continue + } + fragments = append(fragments, packed.Fragment{ + FilePath: lobFile.Path, + StartRow: 0, + EndRow: lobFile.TotalRows, + RowCount: lobFile.TotalRows, + }) + } + require.NotEmpty(t, fragments) + schema := arrow.NewSchema([]arrow.Field{{Name: "text_data", Type: arrow.BinaryTypes.String}}, nil) + reader, err := packed.NewFFIPackedReaderWithFragments( + []string{"text_data"}, "vortex", fragments, schema, []string{"text_data"}, + 1<<20, fix.cfg, nil, packed.ExternalReaderContext{}) + require.NoError(t, err) + defer func() { require.NoError(t, reader.Close()) }() + + var values []string + for { + record, readErr := reader.ReadNext() + if readErr == sio.EOF { + break + } + require.NoError(t, readErr) + column := record.Column(0).(*array.String) + for i := 0; i < column.Len(); i++ { + values = append(values, strings.Clone(column.Value(i))) + } + } + return values +} + // [LOB1][S0] additive keeps TEXT LOB data dereferenceable: every row's LOB // reference bytes survive the bump unchanged AND the referenced .vx blob file // stays byte-identical — together this proves content round-trip without @@ -2178,6 +2228,63 @@ func TestBumpUTFullRewriteTextLOBKeptRowsSurvive(t *testing.T) { require.Equal(t, preLob, listLobFiles(t, fix), "LOB blob files must stay byte-identical") } +// [LOB3][S0] a pre-3.0.2 local source can remain under the legacy `files` +// namespace after upgrade. A full schema rewrite creates a canonical output +// segment, so byte-reusing its TEXT references would make the output reader look +// for the legacy UUID under the wrong partition. The compactor must decode the +// source payload and write a new canonical LOB file instead. +func TestBumpUTFullRewriteTextLOBRewritesAcrossPartitionNamespaces(t *testing.T) { + setupBumpUTEnv(t) + const textID = int64(105) + const droppedID = int64(107) + const rows = 6 + fix := buildBumpFixture(t, withRows(rows), + withSourceFields( + &schemapb.FieldSchema{FieldID: textID, Name: "big_text", DataType: schemapb.DataType_Text}, + &schemapb.FieldSchema{FieldID: droppedID, Name: "dropped", DataType: schemapb.DataType_Int64}, + ), + withFillValue(func(i int, ts uint64, v map[int64]any) { + v[textID] = bumpFxLobText(i) + v[droppedID] = int64(i) + }), + withTextLOBSource(textID), + withLegacySourceNamespace(), + withTargetDroppedField(droppedID), + ) + + sourceBase, _, err := packed.UnmarshalManifestPath(fix.sourceManifest) + require.NoError(t, err) + require.Contains(t, sourceBase, "/files/insert_log/") + sourceRefs := readTextRefs(t, fix, fix.sourceManifest, textID) + + segment := runCompact(t, fix) + outputBase, _, err := packed.UnmarshalManifestPath(segment.GetManifest()) + require.NoError(t, err) + require.Equal(t, + storage.SegmentManifestBasePath(fix.cfg.GetRootPath(), CollectionID, PartitionID, segment.GetSegmentID()), + outputBase) + outputRefs := readTextRefs(t, fix, segment.GetManifest(), textID) + require.NotEqual(t, sourceRefs, outputRefs, "cross-namespace rewrite must allocate new LOB references") + + gotPayloads := readLOBPayloads(t, fix, segment.GetManifest(), textID) + require.Len(t, gotPayloads, rows) + for i := 0; i < rows; i++ { + require.Equal(t, bumpFxLobText(i), gotPayloads[i]) + } + for _, lobFile := range mustManifestLobFiles(t, fix, segment.GetManifest()) { + if lobFile.FieldID == textID { + require.NotContains(t, lobFile.Path, "/files/insert_log/") + require.Contains(t, lobFile.Path, "/insert_log/") + } + } +} + +func mustManifestLobFiles(t *testing.T, fix *bumpFixture, manifest string) []packed.LobFileInfo { + lobFiles, err := packed.GetManifestLobFiles(manifest, fix.cfg) + require.NoError(t, err) + return lobFiles +} + // [ST2][S1] full rewrite with deletes + BM25 materialization: the committed // stats ledger must accumulate ONLY the surviving rows — filtered rows leaking // into stats is invisible to every other oracle in the suite. diff --git a/internal/datanode/compactor/clustering_compactor.go b/internal/datanode/compactor/clustering_compactor.go index f4b73157fe..8d24dfa97d 100644 --- a/internal/datanode/compactor/clustering_compactor.go +++ b/internal/datanode/compactor/clustering_compactor.go @@ -584,7 +584,11 @@ func (t *clusteringCompactionTask) mappingSegment( } } - rr, existingFields, err := newCompactionSegmentRecordReader(ctx, segment, t.plan.Schema, t.compactionParams.StorageConfig, + textDecodeConfigs, err := t.lobContext.GetSourceTextColumnConfigs(segment.GetManifest()) + if err != nil { + return err + } + rr, existingFields, err := newTextDecodedCompactionSegmentRecordReader(ctx, segment, t.plan.Schema, t.compactionParams.StorageConfig, textDecodeConfigs, storage.WithDownloader(func(ctx context.Context, paths []string) ([][]byte, error) { return t.binlogIO.Download(ctx, paths) }), @@ -871,7 +875,11 @@ func (t *clusteringCompactionTask) scalarAnalyzeSegment( readSchema.Fields = selectedFields readSchema.StructArrayFields = nil - rr, _, err := newCompactionSegmentRecordReader(ctx, segment, readSchema, t.compactionParams.StorageConfig, + textDecodeConfigs, err := t.lobContext.GetSourceTextColumnConfigs(segment.GetManifest()) + if err != nil { + return nil, err + } + rr, _, err := newTextDecodedCompactionSegmentRecordReader(ctx, segment, readSchema, t.compactionParams.StorageConfig, textDecodeConfigs, storage.WithDownloader(func(ctx context.Context, paths []string) ([][]byte, error) { return t.binlogIO.Download(ctx, paths) }), @@ -1032,9 +1040,10 @@ func (t *clusteringCompactionTask) getWriterOpts() []storage.RwOption { } if t.lobContext != nil && t.lobContext.ShouldRewriteAnyField() { - // LOB base path at partition level: {root}/insert_log/{coll}/{part} - lobBasePath := path.Join(t.compactionParams.StorageConfig.GetRootPath(), - common.SegmentInsertLogPath, metautil.JoinIDPath(t.collectionID, t.partitionID)) + // LOB base path at partition level, sharing the manifest root rule so + // the reader (which reconstructs lobs/ from the manifest base) resolves + // the same files (see #53051). + lobBasePath := storage.SegmentPartitionBasePath(t.compactionParams.StorageConfig.GetRootPath(), t.collectionID, t.partitionID) textColumnConfigs := t.lobContext.GetTextColumnConfigs( lobBasePath, t.compactionParams.TextInlineThreshold, @@ -1088,6 +1097,25 @@ func (t *clusteringCompactionTask) initLOBCompactionContext(ctx context.Context) // compute strategies (will use forced REWRITE_ALL for all TEXT fields) t.lobContext.ComputeStrategies(textFieldIDs, t.compactionParams.LOBHoleRatioThreshold) + sourceManifests := make(map[int64]string) + for _, segment := range t.plan.GetSegmentBinlogs() { + if segment.GetManifest() != "" { + sourceManifests[segment.GetSegmentID()] = segment.GetManifest() + } + } + outputPartitionBase := storage.SegmentPartitionBasePath( + t.compactionParams.StorageConfig.GetRootPath(), t.collectionID, t.partitionID) + partitionBaseMismatch, err := compaction.LOBSourcePartitionBaseMismatch(sourceManifests, outputPartitionBase) + if err != nil { + return err + } + if partitionBaseMismatch { + t.lobContext.ForceRewriteAllAcrossPartitionBases(textFieldIDs) + mlog.Info(ctx, "forcing source-side TEXT LOB decoding across partition namespaces", + mlog.Int64("planID", t.GetPlanID()), + mlog.String("outputPartitionBase", outputPartitionBase), + mlog.Int64s("textFieldIDs", textFieldIDs)) + } // log strategy decisions for fieldID, decision := range t.lobContext.Decisions { diff --git a/internal/datanode/compactor/clustering_compactor_storage_v2_test.go b/internal/datanode/compactor/clustering_compactor_storage_v2_test.go index 2d17da7bce..b00aa6f62d 100644 --- a/internal/datanode/compactor/clustering_compactor_storage_v2_test.go +++ b/internal/datanode/compactor/clustering_compactor_storage_v2_test.go @@ -20,6 +20,7 @@ import ( "context" "fmt" "os" + "path/filepath" "testing" "time" @@ -57,9 +58,10 @@ func (s *ClusteringCompactionTaskStorageV2Suite) SetupTest() { } func (s *ClusteringCompactionTaskStorageV2Suite) TearDownTest() { - os.RemoveAll(paramtable.Get().LocalStorageCfg.Path.GetValue() + "insert_log") - os.RemoveAll(paramtable.Get().LocalStorageCfg.Path.GetValue() + "delta_log") - os.RemoveAll(paramtable.Get().LocalStorageCfg.Path.GetValue() + "stats_log") + localStoragePath := paramtable.Get().LocalStorageCfg.Path.GetValue() + os.RemoveAll(filepath.Join(localStoragePath, "insert_log")) + os.RemoveAll(filepath.Join(localStoragePath, "delta_log")) + os.RemoveAll(filepath.Join(localStoragePath, "stats_log")) s.ClusteringCompactionTaskSuite.TearDownTest() } diff --git a/internal/datanode/compactor/clustering_compactor_storage_v3_test.go b/internal/datanode/compactor/clustering_compactor_storage_v3_test.go index 0fb6633ba3..86c1ce44be 100644 --- a/internal/datanode/compactor/clustering_compactor_storage_v3_test.go +++ b/internal/datanode/compactor/clustering_compactor_storage_v3_test.go @@ -26,6 +26,7 @@ import ( "time" "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" "github.com/milvus-io/milvus-proto/go-api/v3/commonpb" @@ -55,6 +56,36 @@ func TestClusteringCompactionTaskStorageV3Suite(t *testing.T) { suite.Run(t, new(ClusteringCompactionTaskStorageV3Suite)) } +func TestClusteringInitLOBCompactionContextDecodesAcrossPartitionBases(t *testing.T) { + paramtable.Get().Init(paramtable.NewBaseTable()) + params := compaction.GenParams() + task := &clusteringCompactionTask{ + collectionID: 10, + partitionID: 20, + plan: &datapb.CompactionPlan{ + Type: datapb.CompactionType_ClusteringCompaction, + Schema: &schemapb.CollectionSchema{Fields: []*schemapb.FieldSchema{ + {FieldID: 101, Name: "text", DataType: schemapb.DataType_Text}, + }}, + SegmentBinlogs: []*datapb.CompactionSegmentBinlogs{{ + SegmentID: 1, + Manifest: packed.MarshalManifestPath( + params.StorageConfig.GetRootPath()+"/files/insert_log/10/20/1", 1), + }}, + }, + compactionParams: params, + } + + require.NoError(t, task.initLOBCompactionContext(context.Background())) + require.NotNil(t, task.lobContext) + require.True(t, task.lobContext.ShouldRewriteAnyField()) + require.True(t, task.lobContext.DecodeTextFromSource) + sourceConfigs, err := task.lobContext.GetSourceTextColumnConfigs(task.plan.GetSegmentBinlogs()[0].GetManifest()) + require.NoError(t, err) + require.Len(t, sourceConfigs, 1) + require.Contains(t, sourceConfigs[0].LobBasePath, "/files/insert_log/10/20/lobs/101") +} + type ClusteringCompactionTaskStorageV3Suite struct { ClusteringCompactionTaskSuite } @@ -190,7 +221,7 @@ func (s *ClusteringCompactionTaskStorageV3Suite) initStorageV3Segments(rows int, bfs := pkoracle.NewBloomFilterSet() k := metautil.JoinIDPath(CollectionID, PartitionID, segmentID) - basePath := path.Join(common.SegmentInsertLogPath, k) + basePath := path.Join(rootPath, common.SegmentInsertLogPath, k) manifestPath := packed.MarshalManifestPath(basePath, packed.ManifestEarliest) seg := metacache.NewSegmentInfo(&datapb.SegmentInfo{ @@ -395,7 +426,7 @@ func (s *MixCompactionTaskStorageV3Suite) initStorageV3Segments(rows int, segmen bfs := pkoracle.NewBloomFilterSet() k := metautil.JoinIDPath(CollectionID, PartitionID, segmentID) - basePath := path.Join(common.SegmentInsertLogPath, k) + basePath := path.Join(rootPath, common.SegmentInsertLogPath, k) manifestPath := packed.MarshalManifestPath(basePath, packed.ManifestEarliest) seg := metacache.NewSegmentInfo(&datapb.SegmentInfo{ @@ -443,7 +474,7 @@ func (s *MixCompactionTaskStorageV3Suite) initTextLOBStorageV3Segment(rows int, bfs := pkoracle.NewBloomFilterSet() k := metautil.JoinIDPath(CollectionID, PartitionID, segmentID) - basePath := path.Join(common.SegmentInsertLogPath, k) + basePath := path.Join(rootPath, common.SegmentInsertLogPath, k) manifestPath := packed.MarshalManifestPath(basePath, packed.ManifestEarliest) seg := metacache.NewSegmentInfo(&datapb.SegmentInfo{ diff --git a/internal/datanode/compactor/compactor_common.go b/internal/datanode/compactor/compactor_common.go index 8ea67ad929..79c25cd5b3 100644 --- a/internal/datanode/compactor/compactor_common.go +++ b/internal/datanode/compactor/compactor_common.go @@ -235,6 +235,32 @@ func newCompactionSegmentRecordReader(ctx context.Context, segment *datapb.Compa return newCompactionSegmentRecordReaderWithFields(ctx, segment, schema, storageConfig, existingFields, opts...) } +func newTextDecodedCompactionSegmentRecordReader( + ctx context.Context, + segment *datapb.CompactionSegmentBinlogs, + schema *schemapb.CollectionSchema, + storageConfig *indexpb.StorageConfig, + textColumnConfigs []packed.TextColumnConfig, + opts ...storage.RwOption, +) (storage.RecordReader, map[int64]struct{}, error) { + if segment.GetManifest() == "" || len(textColumnConfigs) == 0 { + return newCompactionSegmentRecordReader(ctx, segment, schema, storageConfig, opts...) + } + existingFields, err := compactionSegmentStorageFields(segment, storageConfig) + if err != nil { + return nil, nil, err + } + readSchema := compactionReadSchema(schema, existingFields) + reader, err := storage.NewTextDecodedManifestRecordReader( + ctx, + segment.GetManifest(), + readSchema, + textColumnConfigs, + append(opts, storage.WithPresentFields(existingFields))..., + ) + return reader, existingFields, err +} + func compactionSegmentStorageFields(segment *datapb.CompactionSegmentBinlogs, storageConfig *indexpb.StorageConfig) (map[int64]struct{}, error) { if segment.GetManifest() != "" { return packed.GetManifestFieldIDs(segment.GetManifest(), storageConfig) diff --git a/internal/datanode/compactor/merge_sort.go b/internal/datanode/compactor/merge_sort.go index 06c789e9c2..2cc661a56a 100644 --- a/internal/datanode/compactor/merge_sort.go +++ b/internal/datanode/compactor/merge_sort.go @@ -72,7 +72,11 @@ func mergeSortMultipleSegments(ctx context.Context, segmentFilters := make([]compaction.EntityFilter, len(binlogs)) for i, s := range binlogs { - reader, existingFields, err := newCompactionSegmentRecordReader(ctx, s, plan.GetSchema(), compactionParams.StorageConfig, + textDecodeConfigs, err := lobContext.GetSourceTextColumnConfigs(s.GetManifest()) + if err != nil { + return nil, err + } + reader, existingFields, err := newTextDecodedCompactionSegmentRecordReader(ctx, s, plan.GetSchema(), compactionParams.StorageConfig, textDecodeConfigs, storage.WithCollectionID(collectionID), storage.WithDownloader(binlogIO.Download), storage.WithVersion(s.StorageVersion), diff --git a/internal/datanode/compactor/mix_compactor.go b/internal/datanode/compactor/mix_compactor.go index 687f4ef4f5..0ac4846efd 100644 --- a/internal/datanode/compactor/mix_compactor.go +++ b/internal/datanode/compactor/mix_compactor.go @@ -21,7 +21,6 @@ import ( "fmt" sio "io" "math" - "path" "time" "github.com/apache/arrow/go/v17/arrow/array" @@ -42,7 +41,6 @@ import ( "github.com/milvus-io/milvus/pkg/v3/proto/indexpb" "github.com/milvus-io/milvus/pkg/v3/util/funcutil" "github.com/milvus-io/milvus/pkg/v3/util/merr" - "github.com/milvus-io/milvus/pkg/v3/util/metautil" "github.com/milvus-io/milvus/pkg/v3/util/paramtable" "github.com/milvus-io/milvus/pkg/v3/util/timerecord" "github.com/milvus-io/milvus/pkg/v3/util/typeutil" @@ -174,9 +172,10 @@ func (t *mixCompactionTask) buildWriterOptions(ctx context.Context) []storage.Rw } if t.lobContext != nil && t.lobContext.ShouldRewriteAnyField() { - // LOB base path at partition level: {root}/insert_log/{coll}/{part} - lobBasePath := path.Join(t.compactionParams.StorageConfig.GetRootPath(), - common.SegmentInsertLogPath, metautil.JoinIDPath(t.collectionID, t.partitionID)) + // LOB base path at partition level, sharing the manifest root rule so + // the reader (which reconstructs lobs/ from the manifest base) resolves + // the same files (see #53051). + lobBasePath := storage.SegmentPartitionBasePath(t.compactionParams.StorageConfig.GetRootPath(), t.collectionID, t.partitionID) textColumnConfigs := t.lobContext.GetTextColumnConfigs( lobBasePath, t.compactionParams.TextInlineThreshold, @@ -289,7 +288,11 @@ func (t *mixCompactionTask) writeSegment(ctx context.Context, } entityFilter := compaction.NewEntityFilter(delta, t.plan.GetCollectionTtl(), t.currentTime, seg.GetCommitTimestamp()) - reader, existingFields, err := newCompactionSegmentRecordReader(ctx, seg, t.plan.GetSchema(), t.compactionParams.StorageConfig, + textDecodeConfigs, err := t.lobContext.GetSourceTextColumnConfigs(seg.GetManifest()) + if err != nil { + return 0, 0, err + } + reader, existingFields, err := newTextDecodedCompactionSegmentRecordReader(ctx, seg, t.plan.GetSchema(), t.compactionParams.StorageConfig, textDecodeConfigs, storage.WithCollectionID(t.collectionID), storage.WithDownloader(t.binlogIO.Download), storage.WithVersion(seg.GetStorageVersion()), @@ -776,6 +779,19 @@ func (t *mixCompactionTask) initLOBCompactionContext(ctx context.Context) error t.lobContext.SetCompactionType(datapb.CompactionType_MixCompaction, sourceSegmentCount, targetSegmentCount) t.lobContext.ComputeStrategies(textFieldIDs, t.compactionParams.LOBHoleRatioThreshold) + outputPartitionBase := storage.SegmentPartitionBasePath( + t.compactionParams.StorageConfig.GetRootPath(), t.collectionID, t.partitionID) + partitionBaseMismatch, err := compaction.LOBSourcePartitionBaseMismatch(sourceManifests, outputPartitionBase) + if err != nil { + return err + } + if partitionBaseMismatch { + t.lobContext.ForceRewriteAllAcrossPartitionBases(textFieldIDs) + mlog.Info(ctx, "forcing TEXT LOB rewrite across partition namespaces", + mlog.Int64("planID", t.GetPlanID()), + mlog.String("outputPartitionBase", outputPartitionBase), + mlog.Int64s("textFieldIDs", textFieldIDs)) + } // log strategy decisions for fieldID, decision := range t.lobContext.Decisions { diff --git a/internal/datanode/compactor/mix_compactor_storage_v2_test.go b/internal/datanode/compactor/mix_compactor_storage_v2_test.go index f3e629cf34..1a5e0e67d0 100644 --- a/internal/datanode/compactor/mix_compactor_storage_v2_test.go +++ b/internal/datanode/compactor/mix_compactor_storage_v2_test.go @@ -21,6 +21,7 @@ import ( "fmt" "math" "os" + "path/filepath" "testing" "time" @@ -67,12 +68,13 @@ func (s *MixCompactionTaskStorageV2Suite) SetupTest() { } func (s *MixCompactionTaskStorageV2Suite) TearDownTest() { + localStoragePath := paramtable.Get().LocalStorageCfg.Path.GetValue() + os.RemoveAll(filepath.Join(localStoragePath, "insert_log")) + os.RemoveAll(filepath.Join(localStoragePath, "delta_log")) + os.RemoveAll(filepath.Join(localStoragePath, "stats_log")) paramtable.Get().Reset(paramtable.Get().CommonCfg.StorageType.Key) paramtable.Get().Reset(paramtable.Get().CommonCfg.UseLoonFFI.Key) paramtable.Get().Reset(paramtable.Get().LocalStorageCfg.Path.Key) - os.RemoveAll(paramtable.Get().LocalStorageCfg.Path.GetValue() + "insert_log") - os.RemoveAll(paramtable.Get().LocalStorageCfg.Path.GetValue() + "delta_log") - os.RemoveAll(paramtable.Get().LocalStorageCfg.Path.GetValue() + "stats_log") initcore.CleanArrowFileSystem() } diff --git a/internal/datanode/compactor/mix_compactor_test.go b/internal/datanode/compactor/mix_compactor_test.go index ba3a853763..bcb5c71e9c 100644 --- a/internal/datanode/compactor/mix_compactor_test.go +++ b/internal/datanode/compactor/mix_compactor_test.go @@ -59,7 +59,12 @@ func TestMixCompactionTaskSuite(t *testing.T) { func TestMixInitLOBCompactionContextKeepsReuseAllDecisionWithoutLobFiles(t *testing.T) { paramtable.Get().Init(paramtable.NewBaseTable()) textFieldIDs := []int64{101, 102} + params := compaction.GenParams() + collectionID, partitionID := int64(10), int64(20) + partitionBase := storage.SegmentPartitionBasePath(params.StorageConfig.GetRootPath(), collectionID, partitionID) task := &mixCompactionTask{ + collectionID: collectionID, + partitionID: partitionID, plan: &datapb.CompactionPlan{ Type: datapb.CompactionType_MixCompaction, Schema: &schemapb.CollectionSchema{Fields: []*schemapb.FieldSchema{ @@ -67,11 +72,11 @@ func TestMixInitLOBCompactionContextKeepsReuseAllDecisionWithoutLobFiles(t *test {FieldID: textFieldIDs[1], Name: "text_2", DataType: schemapb.DataType_Text}, }}, SegmentBinlogs: []*datapb.CompactionSegmentBinlogs{ - {SegmentID: 1, Manifest: "manifest-1"}, - {SegmentID: 2, Manifest: "manifest-2"}, + {SegmentID: 1, Manifest: packed.MarshalManifestPath(partitionBase+"/1", 1)}, + {SegmentID: 2, Manifest: packed.MarshalManifestPath(partitionBase+"/2", 1)}, }, }, - compactionParams: compaction.GenParams(), + compactionParams: params, estimatedOutputSegmentCount: 1, } collectPatch := mockey.Mock(compaction.CollectLobFilesFromManifests).Return(map[int64][]packed.LobFileInfo{ @@ -92,6 +97,41 @@ func TestMixInitLOBCompactionContextKeepsReuseAllDecisionWithoutLobFiles(t *test } } +func TestMixInitLOBCompactionContextRewritesAcrossPartitionBases(t *testing.T) { + paramtable.Get().Init(paramtable.NewBaseTable()) + params := compaction.GenParams() + collectionID, partitionID := int64(10), int64(20) + task := &mixCompactionTask{ + collectionID: collectionID, + partitionID: partitionID, + plan: &datapb.CompactionPlan{ + Type: datapb.CompactionType_MixCompaction, + Schema: &schemapb.CollectionSchema{Fields: []*schemapb.FieldSchema{ + {FieldID: 101, Name: "text", DataType: schemapb.DataType_Text}, + }}, + SegmentBinlogs: []*datapb.CompactionSegmentBinlogs{{ + SegmentID: 1, + Manifest: packed.MarshalManifestPath( + params.StorageConfig.GetRootPath()+"/files/insert_log/10/20/1", 1), + }}, + }, + compactionParams: params, + estimatedOutputSegmentCount: 1, + } + collectPatch := mockey.Mock(compaction.CollectLobFilesFromManifests).Return( + map[int64][]packed.LobFileInfo{1: {{FieldID: 101, TotalRows: 1, ValidRows: 1}}}, nil, + ).Build() + defer collectPatch.UnPatch() + + err := task.initLOBCompactionContext(context.Background()) + assert.NoError(t, err) + if assert.NotNil(t, task.lobContext) { + assert.False(t, task.lobContext.HasReuseAllFields()) + assert.True(t, task.lobContext.ShouldRewriteAnyField()) + assert.Equal(t, compaction.LOBStrategyRewriteAll, task.lobContext.GetStrategy(101)) + } +} + func newMixCompactionStorageV1SuiteForDirectTest(t *testing.T) *MixCompactionTaskStorageV1Suite { s := &MixCompactionTaskStorageV1Suite{} s.SetT(t) diff --git a/internal/datanode/compactor/sort_compaction.go b/internal/datanode/compactor/sort_compaction.go index e62a9eff39..9bce4917d7 100644 --- a/internal/datanode/compactor/sort_compaction.go +++ b/internal/datanode/compactor/sort_compaction.go @@ -182,6 +182,19 @@ func (t *sortCompactionTask) sortSegment(ctx context.Context) (*datapb.Compactio storage.WithUseLoonFFI(t.useLoonFFI), storage.WithWriterFormat(t.compactionParams.GetStorageFormat()), } + if t.lobContext != nil && t.lobContext.ShouldRewriteAnyField() { + lobBasePath := storage.SegmentPartitionBasePath( + t.compactionParams.StorageConfig.GetRootPath(), t.collectionID, t.partitionID) + textColumnConfigs := t.lobContext.GetTextColumnConfigs( + lobBasePath, + t.compactionParams.TextInlineThreshold, + t.compactionParams.TextMaxLobFileBytes, + t.compactionParams.TextFlushThresholdBytes, + ) + if len(textColumnConfigs) > 0 { + writerOpts = append(writerOpts, storage.WithTextColumnConfigs(textColumnConfigs)) + } + } if t.lobContext != nil && t.lobContext.HasReuseAllFields() { writerOpts = append(writerOpts, storage.WithTextRefsAsBinary()) } @@ -249,7 +262,12 @@ func (t *sortCompactionTask) sortSegment(ctx context.Context) (*datapb.Compactio } phaseStart = time.Now() - rr, existingFields, err := newCompactionSegmentRecordReader(ctx, t.plan.GetSegmentBinlogs()[0], t.plan.Schema, t.compactionParams.StorageConfig, + textDecodeConfigs, err := t.lobContext.GetSourceTextColumnConfigs(t.manifest) + if err != nil { + srw.Close() + return nil, err + } + rr, existingFields, err := newTextDecodedCompactionSegmentRecordReader(ctx, t.plan.GetSegmentBinlogs()[0], t.plan.Schema, t.compactionParams.StorageConfig, textDecodeConfigs, storage.WithVersion(t.segmentStorageVersion), storage.WithDownloader(t.binlogIO.Download), storage.WithStorageConfig(t.compactionParams.StorageConfig), @@ -563,9 +581,9 @@ func (t *sortCompactionTask) createTextIndex(ctx context.Context, } // initLOBCompactionContext initializes the LOB compaction context for TEXT columns. -// For sort compaction, data is reordered but not redistributed, so TEXT columns -// use REUSE_ALL strategy (LOB references remain valid after reordering). -// The LOB file references need to be copied to the output segment's manifest. +// Sort compaction normally reuses TEXT references because data is reordered but +// not redistributed. If the output uses a different partition base, references +// must instead be rewritten because they do not encode their source namespace. func (t *sortCompactionTask) initLOBCompactionContext(ctx context.Context) error { // check if there are TEXT fields in schema textFieldIDs := compaction.GetTEXTFieldIDsFromSchema(t.plan.GetSchema()) @@ -616,12 +634,28 @@ func (t *sortCompactionTask) initLOBCompactionContext(ctx context.Context) error // compute strategies (will use forced REUSE_ALL for all TEXT fields) t.lobContext.ComputeStrategies(textFieldIDs, t.compactionParams.LOBHoleRatioThreshold) + outputPartitionBase := storage.SegmentPartitionBasePath( + t.compactionParams.StorageConfig.GetRootPath(), t.collectionID, t.partitionID) + partitionBaseMismatch, err := compaction.LOBSourcePartitionBaseMismatch(sourceManifests, outputPartitionBase) + if err != nil { + return err + } + if partitionBaseMismatch { + t.lobContext.ForceRewriteAllAcrossPartitionBases(textFieldIDs) + log.Info(ctx, "forcing TEXT LOB rewrite across partition namespaces", + mlog.String("outputPartitionBase", outputPartitionBase)) + } // log strategy decisions for fieldID, decision := range t.lobContext.Decisions { log.Info(ctx, "LOB compaction strategy decided", mlog.FieldFieldID(fieldID), - mlog.String("strategy", "REUSE_ALL"), + mlog.String("strategy", func() string { + if decision.Strategy == compaction.LOBStrategyRewriteAll { + return "REWRITE_ALL" + } + return "REUSE_ALL" + }()), mlog.Bool("isForced", t.lobContext.IsForced), mlog.Float64("holeRatio", decision.OverallHoleRatio), ) diff --git a/internal/datanode/compactor/sort_compaction_test.go b/internal/datanode/compactor/sort_compaction_test.go index 5e581dc36e..0442859a0d 100644 --- a/internal/datanode/compactor/sort_compaction_test.go +++ b/internal/datanode/compactor/sort_compaction_test.go @@ -50,9 +50,14 @@ func TestSortCompactionTaskSuite(t *testing.T) { func TestSortInitLOBCompactionContextKeepsReuseAllDecisionWithoutLobFiles(t *testing.T) { paramtable.Get().Init(paramtable.NewBaseTable()) textFieldIDs := []int64{101, 102} + params := compaction.GenParams() + collectionID, partitionID := int64(10), int64(20) task := &sortCompactionTask{ - segmentID: 100, - manifest: "manifest-100", + collectionID: collectionID, + partitionID: partitionID, + segmentID: 100, + manifest: packed.MarshalManifestPath( + storage.SegmentManifestBasePath(params.StorageConfig.GetRootPath(), collectionID, partitionID, 100), 1), plan: &datapb.CompactionPlan{ Type: datapb.CompactionType_SortCompaction, Schema: &schemapb.CollectionSchema{Fields: []*schemapb.FieldSchema{ @@ -60,7 +65,7 @@ func TestSortInitLOBCompactionContextKeepsReuseAllDecisionWithoutLobFiles(t *tes {FieldID: textFieldIDs[1], Name: "text_2", DataType: schemapb.DataType_Text}, }}, }, - compactionParams: compaction.GenParams(), + compactionParams: params, } collectPatch := mockey.Mock(compaction.CollectLobFilesFromManifests).Return(map[int64][]packed.LobFileInfo{ 100: {}, @@ -79,6 +84,37 @@ func TestSortInitLOBCompactionContextKeepsReuseAllDecisionWithoutLobFiles(t *tes } } +func TestSortInitLOBCompactionContextRewritesAcrossPartitionBases(t *testing.T) { + paramtable.Get().Init(paramtable.NewBaseTable()) + params := compaction.GenParams() + task := &sortCompactionTask{ + collectionID: 10, + partitionID: 20, + segmentID: 100, + manifest: packed.MarshalManifestPath( + params.StorageConfig.GetRootPath()+"/files/insert_log/10/20/100", 1), + plan: &datapb.CompactionPlan{ + Type: datapb.CompactionType_SortCompaction, + Schema: &schemapb.CollectionSchema{Fields: []*schemapb.FieldSchema{ + {FieldID: 101, Name: "text", DataType: schemapb.DataType_Text}, + }}, + }, + compactionParams: params, + } + collectPatch := mockey.Mock(compaction.CollectLobFilesFromManifests).Return( + map[int64][]packed.LobFileInfo{100: {{FieldID: 101, TotalRows: 1, ValidRows: 1}}}, nil, + ).Build() + defer collectPatch.UnPatch() + + err := task.initLOBCompactionContext(context.Background()) + assert.NoError(t, err) + if assert.NotNil(t, task.lobContext) { + assert.False(t, task.lobContext.HasReuseAllFields()) + assert.True(t, task.lobContext.ShouldRewriteAnyField()) + assert.Equal(t, compaction.LOBStrategyRewriteAll, task.lobContext.GetStrategy(101)) + } +} + type SortCompactionTaskSuite struct { suite.Suite diff --git a/internal/datanode/external/manager_test.go b/internal/datanode/external/manager_test.go index d3ee523ad8..7c37a5f074 100644 --- a/internal/datanode/external/manager_test.go +++ b/internal/datanode/external/manager_test.go @@ -18,6 +18,7 @@ package external import ( "context" + "path/filepath" "sync/atomic" "testing" "time" @@ -26,6 +27,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/milvus-io/milvus-proto/go-api/v3/schemapb" "github.com/milvus-io/milvus/pkg/v3/proto/datapb" "github.com/milvus-io/milvus/pkg/v3/proto/indexpb" ) @@ -189,6 +191,56 @@ func TestExternalCollectionManager_SubmitTask_Failure(t *testing.T) { assert.Equal(t, expectedError.Error(), info.FailReason) } +func TestExternalCollectionManager_MissingExploreManifestFailsTask(t *testing.T) { + for _, name := range []string{"legacy_relative", "absolute"} { + t.Run(name, func(t *testing.T) { + ctx := t.Context() + root := t.TempDir() + manifest := filepath.Join("__explore_temp__", filepath.Base(filepath.Dir(root)), "_metadata", "manifest-1.avro") + if name == "absolute" { + manifest = filepath.Join(root, manifest) + } + req := &datapb.RefreshExternalCollectionTaskRequest{ + TaskID: 1, + CollectionID: 100, + Schema: &schemapb.CollectionSchema{}, + StorageConfig: &indexpb.StorageConfig{StorageType: "local", RootPath: root}, + ExternalSource: filepath.Join(root, "input"), + ExternalSpec: `{"format":"parquet"}`, + ExploreManifestPath: manifest, + PreAllocatedSegmentIds: &datapb.IDRange{Begin: 1000, End: 1010}, + TargetRowsPerSegment: 100, + } + manager := NewExternalCollectionManager(ctx, 1) + defer manager.Close() + err := manager.SubmitTask("test-cluster", req, func(taskCtx context.Context) (*datapb.RefreshExternalCollectionTaskResponse, error) { + task := NewRefreshExternalCollectionTask(taskCtx, req) + if err := task.PreExecute(taskCtx); err != nil { + return nil, err + } + if err := task.Execute(taskCtx); err != nil { + return nil, err + } + return &datapb.RefreshExternalCollectionTaskResponse{State: indexpb.JobState_JobStateFinished}, nil + }) + require.NoError(t, err) + + // Exercise the real manifest reader: even if the native not-found + // error is classified as transient, this task must terminate. + require.Eventually(t, func() bool { + info := manager.Get("test-cluster", req.GetTaskID()) + return info != nil && info.State == indexpb.JobState_JobStateFailed + }, 5*time.Second, 10*time.Millisecond) + info := manager.Get("test-cluster", req.GetTaskID()) + assert.Contains(t, info.FailReason, "failed to fetch fragments") + assert.Contains(t, info.FailReason, "failed to read explore manifest") + assert.Contains(t, info.FailReason, "manifest-1.avro") + assert.Contains(t, info.FailReason, "File not found") + assert.Empty(t, info.UpdatedSegments) + }) + } +} + // Regression for #49225: a panic inside taskFunc (e.g. divide-by-zero from a // malformed external parquet) must be isolated to the task — the manager pool // goroutine must NOT crash the process, and the task must surface as Failed. diff --git a/internal/datanode/external/task_update.go b/internal/datanode/external/task_update.go index e7141524ed..c21357c4fa 100644 --- a/internal/datanode/external/task_update.go +++ b/internal/datanode/external/task_update.go @@ -42,7 +42,6 @@ package external import ( "context" "fmt" - "path" "sort" "strconv" "strings" @@ -62,7 +61,6 @@ import ( "github.com/milvus-io/milvus/pkg/v3/util/conc" "github.com/milvus-io/milvus/pkg/v3/util/externalspec" "github.com/milvus-io/milvus/pkg/v3/util/merr" - "github.com/milvus-io/milvus/pkg/v3/util/metautil" "github.com/milvus-io/milvus/pkg/v3/util/paramtable" "github.com/milvus-io/milvus/pkg/v3/util/timerecord" "github.com/milvus-io/milvus/pkg/v3/util/typeutil" @@ -1061,8 +1059,8 @@ func (t *RefreshExternalCollectionTask) createManifestForSegment( ) (string, error) { // All segments now use final paths with real IDs (no temporary paths needed) // Pre-allocated IDs ensure we can write directly to final locations - basePath := segmentInsertLogBasePath( - t.req.GetStorageConfig(), + basePath := storage.SegmentManifestBasePath( + t.req.GetStorageConfig().GetRootPath(), t.req.GetCollectionID(), t.req.GetPartitionID(), segmentID, @@ -1091,19 +1089,6 @@ func (t *RefreshExternalCollectionTask) createManifestForSegment( return manifestPath, nil } -func segmentInsertLogBasePath( - storageConfig *indexpb.StorageConfig, - collectionID int64, - partitionID int64, - segmentID int64, -) string { - rootPath := "" - if storageConfig != nil { - rootPath = storageConfig.GetRootPath() - } - return path.Join(rootPath, common.SegmentInsertLogPath, metautil.JoinIDPath(collectionID, partitionID, segmentID)) -} - // hasFunctions returns true if the schema defines any functions. func (t *RefreshExternalCollectionTask) hasFunctions() bool { return len(t.req.GetSchema().GetFunctions()) > 0 @@ -1117,8 +1102,8 @@ func (t *RefreshExternalCollectionTask) createManifestWithFunctions( segmentID int64, fragments []packed.Fragment, ) (string, error) { - basePath := segmentInsertLogBasePath( - t.req.GetStorageConfig(), + basePath := storage.SegmentManifestBasePath( + t.req.GetStorageConfig().GetRootPath(), t.req.GetCollectionID(), t.req.GetPartitionID(), segmentID, diff --git a/internal/datanode/importv2/copy_segment_utils.go b/internal/datanode/importv2/copy_segment_utils.go index 4330141e3d..1798b07460 100644 --- a/internal/datanode/importv2/copy_segment_utils.go +++ b/internal/datanode/importv2/copy_segment_utils.go @@ -20,6 +20,7 @@ import ( "context" "net/url" "path" + "slices" "strconv" "strings" "time" @@ -101,6 +102,10 @@ func transformManifestPath( return "", merr.Wrap(err, "failed to unmarshal manifest path") } + // The manifest stores the complete ChunkManager key on every backend (an + // absolute path under localStorage.path for local, a bucket-relative key + // under minio.rootPath for remote), so the transformed physical path is + // the new manifest base as-is. targetBasePath, err := generateTargetPath(basePath, source, target) if err != nil { return "", merr.Wrap(err, "failed to generate target base path") @@ -244,9 +249,14 @@ func collectSegmentFiles( if err != nil { return nil, merr.Wrapf(err, "failed to unmarshal manifest path %q for segment %d", manifestPath, source.GetSegmentId()) } - basePath = snapshotstorage.NormalizeSnapshotObjectPath(basePath) + // The manifest base is the complete ChunkManager key on every backend; + // only foreign snapshot URIs need their object key extracted. + walkBasePath := snapshotstorage.NormalizeSnapshotObjectPath(basePath) + if strings.TrimSpace(walkBasePath) == "" { + return nil, merr.WrapErrDataIntegrityMsg("storage v3 segment %d has an empty manifest base path", source.GetSegmentId()) + } - allFiles, listErr := listAllFiles(ctx, sourceCM, basePath) + allFiles, listErr := listAllFiles(ctx, sourceCM, walkBasePath) if listErr != nil { return nil, merr.Wrapf(listErr, "failed to list files from manifest base path %q for segment %d", basePath, source.GetSegmentId()) } @@ -272,8 +282,8 @@ func collectSegmentFiles( if lobErr != nil { return nil, merr.Wrapf(lobErr, "failed to collect LOB files from manifest for segment %d", source.GetSegmentId()) } else if len(lobFileInfos) > 0 { - // GetManifestLobFiles returns absolute paths (the manifest - // deserializer calls ToAbsolute internally), so use them directly. + // Manifest::ToAbsolutePaths expands entries against the manifest + // base, which is already the complete ChunkManager key. files.LobFiles = lobFileInfosToPaths(lobFileInfos) mlog.Info(ctx, "collected LOB files from segment manifest", mlog.String("manifestPath", manifestPath), @@ -435,23 +445,31 @@ func CopySegmentAndIndexFiles( mlog.Info(ctx, "all files copied successfully", mlog.Int("fileCount", len(mappings))) - // Step 3.5: When manifest is used (StorageV3+), InsertBinlogs were collected from manifest - // (actual file paths under base_path including _data/ and _metadata/), but - // generateSegmentInfoFromSource needs mappings for the protobuf logical paths too. - // Add these "logical-only" mappings AFTER file copying so they don't trigger actual copy operations. + // Step 3.5: When manifest is used (StorageV3+), physical paths were + // collected from the manifest, but generateSegmentInfoFromSource still + // needs mappings for the protobuf paths (which for V3 are summaries whose + // LogPath may be empty or a manifest-internal key). Add those "logical-only" + // mappings after physical copying so they never trigger a copy. if useManifest { - pbInsertPaths := extractFromPb(source.GetInsertBinlogs()) - for _, srcPath := range pbInsertPaths { - if _, exists := mappings[srcPath]; !exists { + logicalPathGroups := [][]string{ + extractFromPb(source.GetInsertBinlogs()), + extractFromPb(source.GetDeltaBinlogs()), + extractFromPb(source.GetStatsBinlogs()), + extractFromPb(source.GetBm25Binlogs()), + } + logicalPathCount := 0 + for _, logicalPaths := range logicalPathGroups { + logicalPathCount += len(logicalPaths) + for _, srcPath := range logicalPaths { dstPath, pathErr := generateTargetPath(srcPath, source, target) if pathErr != nil { - return nil, copiedFiles, merr.Wrapf(pathErr, "failed to generate target path for pb insert binlog %s", srcPath) + return nil, copiedFiles, merr.Wrapf(pathErr, "failed to generate target path for protobuf binlog %s", srcPath) } mappings[srcPath] = dstPath } } mlog.Info(ctx, "added logical insert binlog mappings for manifest segment", - mlog.Int("pbPathCount", len(pbInsertPaths))) + mlog.Int("pbPathCount", logicalPathCount)) } // Step 4: Build index metadata from source @@ -776,19 +794,29 @@ func splitIndexArtifactPaths(filePaths []string) (string, []string, error) { // Parameters: // - srcFieldBinlogs: Source field binlogs with original paths // - mappings: Pre-calculated map of source path -> target path -// - countRows: If true, accumulate total row count from EntriesNum (for insert logs only) -// - isExternalTable: If true, skip path mapping because external table insert -// binlogs carry row metadata without physical log paths +// - opts: see transformFieldBinlogOptions // // Returns: // - []*datapb.FieldBinlog: Transformed binlog list with target paths // - int64: Total row count (sum of EntriesNum from all binlogs if countRows=true, 0 otherwise) // - error: Non-nil if any source path has no mapping (fail-fast on missing mappings) +// +// transformFieldBinlogOptions controls how one binlog family is transformed. +type transformFieldBinlogOptions struct { + // countRows accumulates total row count from EntriesNum (insert logs only). + countRows bool + // isExternalTable skips path mapping: external table insert binlogs carry + // row metadata without physical log paths. + isExternalTable bool + // preservePathless keeps metadata-only summaries that have no physical + // LogPath (StorageV3 insert and delta summaries). + preservePathless bool +} + func transformFieldBinlogs( srcFieldBinlogs []*datapb.FieldBinlog, mappings map[string]string, - countRows bool, - isExternalTable bool, + opts transformFieldBinlogOptions, ) ([]*datapb.FieldBinlog, int64, error) { result := make([]*datapb.FieldBinlog, 0, len(srcFieldBinlogs)) var totalRows int64 @@ -800,20 +828,23 @@ func transformFieldBinlogs( for _, srcBinlog := range srcFieldBinlog.GetBinlogs() { dstBinlog := proto.Clone(srcBinlog).(*datapb.Binlog) - if !isExternalTable { + if !opts.isExternalTable { srcPath := srcBinlog.GetLogPath() if srcPath == "" { - continue + if !opts.preservePathless { + continue + } + } else { + dstPath, ok := mappings[srcPath] + if !ok { + return nil, 0, merr.WrapErrServiceInternalMsg("no mapping found for source path: %s", srcPath) + } + dstBinlog.LogPath = dstPath } - dstPath, ok := mappings[srcPath] - if !ok { - return nil, 0, merr.WrapErrServiceInternalMsg("no mapping found for source path: %s", srcPath) - } - dstBinlog.LogPath = dstPath } dstFieldBinlog.Binlogs = append(dstFieldBinlog.Binlogs, dstBinlog) - if countRows { + if opts.countRows { totalRows += srcBinlog.GetEntriesNum() } } @@ -862,7 +893,13 @@ func generateSegmentInfoFromSource( } // Process insert binlogs (count rows) - binlogs, totalRows, err := transformFieldBinlogs(source.GetInsertBinlogs(), mappings, true, source.GetIsExternalCollection()) + isStorageV3 := source.GetStorageVersion() >= storage.StorageV3 + binlogs, totalRows, err := transformFieldBinlogs(source.GetInsertBinlogs(), mappings, + transformFieldBinlogOptions{ + countRows: true, + isExternalTable: source.GetIsExternalCollection(), + preservePathless: isStorageV3, + }) if err != nil { return nil, merr.Wrap(err, "failed to transform insert binlogs") } @@ -873,21 +910,21 @@ func generateSegmentInfoFromSource( } // Process stats binlogs (no row counting) - statslogs, _, err := transformFieldBinlogs(source.GetStatsBinlogs(), mappings, false, false) + statslogs, _, err := transformFieldBinlogs(source.GetStatsBinlogs(), mappings, transformFieldBinlogOptions{}) if err != nil { return nil, merr.Wrap(err, "failed to transform stats binlogs") } segmentInfo.Statslogs = statslogs // Process delta binlogs (no row counting) - deltalogs, _, err := transformFieldBinlogs(source.GetDeltaBinlogs(), mappings, false, false) + deltalogs, _, err := transformFieldBinlogs(source.GetDeltaBinlogs(), mappings, transformFieldBinlogOptions{preservePathless: isStorageV3}) if err != nil { return nil, merr.Wrap(err, "failed to transform delta binlogs") } segmentInfo.Deltalogs = deltalogs // Process BM25 binlogs (no row counting) - bm25logs, _, err := transformFieldBinlogs(source.GetBm25Binlogs(), mappings, false, false) + bm25logs, _, err := transformFieldBinlogs(source.GetBm25Binlogs(), mappings, transformFieldBinlogOptions{}) if err != nil { return nil, merr.Wrap(err, "failed to transform BM25 binlogs") } @@ -950,14 +987,29 @@ func remapSourceRootPath(sourcePath string, source *datapb.CopySegmentSource, ta } } - targetRootPath := strings.Trim(target.GetTargetRootPath(), "/") - if targetRootPath == "" { - return relativePath, nil + rawTargetRootPath := strings.TrimSpace(target.GetTargetRootPath()) + targetRootIsAbs := path.IsAbs(rawTargetRootPath) + targetRootPath := strings.Trim(rawTargetRootPath, "/") + var targetPath string + switch { + case targetRootPath == "": + targetPath = relativePath + case relativePath == "": + targetPath = targetRootPath + default: + targetPath = path.Join(targetRootPath, relativePath) } - if relativePath == "" { - return targetRootPath, nil + if targetRootIsAbs { + return "/" + targetPath, nil } - return path.Join(targetRootPath, relativePath), nil + return targetPath, nil +} + +// buildCanonicalTargetPath rebuilds the layout under the target root without +// carrying over the source storage prefix. Joining the root separately preserves +// the leading slash for local paths; empty/dot roots address the remote bucket. +func buildCanonicalTargetPath(parts []string, layoutIndex int, target *datapb.CopySegmentTarget) string { + return path.Join(strings.TrimSpace(target.GetTargetRootPath()), path.Join(parts[layoutIndex:]...)) } // generateTargetPath converts source file path to target path by replacing collection/partition/segment IDs @@ -970,7 +1022,7 @@ func generateTargetPath(sourcePath string, source *datapb.CopySegmentSource, tar return "", err } - // Convert IDs to strings for replacement + // Convert IDs to strings for replacement. targetCollectionIDStr := strconv.FormatInt(target.GetCollectionId(), 10) targetPartitionIDStr := strconv.FormatInt(target.GetPartitionId(), 10) targetSegmentIDStr := strconv.FormatInt(target.GetSegmentId(), 10) @@ -978,8 +1030,7 @@ func generateTargetPath(sourcePath string, source *datapb.CopySegmentSource, tar // Split path into parts parts := strings.Split(sourcePath, "/") - // Find the log type index (insert_log, delta_log, stats_log, bm25_stats) - // Path structure: .../log_type/collectionID/partitionID/segmentID/... + // Find the log layout keyword. logTypeIndex := -1 for i, part := range parts { if part == BinlogTypeInsert || part == BinlogTypeDelta || part == BinlogTypeStats || part == BinlogTypeBM25 { @@ -1001,7 +1052,7 @@ func generateTargetPath(sourcePath string, source *datapb.CopySegmentSource, tar parts[logTypeIndex+2] = targetPartitionIDStr parts[logTypeIndex+3] = targetSegmentIDStr - return path.Join(parts...), nil + return buildCanonicalTargetPath(parts, logTypeIndex, target), nil } // generateTargetLOBPath replaces collection and partition IDs in a LOB file path. @@ -1016,13 +1067,7 @@ func generateTargetLOBPath(sourcePath string, source *datapb.CopySegmentSource, parts := strings.Split(sourcePath, "/") - logTypeIndex := -1 - for i, part := range parts { - if part == BinlogTypeInsert { - logTypeIndex = i - break - } - } + logTypeIndex := slices.Index(parts, BinlogTypeInsert) // Path: .../{insert_log}/{coll}/{part}/lobs/... // Need at least logTypeIndex + 2 (coll and part) after insert_log @@ -1033,7 +1078,7 @@ func generateTargetLOBPath(sourcePath string, source *datapb.CopySegmentSource, parts[logTypeIndex+1] = strconv.FormatInt(target.GetCollectionId(), 10) parts[logTypeIndex+2] = strconv.FormatInt(target.GetPartitionId(), 10) - return path.Join(parts...), nil + return buildCanonicalTargetPath(parts, logTypeIndex, target), nil } // buildIndexInfoFromSource builds complete index metadata from source information. @@ -1166,10 +1211,10 @@ func buildIndexInfoFromSource( // File Type Constants // ============================================================================ -// lobFileInfosToPaths extracts absolute file paths from LobFileInfo structs. -// GetManifestLobFiles returns paths that have already been resolved to absolute -// form by the C++ manifest deserializer (Manifest::ToAbsolutePaths), so we use -// them directly without any path concatenation. +// lobFileInfosToPaths extracts manifest-expanded paths from LobFileInfo structs. +// The manifest deserializer (Manifest::ToAbsolutePaths) expands entries against +// the manifest base, which is the complete ChunkManager key on every backend, +// so the paths are used as-is. func lobFileInfosToPaths(infos []packed.LobFileInfo) []string { paths := make([]string, 0, len(infos)) for _, info := range infos { @@ -1238,11 +1283,7 @@ func generateTargetIndexPath( return "", err } - // Split path into parts - parts := strings.Split(sourcePath, "/") - - // Determine keyword and offsets based on index type - var keywordIdx int + // Determine keyword and offsets based on index type. var collectionOffset, partitionOffset, segmentOffset int keyword := indexType @@ -1252,19 +1293,6 @@ func generateTargetIndexPath( keyword = IndexTypeVectorScalarV1 } - // Find the keyword position in the path - keywordIdx = -1 - for i, part := range parts { - if part == keyword { - keywordIdx = i - break - } - } - - if keywordIdx == -1 { - return "", merr.WrapErrServiceInternalMsg("keyword '%s' not found in path: %s", keyword, sourcePath) - } - // Set offsets based on index type // collectionOffset = -1 means collectionID is not present in the path var buildIDOffset int @@ -1298,10 +1326,11 @@ func generateTargetIndexPath( indexType, IndexTypeVectorScalarV0, IndexTypeText, IndexTypeJSONKey, IndexTypeJSONStats) } - // Validate path structure has enough components - if keywordIdx+segmentOffset >= len(parts) { - return "", merr.WrapErrParameterInvalidMsg("invalid %s path structure: %s (expected '%s' with at least %d components after it)", - indexType, sourcePath, indexType, segmentOffset+1) + parts := strings.Split(sourcePath, "/") + keywordIdx := slices.Index(parts, keyword) + if keywordIdx == -1 || keywordIdx+segmentOffset >= len(parts) { + return "", merr.WrapErrParameterInvalidMsg("invalid %s path structure: %s (expected '%s' layout)", + indexType, sourcePath, keyword) } // Replace buildID if a mapping exists in target.NewBuildIds @@ -1323,7 +1352,7 @@ func generateTargetIndexPath( parts[keywordIdx+partitionOffset] = strconv.FormatInt(target.GetPartitionId(), 10) parts[keywordIdx+segmentOffset] = strconv.FormatInt(target.GetSegmentId(), 10) - return path.Join(parts...), nil + return buildCanonicalTargetPath(parts, keywordIdx, target), nil } // ============================================================================ @@ -1385,56 +1414,52 @@ func shortenIndexFilePaths(fullPaths []string) []string { func shortenJSONStatsPath(jsonStats map[int64]*datapb.JsonKeyStats) map[int64]*datapb.JsonKeyStats { for _, stats := range jsonStats { for i, file := range stats.GetFiles() { - stats.Files[i] = shortenSingleJSONStatsPath(file) + stats.Files[i] = shortenJSONStatsFilePath(file, stats) } } return jsonStats } -// shortenSingleJSONStatsPath shortens a single JSON stats file path. -// -// This function extracts the relative path from a full JSON stats file path by: -// 1. Finding "shared_key_index" or "shredding_data" keywords and extracting from that position -// 2. For files directly under fieldID directory (e.g., meta.json), extracting everything after -// the 7 path components following "json_stats" +// shortenJSONStatsFilePath extracts the path after the fieldID component of a +// JSON-stats layout matching stats. The caller only passes non-nil stats that +// contain files; nil map entries are skipped by shortenJSONStatsPath. // // Path format: {root}/json_stats/{dataFormat}/{buildID}/{version}/{collID}/{partID}/{segID}/{fieldID}/... -// -// Path examples: -// - Input: "files/json_stats/2/123/1/444/555/666/100/shared_key_index/inverted_index_0" -// Output: "shared_key_index/inverted_index_0" -// - Input: "files/json_stats/2/123/1/444/555/666/100/shredding_data/parquet_data_0" -// Output: "shredding_data/parquet_data_0" -// - Input: "files/json_stats/2/123/1/444/555/666/100/meta.json" -// Output: "meta.json" -// - Input: "shared_key_index/inverted_index_0" (already shortened) -// Output: "shared_key_index/inverted_index_0" (idempotent) -// - Input: "meta.json" (already shortened) -// Output: "meta.json" (idempotent) -// -// Parameters: -// - fullPath: Full or partial JSON stats file path -// -// Returns: -// - Shortened path relative to fieldID directory -func shortenSingleJSONStatsPath(fullPath string) string { - // Find "shared_key_index" in path - if idx := strings.Index(fullPath, jsonStatsSharedIndexPath); idx != -1 { - return fullPath[idx:] - } - // Find "shredding_data" in path - if idx := strings.Index(fullPath, jsonStatsShreddingDataPath); idx != -1 { - return fullPath[idx:] +func shortenJSONStatsFilePath(fullPath string, stats *datapb.JsonKeyStats) string { + parts := strings.Split(fullPath, "/") + if parts[0] == jsonStatsSharedIndexPath || parts[0] == jsonStatsShreddingDataPath { + return path.Join(parts...) } - // Handle files directly under fieldID directory (e.g., meta.json) - // Path format: .../json_stats/{dataFormat}/{build}/{ver}/{coll}/{part}/{seg}/{field}/filename - // json_stats is followed by 7 components, the 8th onwards is the file path - parts := strings.Split(fullPath, "/") + // Find the last structurally valid JSON-stats layout. Root prefixes may + // themselves contain json_stats/shared_key_index, so substring or first-key + // matching is not safe. Also match the layout's data format, build, version, + // and field components against the stats metadata. + anchor := -1 for i, part := range parts { - if part == common.JSONStatsPath && i+8 < len(parts) { - return path.Join(parts[i+8:]...) + if part != common.JSONStatsPath || i+8 >= len(parts) { + continue } + valid := true + for offset := 1; offset <= 7; offset++ { + if _, err := strconv.ParseInt(parts[i+offset], 10, 64); err != nil { + valid = false + break + } + } + if !valid { + continue + } + if parts[i+1] != strconv.FormatInt(stats.GetJsonKeyStatsDataFormat(), 10) || + parts[i+2] != strconv.FormatInt(stats.GetBuildID(), 10) || + parts[i+3] != strconv.FormatInt(stats.GetVersion(), 10) || + parts[i+7] != strconv.FormatInt(stats.GetFieldID(), 10) { + continue + } + anchor = i + } + if anchor >= 0 { + return path.Join(parts[anchor+8:]...) } // If already shortened or no json_stats found, return as-is diff --git a/internal/datanode/importv2/copy_segment_utils_test.go b/internal/datanode/importv2/copy_segment_utils_test.go index 19e6af707e..52abe1f357 100644 --- a/internal/datanode/importv2/copy_segment_utils_test.go +++ b/internal/datanode/importv2/copy_segment_utils_test.go @@ -20,6 +20,7 @@ import ( "context" "fmt" "path" + "path/filepath" "strings" "testing" "time" @@ -35,6 +36,7 @@ import ( "github.com/milvus-io/milvus/internal/storage" "github.com/milvus-io/milvus/internal/storagev2/packed" "github.com/milvus-io/milvus/pkg/v3/common" + "github.com/milvus-io/milvus/pkg/v3/objectstorage" "github.com/milvus-io/milvus/pkg/v3/proto/datapb" "github.com/milvus-io/milvus/pkg/v3/proto/indexpb" "github.com/milvus-io/milvus/pkg/v3/util/paramtable" @@ -120,12 +122,14 @@ func TestGenerateTargetPath(t *testing.T) { SegmentId: 333, } target := &datapb.CopySegmentTarget{ - CollectionId: 444, - PartitionId: 555, - SegmentId: 666, + TargetRootPath: "files", + CollectionId: 444, + PartitionId: 555, + SegmentId: 666, } tests := []struct { + targetRoot string name string sourcePath string wantPath string @@ -161,10 +165,17 @@ func TestGenerateTargetPath(t *testing.T) { wantPath: "files/insert_log/444/555/666/_stats/bm25.100/12345", wantErr: false, }, + { + name: "local complete filesystem path", + targetRoot: "/var/lib/milvus/data", + sourcePath: "/var/lib/milvus/data/insert_log/111/222/333/_data/0", + wantPath: "/var/lib/milvus/data/insert_log/444/555/666/_data/0", + wantErr: false, + }, { name: "external component in regular binlog root is not external table", sourcePath: "files/external/insert_log/111/222/333/100/log1.log", - wantPath: "files/external/insert_log/444/555/666/100/log1.log", + wantPath: "files/insert_log/444/555/666/100/log1.log", wantErr: false, }, { @@ -183,6 +194,10 @@ func TestGenerateTargetPath(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + target.TargetRootPath = tt.targetRoot + if target.TargetRootPath == "" { + target.TargetRootPath = "files" + } gotPath, err := generateTargetPath(tt.sourcePath, source, target) if tt.wantErr { assert.Error(t, err) @@ -204,9 +219,10 @@ func TestGenerateMappingsFromFiles_ExternalTable(t *testing.T) { IsExternalCollection: true, } target := &datapb.CopySegmentTarget{ - CollectionId: 444, - PartitionId: 555, - SegmentId: 666, + TargetRootPath: "files", + CollectionId: 444, + PartitionId: 555, + SegmentId: 666, } files := &SegmentFiles{ InsertBinlogs: []string{"files/insert_log/111/222/333/_metadata/manifest.json"}, @@ -261,6 +277,70 @@ func TestCollectSegmentFiles_UsesSourceStorageConfigForV3(t *testing.T) { assert.Equal(t, []string{"foreign-root/files/insert_log/100/1/lobs/101/_data/0"}, files.LobFiles) } +// On local storage the manifest base, the LOB paths the manifest expands and +// the protobuf paths are all complete filesystem paths; they are walked and +// copied as-is, with no root joined anywhere. +func TestCollectSegmentFiles_LocalV3ManifestKeysAreCompletePaths(t *testing.T) { + root := t.TempDir() + sourceCM := storage.NewLocalChunkManager(objectstorage.RootPath(root)) + sourceCfg := &indexpb.StorageConfig{ + StorageType: "local", + RootPath: root, + } + basePath := path.Join(root, "insert_log/100/1/10") + manifestPath := packed.MarshalManifestPath(basePath, 2) + source := &datapb.CopySegmentSource{ + CollectionId: 100, + PartitionId: 1, + SegmentId: 10, + StorageVersion: storage.StorageV3, + ManifestPath: manifestPath, + DeltaBinlogs: []*datapb.FieldBinlog{{ + Binlogs: []*datapb.Binlog{{LogPath: path.Join(basePath, "_delta/7")}}, + }}, + } + + mList := mockey.Mock(listAllFiles).To( + func(_ context.Context, cm storage.ChunkManager, gotBasePath string) ([]string, error) { + assert.Same(t, sourceCM, cm) + assert.Equal(t, basePath, gotBasePath) + return []string{path.Join(basePath, "_data/0")}, nil + }).Build() + defer mList.UnPatch() + + mLob := mockey.Mock(packed.GetManifestLobFiles).To( + func(gotManifestPath string, cfg *indexpb.StorageConfig) ([]packed.LobFileInfo, error) { + assert.Equal(t, manifestPath, gotManifestPath) + assert.Equal(t, root, cfg.GetRootPath()) + return []packed.LobFileInfo{{ + Path: path.Join(root, "insert_log/100/1/lobs/101/_data/0"), + }}, nil + }).Build() + defer mLob.UnPatch() + + files, err := collectSegmentFiles(context.Background(), sourceCM, sourceCfg, source) + require.NoError(t, err) + assert.Equal(t, []string{path.Join(basePath, "_data/0")}, files.InsertBinlogs) + assert.Equal(t, []string{path.Join(root, "insert_log/100/1/lobs/101/_data/0")}, files.LobFiles) + assert.Equal(t, []string{path.Join(basePath, "_delta/7")}, files.DeltaBinlogs) +} + +func TestCollectSegmentFiles_LocalV3RejectsEmptyManifestBase(t *testing.T) { + root := t.TempDir() + _, err := collectSegmentFiles( + context.Background(), + storage.NewLocalChunkManager(objectstorage.RootPath(root)), + &indexpb.StorageConfig{StorageType: "local", RootPath: root}, + &datapb.CopySegmentSource{ + SegmentId: 10, + StorageVersion: storage.StorageV3, + ManifestPath: packed.MarshalManifestPath("", 1), + }, + ) + require.Error(t, err) + assert.Contains(t, err.Error(), "empty manifest base path") +} + func TestCollectSegmentFiles_NormalizesV3ManifestURIForListing(t *testing.T) { sourceCM := &struct{ storage.ChunkManager }{} basePath := "s3://foreign-source/foreign-root/files/insert_log/100/1/10" @@ -315,7 +395,7 @@ func TestGenerateTargetPath_RemapExternalRoot(t *testing.T) { ) assert.NoError(t, err) - assert.Equal(t, "target-cluster-root/files/insert_log/444/555/666/100/log1.log", gotPath) + assert.Equal(t, "target-cluster-root/insert_log/444/555/666/100/log1.log", gotPath) } func TestGenerateTargetPath_RemapExternalS3Root(t *testing.T) { @@ -339,7 +419,31 @@ func TestGenerateTargetPath_RemapExternalS3Root(t *testing.T) { ) assert.NoError(t, err) - assert.Equal(t, "target-cluster-root/files/insert_log/444/555/666/100/log1.log", gotPath) + assert.Equal(t, "target-cluster-root/insert_log/444/555/666/100/log1.log", gotPath) +} + +func TestGenerateTargetPath_RemapExternalRootToLocalCompletePath(t *testing.T) { + source := &datapb.CopySegmentSource{ + CollectionId: 111, + PartitionId: 222, + SegmentId: 333, + SourceRootPath: "s3://bucket/source-cluster-root", + } + target := &datapb.CopySegmentTarget{ + CollectionId: 444, + PartitionId: 555, + SegmentId: 666, + TargetRootPath: "/var/lib/milvus/data", + } + + gotPath, err := generateTargetPath( + "source-cluster-root/files/insert_log/111/222/333/100/log1.log", + source, + target, + ) + + require.NoError(t, err) + assert.Equal(t, "/var/lib/milvus/data/insert_log/444/555/666/100/log1.log", gotPath) } func TestGenerateTargetPath_RemapEndpointStyleExternalRoot(t *testing.T) { @@ -388,7 +492,7 @@ func TestGenerateTargetPath_RemapEndpointStyleExternalRoot(t *testing.T) { gotPath, err := generateTargetPath(tt.sourcePath, source, target) assert.NoError(t, err) - assert.Equal(t, "target-cluster-root/files/insert_log/444/555/666/100/log1.log", gotPath) + assert.Equal(t, "target-cluster-root/insert_log/444/555/666/100/log1.log", gotPath) }) } } @@ -414,7 +518,7 @@ func TestGenerateTargetPath_RemapExternalS3SourcePath(t *testing.T) { ) assert.NoError(t, err) - assert.Equal(t, "target-cluster-root/files/insert_log/444/555/666/100/log1.log", gotPath) + assert.Equal(t, "target-cluster-root/insert_log/444/555/666/100/log1.log", gotPath) } func TestGenerateTargetPath_RejectsPathOutsideExternalRoot(t *testing.T) { @@ -457,11 +561,16 @@ func TestGenerateTargetPath_RemapExternalBucketRoot(t *testing.T) { { name: "configured target root", targetRoot: "target-cluster-root", - want: "target-cluster-root/files/insert_log/444/555/666/100/log1.log", + want: "target-cluster-root/insert_log/444/555/666/100/log1.log", }, { name: "target bucket root", - want: "files/insert_log/444/555/666/100/log1.log", + want: "insert_log/444/555/666/100/log1.log", + }, + { + name: "dot target bucket root", + targetRoot: ".", + want: "insert_log/444/555/666/100/log1.log", }, } @@ -486,6 +595,20 @@ func TestGenerateTargetPath_RemapExternalBucketRoot(t *testing.T) { } } +func TestGenerateTargetPath_InternalCopyBucketRoot(t *testing.T) { + source := &datapb.CopySegmentSource{CollectionId: 111, PartitionId: 222, SegmentId: 333} + for _, root := range []string{"", "."} { + t.Run(fmt.Sprintf("root_%q", root), func(t *testing.T) { + target := &datapb.CopySegmentTarget{ + CollectionId: 444, PartitionId: 555, SegmentId: 666, TargetRootPath: root, + } + got, err := generateTargetPath("insert_log/111/222/333/_data/0", source, target) + require.NoError(t, err) + assert.Equal(t, "insert_log/444/555/666/_data/0", got) + }) + } +} + func TestGenerateTargetPath_AllowsEquivalentExternalSchemes(t *testing.T) { tests := []struct { name string @@ -522,7 +645,7 @@ func TestGenerateTargetPath_AllowsEquivalentExternalSchemes(t *testing.T) { gotPath, err := generateTargetPath(tt.sourcePath, source, target) require.NoError(t, err) - assert.Equal(t, "target-cluster-root/files/insert_log/444/555/666/100/log1.log", gotPath) + assert.Equal(t, "target-cluster-root/insert_log/444/555/666/100/log1.log", gotPath) }) } } @@ -581,12 +704,14 @@ func TestGenerateTargetLOBPath(t *testing.T) { SegmentId: 333, } target := &datapb.CopySegmentTarget{ - CollectionId: 444, - PartitionId: 555, - SegmentId: 666, + TargetRootPath: "files", + CollectionId: 444, + PartitionId: 555, + SegmentId: 666, } tests := []struct { + targetRoot string name string sourcePath string wantPath string @@ -600,10 +725,18 @@ func TestGenerateTargetLOBPath(t *testing.T) { }, { name: "LOB file with nested field path", + targetRoot: "root/data", sourcePath: "root/data/insert_log/111/222/lobs/200/_data/xyz.vx", wantPath: "root/data/insert_log/444/555/lobs/200/_data/xyz.vx", wantErr: false, }, + { + name: "local complete filesystem path", + targetRoot: "/var/lib/milvus/data", + sourcePath: "/var/lib/milvus/data/insert_log/111/222/lobs/200/_data/xyz.vx", + wantPath: "/var/lib/milvus/data/insert_log/444/555/lobs/200/_data/xyz.vx", + wantErr: false, + }, { name: "invalid path - no insert_log", sourcePath: "files/other/111/222/lobs/100/_data/abc.vx", @@ -620,6 +753,10 @@ func TestGenerateTargetLOBPath(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + target.TargetRootPath = tt.targetRoot + if target.TargetRootPath == "" { + target.TargetRootPath = "files" + } gotPath, err := generateTargetLOBPath(tt.sourcePath, source, target) if tt.wantErr { assert.Error(t, err) @@ -652,7 +789,7 @@ func TestGenerateTargetLOBPath_RemapExternalRoot(t *testing.T) { ) assert.NoError(t, err) - assert.Equal(t, "target-cluster-root/files/insert_log/444/555/lobs/100/_data/abc123.vx", gotPath) + assert.Equal(t, "target-cluster-root/insert_log/444/555/lobs/100/_data/abc123.vx", gotPath) } func TestLobFileInfosToPaths(t *testing.T) { @@ -740,8 +877,8 @@ func TestCollectSegmentFiles_LOBFromManifest(t *testing.T) { assert.NotNil(t, files) // Verify LOB files come from manifest, not from directory listing. - // GetManifestLobFiles returns absolute paths (after ToAbsolutePaths in C++), - // so lobFileInfosToPaths uses them directly. + // The remote manifest base already contains its object root, so the + // manifest-expanded LOB paths are complete remote object keys. assert.Equal(t, 2, len(files.LobFiles)) assert.Equal(t, "root/insert_log/100/200/lobs/300/_data/seg1_file1.vx", files.LobFiles[0]) assert.Equal(t, "root/insert_log/100/200/lobs/300/_data/seg1_file2.vx", files.LobFiles[1]) @@ -758,12 +895,14 @@ func TestGenerateTargetIndexPath(t *testing.T) { SegmentId: 333, } target := &datapb.CopySegmentTarget{ - CollectionId: 444, - PartitionId: 555, - SegmentId: 666, + TargetRootPath: "files", + CollectionId: 444, + PartitionId: 555, + SegmentId: 666, } tests := []struct { + targetRoot string name string sourcePath string indexType string @@ -786,6 +925,14 @@ func TestGenerateTargetIndexPath(t *testing.T) { wantPath: "files/text_log/123/1/444/555/666/100/index_file", wantErr: false, }, + { + name: "local complete filesystem path", + targetRoot: "/var/lib/milvus/data", + sourcePath: "/var/lib/milvus/data/text_log/123/1/111/222/333/100/index_file", + indexType: IndexTypeText, + wantPath: "/var/lib/milvus/data/text_log/123/1/444/555/666/100/index_file", + wantErr: false, + }, { name: "json key index path", sourcePath: "files/json_key_index_log/123/1/111/222/333/100/index_file", @@ -840,6 +987,10 @@ func TestGenerateTargetIndexPath(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + target.TargetRootPath = tt.targetRoot + if target.TargetRootPath == "" { + target.TargetRootPath = "files" + } gotPath, err := generateTargetIndexPath(tt.sourcePath, source, target, tt.indexType, tt.pathVersion) if tt.wantErr { assert.Error(t, err) @@ -861,10 +1012,11 @@ func TestGenerateTargetIndexPath_BuildIDMapping(t *testing.T) { SegmentId: 333, } target := &datapb.CopySegmentTarget{ - CollectionId: 444, - PartitionId: 555, - SegmentId: 666, - NewBuildIds: map[int64]int64{1001: 2001}, + TargetRootPath: "files", + CollectionId: 444, + PartitionId: 555, + SegmentId: 666, + NewBuildIds: map[int64]int64{1001: 2001}, } tests := []struct { @@ -932,10 +1084,11 @@ func TestGenerateMappingsFromFiles_VectorScalarUsesSourcePathVersion(t *testing. }, } target := &datapb.CopySegmentTarget{ - CollectionId: 444, - PartitionId: 555, - SegmentId: 666, - NewBuildIds: map[int64]int64{1001: 2001, 1002: 2002}, + TargetRootPath: "files", + CollectionId: 444, + PartitionId: 555, + SegmentId: 666, + NewBuildIds: map[int64]int64{1001: 2001, 1002: 2002}, } files := &SegmentFiles{ VectorScalarIndex: []string{ @@ -973,7 +1126,7 @@ func TestGenerateTargetIndexPath_RemapExternalRoot(t *testing.T) { ) assert.NoError(t, err) - assert.Equal(t, "target-cluster-root/files/text_log/123/1/444/555/666/100/index_file", gotPath) + assert.Equal(t, "target-cluster-root/text_log/123/1/444/555/666/100/index_file", gotPath) } func TestTransformFieldBinlogs(t *testing.T) { @@ -1012,7 +1165,7 @@ func TestTransformFieldBinlogs(t *testing.T) { } t.Run("count rows for insert logs", func(t *testing.T) { - result, totalRows, err := transformFieldBinlogs(srcFieldBinlogs, mappings, true, false) + result, totalRows, err := transformFieldBinlogs(srcFieldBinlogs, mappings, transformFieldBinlogOptions{countRows: true}) assert.NoError(t, err) assert.Equal(t, int64(3000), totalRows) assert.Equal(t, 2, len(result)) @@ -1032,7 +1185,7 @@ func TestTransformFieldBinlogs(t *testing.T) { }) t.Run("no row counting for stats logs", func(t *testing.T) { - result, totalRows, err := transformFieldBinlogs(srcFieldBinlogs, mappings, false, false) + result, totalRows, err := transformFieldBinlogs(srcFieldBinlogs, mappings, transformFieldBinlogOptions{}) assert.NoError(t, err) assert.Equal(t, int64(0), totalRows) assert.Equal(t, 2, len(result)) @@ -1050,7 +1203,7 @@ func TestTransformFieldBinlogs(t *testing.T) { }, }, } - result, _, err := transformFieldBinlogs(srcWithEmpty, mappings, false, false) + result, _, err := transformFieldBinlogs(srcWithEmpty, mappings, transformFieldBinlogOptions{}) assert.NoError(t, err) assert.Equal(t, 0, len(result)) }) @@ -1067,7 +1220,7 @@ func TestTransformFieldBinlogs(t *testing.T) { }, }, } - result, totalRows, err := transformFieldBinlogs(srcWithUnmapped, mappings, true, false) + result, totalRows, err := transformFieldBinlogs(srcWithUnmapped, mappings, transformFieldBinlogOptions{countRows: true}) assert.Error(t, err) assert.Contains(t, err.Error(), "no mapping found") assert.Nil(t, result) @@ -1403,9 +1556,10 @@ func TestCopySegmentAndIndexFiles_ReturnsFileList(t *testing.T) { }, } target := &datapb.CopySegmentTarget{ - CollectionId: 444, - PartitionId: 555, - SegmentId: 666, + TargetRootPath: "files", + CollectionId: 444, + PartitionId: 555, + SegmentId: 666, } result, copiedFiles, err := CopySegmentAndIndexFiles( @@ -1454,9 +1608,10 @@ func TestCopySegmentAndIndexFiles_ReturnsFileList(t *testing.T) { }, } target := &datapb.CopySegmentTarget{ - CollectionId: 444, - PartitionId: 555, - SegmentId: 666, + TargetRootPath: "files", + CollectionId: 444, + PartitionId: 555, + SegmentId: 666, } result, copiedFiles, err := CopySegmentAndIndexFiles( @@ -1484,7 +1639,7 @@ func TestCopySegmentAndIndexFiles_NormalizesProviderSourceURI(t *testing.T) { copier := newCopySegmentCopierMock(t, func(_ context.Context, srcBucket, srcObject, dstBucket, dstObject string) error { assert.Equal(t, "source-bucket", srcBucket) assert.Equal(t, "target-bucket", dstBucket) - assert.Equal(t, "target-root/files/insert_log/444/555/666/1/10001", dstObject) + assert.Equal(t, "target-root/insert_log/444/555/666/1/10001", dstObject) copiedSource = srcObject return nil }) @@ -1581,9 +1736,10 @@ func TestGenerateTargetIndexPath_VectorScalarPreservesIDs(t *testing.T) { SegmentId: 333, } target := &datapb.CopySegmentTarget{ - CollectionId: 444, - PartitionId: 555, - SegmentId: 666, + TargetRootPath: "files", + CollectionId: 444, + PartitionId: 555, + SegmentId: 666, } // Verify buildID (1001) and indexVersion (1) are preserved, NOT replaced @@ -1852,7 +2008,7 @@ func TestBuildIndexInfoFromSource_UnmappedPaths(t *testing.T) { } func TestTransformFieldBinlogs_NilInput(t *testing.T) { - result, totalRows, err := transformFieldBinlogs(nil, map[string]string{}, true, false) + result, totalRows, err := transformFieldBinlogs(nil, map[string]string{}, transformFieldBinlogOptions{countRows: true}) assert.NoError(t, err) assert.Equal(t, int64(0), totalRows) assert.Equal(t, 0, len(result)) @@ -1876,7 +2032,7 @@ func TestTransformFieldBinlogs_MultipleBinlogsPerField(t *testing.T) { }, } - result, totalRows, err := transformFieldBinlogs(srcFieldBinlogs, mappings, true, false) + result, totalRows, err := transformFieldBinlogs(srcFieldBinlogs, mappings, transformFieldBinlogOptions{countRows: true}) assert.NoError(t, err) assert.Equal(t, int64(600), totalRows) assert.Equal(t, 1, len(result)) @@ -1899,7 +2055,7 @@ func TestTransformFieldBinlogs_UnmappedPath(t *testing.T) { }, } - _, _, err := transformFieldBinlogs(srcFieldBinlogs, mappings, true, false) + _, _, err := transformFieldBinlogs(srcFieldBinlogs, mappings, transformFieldBinlogOptions{countRows: true}) assert.Error(t, err) assert.Contains(t, err.Error(), "no mapping found for source path") } @@ -1976,7 +2132,7 @@ func TestShortenIndexFilePaths(t *testing.T) { } } -func TestShortenSingleJsonStatsPath(t *testing.T) { +func TestShortenJsonStatsPath_Files(t *testing.T) { tests := []struct { name string inputPath string @@ -2007,12 +2163,74 @@ func TestShortenSingleJsonStatsPath(t *testing.T) { inputPath: "files/other/path/file.json", expectedPath: "files/other/path/file.json", }, + { + name: "already shortened - meta.json", + inputPath: "meta.json", + expectedPath: "meta.json", + }, + { + name: "full path - meta.json", + inputPath: "files/json_stats/2/123/1/444/555/666/100/meta.json", + expectedPath: "meta.json", + }, + { + name: "full path - nested file", + inputPath: "files/json_stats/2/123/1/444/555/666/100/subdir/file.dat", + expectedPath: "subdir/file.dat", + }, + { + name: "local absolute path", + inputPath: "/var/lib/milvus/data/json_stats/2/123/1/444/555/666/100/meta.json", + expectedPath: "meta.json", + }, + { + name: "reserved names in root", + inputPath: "archive/json_stats/shared_key_index/files/json_stats/2/123/1/444/555/666/100/shared_key_index/index", + expectedPath: "shared_key_index/index", + }, + { + name: "shared key component in root", + inputPath: "archive/shared_key_index/files/json_stats/2/123/1/444/555/666/100/meta.json", + expectedPath: "meta.json", + }, + { + name: "shared key substring is not an anchor", + inputPath: "archive/not_shared_key_index/file", + expectedPath: "archive/not_shared_key_index/file", + }, + { + name: "data format mismatch", + inputPath: "files/json_stats/3/123/1/444/555/666/100/meta.json", + expectedPath: "files/json_stats/3/123/1/444/555/666/100/meta.json", + }, + { + name: "build ID mismatch", + inputPath: "files/json_stats/2/456/1/444/555/666/100/meta.json", + expectedPath: "files/json_stats/2/456/1/444/555/666/100/meta.json", + }, + { + name: "version mismatch", + inputPath: "files/json_stats/2/123/2/444/555/666/100/meta.json", + expectedPath: "files/json_stats/2/123/2/444/555/666/100/meta.json", + }, + { + name: "field ID mismatch", + inputPath: "files/json_stats/2/123/1/444/555/666/200/meta.json", + expectedPath: "files/json_stats/2/123/1/444/555/666/200/meta.json", + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - result := shortenSingleJSONStatsPath(tt.inputPath) - assert.Equal(t, tt.expectedPath, result) + stats := &datapb.JsonKeyStats{ + FieldID: 100, + Version: 1, + BuildID: 123, + JsonKeyStatsDataFormat: 2, + Files: []string{tt.inputPath}, + } + result := shortenJSONStatsPath(map[int64]*datapb.JsonKeyStats{100: stats}) + assert.Equal(t, []string{tt.expectedPath}, result[100].GetFiles()) }) } } @@ -2093,28 +2311,19 @@ func TestShortenJsonStatsPath_MetaJson(t *testing.T) { assert.Equal(t, "meta.json", result[102].Files[0]) } -func TestShortenSingleJsonStatsPath_EdgeCases(t *testing.T) { - t.Run("already_shortened_meta", func(t *testing.T) { - result := shortenSingleJSONStatsPath("meta.json") - assert.Equal(t, "meta.json", result) - }) +func TestShortenJsonStatsPath_NilEntries(t *testing.T) { + assert.Nil(t, shortenJSONStatsPath(nil)) + assert.Empty(t, shortenJSONStatsPath(map[int64]*datapb.JsonKeyStats{})) - t.Run("already_shortened_shared_key", func(t *testing.T) { - result := shortenSingleJSONStatsPath("shared_key_index/inverted_index_0") - assert.Equal(t, "shared_key_index/inverted_index_0", result) - }) - - t.Run("full_path_meta_json", func(t *testing.T) { - fullPath := "files/json_stats/2/123/1/444/555/666/100/meta.json" - result := shortenSingleJSONStatsPath(fullPath) - assert.Equal(t, "meta.json", result) - }) - - t.Run("full_path_nested_file", func(t *testing.T) { - fullPath := "files/json_stats/2/123/1/444/555/666/100/subdir/file.dat" - result := shortenSingleJSONStatsPath(fullPath) - assert.Equal(t, "subdir/file.dat", result) - }) + jsonStats := map[int64]*datapb.JsonKeyStats{ + 100: nil, + 200: {}, + } + result := shortenJSONStatsPath(jsonStats) + require.Len(t, result, 2) + assert.Nil(t, result[100]) + assert.Same(t, jsonStats[200], result[200]) + assert.Nil(t, result[200].GetFiles()) } func TestTransformManifestPath(t *testing.T) { @@ -2129,7 +2338,11 @@ func TestTransformManifestPath(t *testing.T) { 2, ) - source := &datapb.CopySegmentSource{} + source := &datapb.CopySegmentSource{ + CollectionId: 449104612037410004, + PartitionId: 449104621518610066, + SegmentId: 449104621518610065, + } targetManifest, err := transformManifestPath(sourceManifest, source, target) assert.NoError(t, err) @@ -2141,15 +2354,58 @@ func TestTransformManifestPath(t *testing.T) { assert.Contains(t, basePath, "2001") } +// Restoring from an external snapshot into local storage produces a manifest +// base in the target's canonical layout. The source's legacy files prefix +// must not survive below TargetRootPath. +func TestTransformManifestPath_ExternalSourceToLocalProducesCompletePath(t *testing.T) { + target := &datapb.CopySegmentTarget{ + SegmentId: 666, + CollectionId: 444, + PartitionId: 555, + TargetRootPath: "/var/lib/milvus/data", + } + source := &datapb.CopySegmentSource{ + CollectionId: 111, + PartitionId: 222, + SegmentId: 333, + SourceRootPath: "s3://source-bucket/source-root", + } + sourceManifest := packed.MarshalManifestPath( + "source-root/files/insert_log/111/222/333", + 2, + ) + + targetManifest, err := transformManifestPath(sourceManifest, source, target) + require.NoError(t, err) + basePath, version, err := packed.UnmarshalManifestPath(targetManifest) + require.NoError(t, err) + assert.Equal(t, int64(2), version) + assert.Equal(t, "/var/lib/milvus/data/insert_log/444/555/666", basePath) + + physicalPath, err := generateTargetPath( + "source-root/files/insert_log/111/222/333/_data/0", + source, + target, + ) + require.NoError(t, err) + assert.Equal(t, "/var/lib/milvus/data/insert_log/444/555/666/_data/0", physicalPath) +} + func TestTransformManifestPath_ExternalTable(t *testing.T) { target := &datapb.CopySegmentTarget{ - SegmentId: 666, - CollectionId: 444, - PartitionId: 555, + TargetRootPath: "files", + SegmentId: 666, + CollectionId: 444, + PartitionId: 555, } sourceManifest := packed.MarshalManifestPath("files/insert_log/111/222/333", 2) - source := &datapb.CopySegmentSource{IsExternalCollection: true} + source := &datapb.CopySegmentSource{ + CollectionId: 111, + PartitionId: 222, + SegmentId: 333, + IsExternalCollection: true, + } targetManifest, err := transformManifestPath(sourceManifest, source, target) assert.NoError(t, err) @@ -2167,7 +2423,12 @@ func TestTransformManifestPath_LegacyExternalTableLayoutUnsupported(t *testing.T } sourceManifest := packed.MarshalManifestPath("external/111/segments/333", 2) - source := &datapb.CopySegmentSource{IsExternalCollection: true} + source := &datapb.CopySegmentSource{ + CollectionId: 111, + PartitionId: 222, + SegmentId: 333, + IsExternalCollection: true, + } targetManifest, err := transformManifestPath(sourceManifest, source, target) assert.Error(t, err) assert.Empty(t, targetManifest) @@ -2188,7 +2449,7 @@ func TestTransformFieldBinlogs_SkipsPathMappingForExternalTable(t *testing.T) { }, }, } - got, totalRows, err := transformFieldBinlogs(src, nil, true, true) + got, totalRows, err := transformFieldBinlogs(src, nil, transformFieldBinlogOptions{countRows: true, isExternalTable: true}) assert.NoError(t, err) assert.Equal(t, int64(123), totalRows) assert.Len(t, got, 1) @@ -2200,7 +2461,7 @@ func TestTransformFieldBinlogs_SkipsPathMappingForExternalTable(t *testing.T) { assert.Equal(t, int64(456), got[0].GetBinlogs()[0].GetMemorySize()) } -func TestTransformFieldBinlogs_SkipsEmptyPathForInternalTable(t *testing.T) { +func TestTransformFieldBinlogs_DropsPathlessSummaryWhenNotPreserved(t *testing.T) { src := []*datapb.FieldBinlog{ { FieldID: 100, @@ -2210,12 +2471,38 @@ func TestTransformFieldBinlogs_SkipsEmptyPathForInternalTable(t *testing.T) { }, } - got, totalRows, err := transformFieldBinlogs(src, nil, false, false) + got, totalRows, err := transformFieldBinlogs(src, nil, transformFieldBinlogOptions{}) assert.NoError(t, err) assert.Zero(t, totalRows) assert.Empty(t, got) } +func TestTransformFieldBinlogs_PreservesPathlessSummary(t *testing.T) { + src := []*datapb.FieldBinlog{ + { + FieldID: 100, + Binlogs: []*datapb.Binlog{ + { + LogID: 10, + EntriesNum: 123, + MemorySize: 456, + TimestampFrom: 1000, + TimestampTo: 2000, + }, + }, + }, + } + + got, totalRows, err := transformFieldBinlogs(src, nil, transformFieldBinlogOptions{countRows: true, preservePathless: true}) + require.NoError(t, err) + require.Len(t, got, 1) + require.Len(t, got[0].GetBinlogs(), 1) + assert.Equal(t, int64(123), totalRows) + assert.Empty(t, got[0].GetBinlogs()[0].GetLogPath()) + assert.Equal(t, src[0].GetBinlogs()[0], got[0].GetBinlogs()[0]) + assert.NotSame(t, src[0].GetBinlogs()[0], got[0].GetBinlogs()[0]) +} + func TestGenerateSegmentInfoFromSource_PreservesExternalTableBinlogMetadata(t *testing.T) { source := &datapb.CopySegmentSource{ CollectionId: 111, @@ -2261,6 +2548,64 @@ func TestGenerateSegmentInfoFromSource_V3UsesDeclaredRowCount(t *testing.T) { assert.Equal(t, int64(4321), segmentInfo.GetImportedRows()) } +func TestGenerateSegmentInfoFromSource_V3PreservesPathlessSummaries(t *testing.T) { + source := &datapb.CopySegmentSource{ + StorageVersion: storage.StorageV3, + NumOfRows: 4321, + InsertBinlogs: []*datapb.FieldBinlog{ + { + FieldID: 10, + ChildFields: []int64{10}, + Format: "parquet", + Binlogs: []*datapb.Binlog{ + { + LogID: 1, + EntriesNum: 123, + MemorySize: 456, + TimestampFrom: 100, + TimestampTo: 200, + }, + }, + }, + }, + DeltaBinlogs: []*datapb.FieldBinlog{ + { + FieldID: 10, + Binlogs: []*datapb.Binlog{ + { + LogID: 2, + EntriesNum: 3, + MemorySize: 128, + TimestampFrom: 300, + TimestampTo: 400, + }, + }, + }, + }, + } + + segmentInfo, err := generateSegmentInfoFromSource( + source, + &datapb.CopySegmentTarget{SegmentId: 666}, + nil, + ) + require.NoError(t, err) + assert.Equal(t, int64(4321), segmentInfo.GetImportedRows()) + require.Len(t, segmentInfo.GetBinlogs(), 1) + require.Len(t, segmentInfo.GetDeltalogs(), 1) + assert.Equal(t, source.GetInsertBinlogs()[0], segmentInfo.GetBinlogs()[0]) + assert.Equal(t, source.GetDeltaBinlogs()[0], segmentInfo.GetDeltalogs()[0]) + + stats := storage.BuildStatsFromFieldBinlogs(segmentInfo.GetBinlogs(), nil, nil, segmentInfo.GetDeltalogs()) + assert.EqualValues(t, 456, stats.GetInsertBinlogSize()) + assert.EqualValues(t, 1, stats.GetInsertBinlogCount()) + assert.EqualValues(t, 128, stats.GetDeltaBinlogSize()) + assert.EqualValues(t, 3, stats.GetDeleteNumRows()) + assert.EqualValues(t, 1, stats.GetDeltaBinlogCount()) + assert.EqualValues(t, 300, stats.GetDeltaTimestampFrom()) + assert.EqualValues(t, 400, stats.GetDeltaTimestampTo()) +} + func TestCopySegmentAndIndexFiles_StorageV2CopiesManifest(t *testing.T) { sourceManifest := "files/insert_log/111/222/333/_metadata/manifest.json" sourceBinlog := "files/insert_log/111/222/333/100/1" @@ -2279,9 +2624,10 @@ func TestCopySegmentAndIndexFiles_StorageV2CopiesManifest(t *testing.T) { }}, } target := &datapb.CopySegmentTarget{ - CollectionId: 444, - PartitionId: 555, - SegmentId: 666, + TargetRootPath: "files", + CollectionId: 444, + PartitionId: 555, + SegmentId: 666, } copied := make(map[string]string) copier := newCopySegmentCopierMock(t, func(_ context.Context, _, src, _, dst string) error { @@ -2332,9 +2678,10 @@ func TestCopySegmentAndIndexFiles_ExternalTable(t *testing.T) { }, } target := &datapb.CopySegmentTarget{ - CollectionId: 444, - PartitionId: 555, - SegmentId: 666, + TargetRootPath: "files", + CollectionId: 444, + PartitionId: 555, + SegmentId: 666, } mList := mockey.Mock(listAllFiles).Return([]string{ @@ -2660,9 +3007,10 @@ func TestGenerateMappingsFromFiles(t *testing.T) { SegmentId: 333, } target := &datapb.CopySegmentTarget{ - CollectionId: 444, - PartitionId: 555, - SegmentId: 666, + TargetRootPath: "files", + CollectionId: 444, + PartitionId: 555, + SegmentId: 666, } t.Run("generate mappings for all file types", func(t *testing.T) { @@ -2753,9 +3101,10 @@ func TestCopySegmentAndIndexFiles_WithManifest(t *testing.T) { } target := &datapb.CopySegmentTarget{ - CollectionId: 444, - PartitionId: 555, - SegmentId: 666, + TargetRootPath: "files", + CollectionId: 444, + PartitionId: 555, + SegmentId: 666, } t.Run("successful copy with manifest", func(t *testing.T) { @@ -2902,7 +3251,7 @@ func TestCopySegmentAndIndexFiles_WithManifest(t *testing.T) { assert.Error(t, err) assert.Nil(t, result) - assert.Contains(t, err.Error(), "failed to generate target path for pb insert binlog") + assert.Contains(t, err.Error(), "failed to generate target path for protobuf binlog") }) } @@ -3051,6 +3400,241 @@ func TestListAllFiles(t *testing.T) { }) } +func TestCopySegmentAndIndexFiles_LocalStorageV3UsesCompletePaths(t *testing.T) { + mockNoManifestLobFiles(t) + + ctx := context.Background() + root := filepath.Join(t.TempDir(), "local", "root") + cm := storage.NewLocalChunkManager(objectstorage.RootPath(root)) + sourceKey := "insert_log/111/222/333/100/10001" + sourcePath := path.Join(root, sourceKey) + targetPath := path.Join(root, "insert_log/444/555/666/100/10001") + sourceDeltaKey := "insert_log/111/222/333/_delta/10002" + sourceDeltaPath := path.Join(root, sourceDeltaKey) + targetDeltaPath := path.Join(root, "insert_log/444/555/666/_delta/10002") + sourcePathlessDeltaKey := "insert_log/111/222/333/_delta/10003" + sourcePathlessDeltaPath := path.Join(root, sourcePathlessDeltaKey) + targetPathlessDeltaPath := path.Join(root, "insert_log/444/555/666/_delta/10003") + require.NoError(t, cm.Write(ctx, sourcePath, []byte("segment-data"))) + require.NoError(t, cm.Write(ctx, sourceDeltaPath, []byte("delta-data"))) + require.NoError(t, cm.Write(ctx, sourcePathlessDeltaPath, []byte("pathless-delta-data"))) + + source := &datapb.CopySegmentSource{ + CollectionId: 111, + PartitionId: 222, + SegmentId: 333, + StorageVersion: storage.StorageV3, + ManifestHasIndex: proto.Bool(false), + ManifestPath: packed.MarshalManifestPath(path.Join(root, "insert_log/111/222/333"), 1), + InsertBinlogs: []*datapb.FieldBinlog{{ + FieldID: 100, + Binlogs: []*datapb.Binlog{{ + LogPath: sourcePath, + EntriesNum: 10, + }}, + }}, + DeltaBinlogs: []*datapb.FieldBinlog{{ + Binlogs: []*datapb.Binlog{ + { + // Protobuf paths are the same complete keys as the physical + // files; step 3.5 maps them to the complete target path. + LogPath: sourceDeltaPath, + EntriesNum: 1, + }, + { + // V3 persists the physical delta in the manifest and keeps only + // this pathless summary in the protobuf metadata. + LogID: 10003, + EntriesNum: 2, + MemorySize: 128, + TimestampFrom: 300, + TimestampTo: 400, + }, + }, + }}, + } + target := &datapb.CopySegmentTarget{ + CollectionId: 444, + PartitionId: 555, + SegmentId: 666, + TargetRootPath: root, + } + storageConfig := &indexpb.StorageConfig{StorageType: "local", RootPath: root} + copySources := make(map[string]string) + copier := newCopySegmentCopierMock(t, func(ctx context.Context, _, src, _, dst string) error { + copySources[src] = dst + assert.True(t, path.IsAbs(src)) + assert.True(t, path.IsAbs(dst)) + return cm.Copy(ctx, src, dst) + }) + + result, copiedFiles, err := CopySegmentAndIndexFiles( + ctx, + cm, + storageConfig, + storageConfig, + copier, + "", + "", + source, + target, + nil, + ) + + require.NoError(t, err) + assert.Equal(t, map[string]string{ + sourcePath: targetPath, + sourceDeltaPath: targetDeltaPath, + sourcePathlessDeltaPath: targetPathlessDeltaPath, + }, copySources) + assert.ElementsMatch(t, []string{targetPath, targetDeltaPath, targetPathlessDeltaPath}, copiedFiles) + assert.Equal(t, packed.MarshalManifestPath(path.Join(root, "insert_log/444/555/666"), 1), result.GetManifestPath()) + require.Len(t, result.GetDeltalogs(), 1) + require.Len(t, result.GetDeltalogs()[0].GetBinlogs(), 2) + assert.Equal(t, int64(10002), result.GetDeltalogs()[0].GetBinlogs()[0].GetLogID()) + assert.Equal(t, int64(10003), result.GetDeltalogs()[0].GetBinlogs()[1].GetLogID()) + assert.Empty(t, result.GetDeltalogs()[0].GetBinlogs()[1].GetLogPath()) + assert.Equal(t, int64(2), result.GetDeltalogs()[0].GetBinlogs()[1].GetEntriesNum()) + assert.Equal(t, int64(128), result.GetDeltalogs()[0].GetBinlogs()[1].GetMemorySize()) + content, err := cm.Read(ctx, targetPath) + require.NoError(t, err) + assert.Equal(t, []byte("segment-data"), content) + deltaContent, err := cm.Read(ctx, targetDeltaPath) + require.NoError(t, err) + assert.Equal(t, []byte("delta-data"), deltaContent) + pathlessDeltaContent, err := cm.Read(ctx, targetPathlessDeltaPath) + require.NoError(t, err) + assert.Equal(t, []byte("pathless-delta-data"), pathlessDeltaContent) + assert.NoDirExists(t, path.Join(root, root), "local root must not be prepended twice") +} + +func TestCopySegmentAndIndexFiles_ExternalSourceToLocalCanonicalLayout(t *testing.T) { + ctx := context.Background() + root := t.TempDir() + cm := storage.NewLocalChunkManager(objectstorage.RootPath(root)) + const sourcePrefix = "source-root/files/" + const sourceBase = sourcePrefix + "insert_log/111/222/333" + const sourceLob = sourcePrefix + "insert_log/111/222/lobs/100/_data/lob.vx" + const sourceStats = sourcePrefix + "stats_log/111/222/333/100/10002" + const sourceBM25 = sourcePrefix + "bm25_stats/111/222/333/100/10003" + const sourceDelta = sourcePrefix + "delta_log/111/222/333/10004" + const sourceIndex = sourcePrefix + "index_v1/111/222/333/7000/1/index.v3" + mappings := map[string]string{ + sourceBase + "/_data/10001": "insert_log/444/555/666/_data/10001", + sourceBase + "/_manifest/17.avro": "insert_log/444/555/666/_manifest/17.avro", + sourceBase + "/_stats/text_index.100/idx": "insert_log/444/555/666/_stats/text_index.100/idx", + sourceBase + "/_stats/json_stats.101/meta.json": "insert_log/444/555/666/_stats/json_stats.101/meta.json", + sourceLob: "insert_log/444/555/lobs/100/_data/lob.vx", + sourceStats: "stats_log/444/555/666/100/10002", + sourceBM25: "bm25_stats/444/555/666/100/10003", + sourceDelta: "delta_log/444/555/666/10004", + sourceIndex: "index_v1/444/555/666/7777/1/index.v3", + } + // Mock the foreign provider's enumeration, but perform every destination + // write through a real LocalChunkManager. Manifest contents are opaque here; + // this test checks copy destinations, not native manifest deserialization. + mList := mockey.Mock(listAllFiles).To(func(_ context.Context, _ storage.ChunkManager, base string) ([]string, error) { + require.Equal(t, sourceBase, base) + return []string{ + sourceBase + "/_data/10001", sourceBase + "/_manifest/17.avro", + sourceBase + "/_stats/text_index.100/idx", sourceBase + "/_stats/json_stats.101/meta.json", + }, nil + }).Build() + defer mList.UnPatch() + mLob := mockey.Mock(packed.GetManifestLobFiles).Return([]packed.LobFileInfo{{Path: sourceLob, FieldID: 100}}, nil).Build() + defer mLob.UnPatch() + binlog := func(file string) []*datapb.FieldBinlog { + return []*datapb.FieldBinlog{{FieldID: 100, Binlogs: []*datapb.Binlog{{LogPath: file, EntriesNum: 1}}}} + } + source := &datapb.CopySegmentSource{ + CollectionId: 111, PartitionId: 222, SegmentId: 333, + SourceRootPath: "s3://source-bucket/source-root", + StorageVersion: storage.StorageV3, ManifestHasIndex: proto.Bool(false), ManifestPath: packed.MarshalManifestPath(sourceBase, 17), NumOfRows: 1, + StatsBinlogs: binlog(sourceStats), Bm25Binlogs: binlog(sourceBM25), DeltaBinlogs: binlog(sourceDelta), + IndexFiles: []*indexpb.IndexFilePathInfo{{ + BuildID: 7000, IndexFilePaths: []string{sourceIndex}, + IndexStorePathVersion: indexpb.IndexStorePathVersion_INDEX_STORE_PATH_VERSION_COLLECTION_ROOTED, + }}, + } + target := &datapb.CopySegmentTarget{ + CollectionId: 444, PartitionId: 555, SegmentId: 666, + TargetRootPath: root, NewBuildIds: map[int64]int64{7000: 7777}, + } + copied := make(map[string]string) + copier := newCopySegmentCopierMock(t, func(ctx context.Context, srcBucket, src, dstBucket, dst string) error { + assert.Equal(t, "source-bucket", srcBucket) + assert.Empty(t, dstBucket) + suffix, ok := mappings[src] + require.True(t, ok, "unexpected source file %q", src) + assert.Equal(t, path.Join(root, suffix), dst) + copied[src] = dst + return cm.Write(ctx, dst, []byte(src)) + }) + result, copiedFiles, err := CopySegmentAndIndexFiles(ctx, &struct{ storage.ChunkManager }{}, + &indexpb.StorageConfig{StorageType: "remote", RootPath: "source-root", BucketName: "source-bucket"}, + &indexpb.StorageConfig{StorageType: "local", RootPath: root}, + copier, "source-bucket", "", source, target, nil) + require.NoError(t, err) + require.Len(t, copiedFiles, len(mappings)) + require.Len(t, copied, len(mappings)) + assert.Equal(t, packed.MarshalManifestPath(path.Join(root, "insert_log/444/555/666"), 17), result.GetManifestPath()) + for src, suffix := range mappings { + contents, err := cm.Read(ctx, path.Join(root, suffix)) + require.NoError(t, err) + assert.Equal(t, src, string(contents)) + } + assert.NoDirExists(t, path.Join(root, "files"), "a restore must not create the source's legacy namespace") + assert.NoDirExists(t, path.Join(root, "source-root")) +} + +func TestGenerateMappingsFromFiles_RemoteCanonicalTargetRoot(t *testing.T) { + const prefix = "source-root/files/" + const insert = prefix + "insert_log/111/222/333/_data/10001" + const delta = prefix + "delta_log/111/222/333/10002" + const stats = prefix + "stats_log/111/222/333/100/10003" + const bm25 = prefix + "bm25_stats/111/222/333/100/10004" + const lob = prefix + "insert_log/111/222/lobs/100/_data/lob.vx" + const v0 = prefix + "index_files/7000/1/222/333/index.v3" + const v1 = prefix + "index_v1/111/222/333/7000/1/index.v3" + const text = prefix + "text_log/7000/1/111/222/333/100/index" + const jsonKey = prefix + "json_key_index_log/7000/1/111/222/333/100/index" + const jsonStats = prefix + "json_stats/2/7000/1/111/222/333/100/meta.json" + source := &datapb.CopySegmentSource{ + CollectionId: 111, PartitionId: 222, SegmentId: 333, + SourceRootPath: "s3://source-bucket/source-root", + IndexFiles: []*indexpb.IndexFilePathInfo{{ + IndexFilePaths: []string{v1}, + IndexStorePathVersion: indexpb.IndexStorePathVersion_INDEX_STORE_PATH_VERSION_COLLECTION_ROOTED, + }}, + } + files := &SegmentFiles{ + InsertBinlogs: []string{insert}, DeltaBinlogs: []string{delta}, StatsBinlogs: []string{stats}, Bm25Binlogs: []string{bm25}, + LobFiles: []string{lob}, VectorScalarIndex: []string{v0, v1}, TextIndex: []string{text}, JSONKeyIndex: []string{jsonKey}, JSONStats: []string{jsonStats}, + } + for _, root := range []string{"files", "tenant/files", "", "."} { + t.Run(fmt.Sprintf("root_%q", root), func(t *testing.T) { + target := &datapb.CopySegmentTarget{ + CollectionId: 444, PartitionId: 555, SegmentId: 666, + TargetRootPath: root, NewBuildIds: map[int64]int64{7000: 7777}, + } + got, err := generateMappingsFromFiles(files, source, target) + require.NoError(t, err) + assert.Equal(t, map[string]string{ + insert: path.Join(root, "insert_log/444/555/666/_data/10001"), + delta: path.Join(root, "delta_log/444/555/666/10002"), + stats: path.Join(root, "stats_log/444/555/666/100/10003"), + bm25: path.Join(root, "bm25_stats/444/555/666/100/10004"), + lob: path.Join(root, "insert_log/444/555/lobs/100/_data/lob.vx"), + v0: path.Join(root, "index_files/7777/1/555/666/index.v3"), + v1: path.Join(root, "index_v1/444/555/666/7777/1/index.v3"), + text: path.Join(root, "text_log/7777/1/444/555/666/100/index"), + jsonKey: path.Join(root, "json_key_index_log/7777/1/444/555/666/100/index"), + jsonStats: path.Join(root, "json_stats/2/7777/1/444/555/666/100/meta.json"), + }, got) + }) + } +} + // TestCopySegmentAndIndexFiles_V3WithTextAndJsonStats verifies that V3 segments // with text index and JSON key stats succeed during copy. Before the fix, V3 // segments had wrong-format paths in TextIndexFiles/JsonKeyIndexFiles (etcd @@ -3099,10 +3683,11 @@ func TestCopySegmentAndIndexFiles_V3WithTextAndJsonStats(t *testing.T) { } target := &datapb.CopySegmentTarget{ - CollectionId: 444, - PartitionId: 555, - SegmentId: 666, - NewBuildIds: map[int64]int64{7000: 7777, 8000: 9000}, + TargetRootPath: "files", + CollectionId: 444, + PartitionId: 555, + SegmentId: 666, + NewBuildIds: map[int64]int64{7000: 7777, 8000: 9000}, } mList := mockey.Mock(listAllFiles).To(func(_ context.Context, _ storage.ChunkManager, basePath string) ([]string, error) { diff --git a/internal/datanode/importv2/task_copy_segment_test.go b/internal/datanode/importv2/task_copy_segment_test.go index ea4e1e8f19..794a7f8c96 100644 --- a/internal/datanode/importv2/task_copy_segment_test.go +++ b/internal/datanode/importv2/task_copy_segment_test.go @@ -35,6 +35,7 @@ import ( "github.com/milvus-io/milvus/pkg/v3/proto/datapb" "github.com/milvus-io/milvus/pkg/v3/proto/indexpb" "github.com/milvus-io/milvus/pkg/v3/util/conc" + "github.com/milvus-io/milvus/pkg/v3/util/metautil" ) type crossBucketCopyCall struct { @@ -57,7 +58,15 @@ func copySegmentTaskTestDependencies( t.Helper() storageConfig := req.GetStorageConfig() if storageConfig == nil { - storageConfig = &indexpb.StorageConfig{BucketName: "test-bucket"} + storageConfig = &indexpb.StorageConfig{BucketName: "test-bucket", RootPath: "files"} + req.StorageConfig = storageConfig + } + // AssembleCopySegmentRequest supplies the target storage root in production. + // Keep explicit empty/dot storage roots as bucket-root configurations. + for _, target := range req.GetTargets() { + if target.GetTargetRootPath() == "" { + target.TargetRootPath = storageConfig.GetRootPath() + } } var copier storage.CrossBucketCopier if len(copiers) > 0 { @@ -965,7 +974,7 @@ func TestCopySegmentTaskWithIndexFiles(t *testing.T) { FieldID: 100, IndexID: 1001, BuildID: 1002, - IndexFilePaths: []string{"files/index_files/111/222/333/100/1001/1002/index1"}, + IndexFilePaths: metautil.NewIndexPathBuilder("files", indexpb.IndexStorePathVersion_INDEX_STORE_PATH_VERSION_BUILD_ROOTED, 111, 222, 333, 1002, 1).BuildFilePaths([]string{"index1"}), SerializedSize: 5000, }, }, @@ -1167,7 +1176,7 @@ func TestCopySegmentTaskWithIndexFiles(t *testing.T) { FieldID: 100, IndexID: 1001, BuildID: 1002, - IndexFilePaths: []string{"files/index_files/111/222/333/100/1001/1002/index1"}, + IndexFilePaths: metautil.NewIndexPathBuilder("files", indexpb.IndexStorePathVersion_INDEX_STORE_PATH_VERSION_BUILD_ROOTED, 111, 222, 333, 1002, 1).BuildFilePaths([]string{"index1"}), SerializedSize: 5000, }, }, diff --git a/internal/datanode/importv2/util.go b/internal/datanode/importv2/util.go index 7d5d05b07f..77e493e101 100644 --- a/internal/datanode/importv2/util.go +++ b/internal/datanode/importv2/util.go @@ -20,7 +20,6 @@ import ( "context" "fmt" "math/rand" - "path" "strconv" "time" @@ -42,7 +41,6 @@ import ( "github.com/milvus-io/milvus/pkg/v3/proto/datapb" "github.com/milvus-io/milvus/pkg/v3/proto/indexpb" "github.com/milvus-io/milvus/pkg/v3/util/merr" - "github.com/milvus-io/milvus/pkg/v3/util/metautil" "github.com/milvus-io/milvus/pkg/v3/util/paramtable" "github.com/milvus-io/milvus/pkg/v3/util/retry" "github.com/milvus-io/milvus/pkg/v3/util/typeutil" @@ -76,8 +74,7 @@ func NewSyncTask(ctx context.Context, } // init first manifest path if useLoonFFI { - k := metautil.JoinIDPath(collectionID, partitionID, segmentID) - basePath := path.Join(storageConfig.GetRootPath(), common.SegmentInsertLogPath, k) + basePath := storage.SegmentManifestBasePath(storageConfig.GetRootPath(), collectionID, partitionID, segmentID) // ManifestEarliest for first write segment.ManifestPath = packed.MarshalManifestPath(basePath, packed.ManifestEarliest) } diff --git a/internal/flushcommon/syncmgr/growing_source.go b/internal/flushcommon/syncmgr/growing_source.go index e8a9c24748..07b242a8ec 100644 --- a/internal/flushcommon/syncmgr/growing_source.go +++ b/internal/flushcommon/syncmgr/growing_source.go @@ -42,7 +42,6 @@ import ( "github.com/milvus-io/milvus/pkg/v3/proto/datapb" "github.com/milvus-io/milvus/pkg/v3/proto/indexpb" "github.com/milvus-io/milvus/pkg/v3/util/merr" - "github.com/milvus-io/milvus/pkg/v3/util/metautil" "github.com/milvus-io/milvus/pkg/v3/util/paramtable" "github.com/milvus-io/milvus/pkg/v3/util/retry" "github.com/milvus-io/milvus/pkg/v3/util/timerecord" @@ -873,10 +872,24 @@ func (t *GrowingSourceSyncTask) buildFlushConfig(segment *metacache.SegmentInfo, return nil, merr.WrapErrDataIntegrityMsg("growing source flush requires StorageV3 segment, segmentID=%d storageVersion=%d", t.segmentID, segment.GetStorageVersion()) } - segmentBasePath := path.Join(t.chunkManager.RootPath(), common.SegmentInsertLogPath, - metautil.JoinIDPath(t.collectionID, t.partitionID, t.segmentID)) - partitionBasePath := path.Join(t.chunkManager.RootPath(), common.SegmentInsertLogPath, - metautil.JoinIDPath(t.collectionID, t.partitionID)) + // Same key rule as ordinary flush and the initial manifest: the chunk + // manager's root is the storage prefix on every backend. + rootPath := t.chunkManager.RootPath() + segmentBasePath := storage.SegmentManifestBasePath(rootPath, t.collectionID, t.partitionID, t.segmentID) + partitionBasePath := storage.SegmentPartitionBasePath(rootPath, t.collectionID, t.partitionID) + manifestBasePath, readVersion, err := growingSourceManifest(segment.ManifestPath(), columnGroups) + if err != nil { + return nil, err + } + if manifestBasePath != "" { + // A base and a read version identify one manifest together, so they must + // come from the same place on every backend: changing only the base would + // reopen that version somewhere it does not exist. A recovered local + // segment can still live under the legacy minio prefix, which is why the + // stored base wins over the recomputed one. + segmentBasePath = manifestBasePath + partitionBasePath = path.Dir(manifestBasePath) + } allowedFieldIDs, allowedFieldSet := allowedFieldsFromColumnGroups(columnGroups) var textFieldIDs []int64 @@ -915,10 +928,6 @@ func (t *GrowingSourceSyncTask) buildFlushConfig(segment *metacache.SegmentInfo, if err != nil { return nil, err } - readVersion, err := growingSourceReadVersion(segment.ManifestPath(), columnGroups) - if err != nil { - return nil, err - } schemaBasedFormats := strings.Join(storagecommon.ColumnGroupFormats(columnGroups, writerFormat), ",") return &GrowingFlushConfig{ @@ -1018,24 +1027,24 @@ func (t *GrowingSourceSyncTask) fillPrimaryKeyStatsConfig(ctx context.Context, s return nil } -func growingSourceReadVersion(manifestPath string, columnGroups []storagecommon.ColumnGroup) (int64, error) { +func growingSourceManifest(manifestPath string, columnGroups []storagecommon.ColumnGroup) (string, int64, error) { if manifestPath == "" { - return packed.ManifestEarliest, nil + return "", packed.ManifestEarliest, nil } - _, version, err := packedManifestVersion(manifestPath) + basePath, version, err := packedManifestVersion(manifestPath) if err != nil { - return 0, err + return "", 0, err } if version == packed.ManifestEarliest { - return version, nil + return basePath, version, nil } for _, columnGroup := range columnGroups { if columnGroup.Format == "" { - return 0, merr.WrapErrDataIntegrityMsg("column group %d fields %v missing format for existing manifest %s", + return "", 0, merr.WrapErrDataIntegrityMsg("column group %d fields %v missing format for existing manifest %s", columnGroup.GroupID, columnGroup.Fields, manifestPath) } } - return version, nil + return basePath, version, nil } func allowedFieldsFromColumnGroups(columnGroups []storagecommon.ColumnGroup) ([]int64, map[int64]struct{}) { diff --git a/internal/flushcommon/syncmgr/growing_source_test.go b/internal/flushcommon/syncmgr/growing_source_test.go index 1bed0f8d25..099e500674 100644 --- a/internal/flushcommon/syncmgr/growing_source_test.go +++ b/internal/flushcommon/syncmgr/growing_source_test.go @@ -2,6 +2,7 @@ package syncmgr import ( "context" + "path" "testing" "github.com/cockroachdb/errors" @@ -20,7 +21,9 @@ import ( "github.com/milvus-io/milvus/internal/storagecommon" "github.com/milvus-io/milvus/internal/storagev2/packed" "github.com/milvus-io/milvus/pkg/v3/metrics" + "github.com/milvus-io/milvus/pkg/v3/objectstorage" "github.com/milvus-io/milvus/pkg/v3/proto/datapb" + "github.com/milvus-io/milvus/pkg/v3/proto/indexpb" "github.com/milvus-io/milvus/pkg/v3/util/merr" "github.com/milvus-io/milvus/pkg/v3/util/paramtable" ) @@ -277,6 +280,127 @@ func TestGrowingSourceSyncTaskBuildFlushConfigStartsFromEarliestManifest(t *test require.EqualValues(t, packed.ManifestEarliest, config.ReadVersion) } +func TestGrowingSourceFlushManifestNamespace(t *testing.T) { + params := paramtable.Get() + params.Init(paramtable.NewBaseTable()) + // New segments use the canonical storage prefix on every backend. Existing + // local segments also retain legacy namespaces recorded in their manifests. + for _, tc := range []struct { + name, root, legacyPrefix string + local bool + }{ + {"local", t.TempDir(), "", true}, + {"legacy local", t.TempDir(), "files", true}, + {"remote", "files", "", false}, + {"bucket root", "", "", false}, + {"dot bucket root", ".", "", false}, + } { + t.Run(tc.name, func(t *testing.T) { + for _, existing := range []bool{false, true} { + name := "new segment" + if existing { + name = "existing segment" + } + t.Run(name, func(t *testing.T) { + segmentBase := path.Join(tc.root, "insert_log/3/2/1") + manifestPath := "" + version := packed.ManifestEarliest + if existing { + segmentBase = path.Join(tc.root, tc.legacyPrefix, "insert_log/3/2/1") + version = 7 + manifestPath = packed.MarshalManifestPath(segmentBase, version) + } + partitionBase := path.Dir(segmentBase) + if tc.local { + require.True(t, path.IsAbs(segmentBase)) + } + var cm storage.ChunkManager + if tc.local { + cm = storage.NewLocalChunkManager(objectstorage.RootPath(tc.root)) + } else { + mockCM := mock_storage.NewMockChunkManager(t) + mockCM.EXPECT().RootPath().Return(tc.root) + cm = mockCM + } + segment := metacache.NewSegmentInfo(&datapb.SegmentInfo{ + ID: 1, PartitionID: 2, StorageVersion: storage.StorageV3, + ManifestPath: manifestPath, + }, pkoracle.NewBloomFilterSet(), nil, nil) + task := NewGrowingSourceSyncTask().WithCollectionID(3).WithPartitionID(2). + WithSegmentID(1).WithChunkManager(cm).WithSchema(&schemapb.CollectionSchema{ + Fields: []*schemapb.FieldSchema{{FieldID: 101, DataType: schemapb.DataType_Text}}, + }) + config, err := task.buildFlushConfig(segment, nil) + require.NoError(t, err) + require.Equal(t, segmentBase, config.SegmentBasePath) + require.Equal(t, partitionBase, config.PartitionBasePath) + require.Equal(t, []string{partitionBase + "/lobs/101"}, config.TextLobPaths) + require.EqualValues(t, version, config.ReadVersion) + }) + } + }) + } +} + +func TestGrowingSourceFlushResumesLegacyLocalManifest(t *testing.T) { + params := paramtable.Get() + params.Init(paramtable.NewBaseTable()) + root := t.TempDir() + for _, setting := range []struct{ key, value, previous string }{ + {params.CommonCfg.StorageType.Key, "local", params.CommonCfg.StorageType.GetValue()}, + {params.LocalStorageCfg.Path.Key, root, params.LocalStorageCfg.Path.GetValue()}, + } { + require.NoError(t, params.Save(setting.key, setting.value)) + t.Cleanup(func() { require.NoError(t, params.Save(setting.key, setting.previous)) }) + } + storageConfig := &indexpb.StorageConfig{StorageType: "local", RootPath: root} + legacyBase := path.Join(root, "files/insert_log/3/2/1") + statPath := path.Join(legacyBase, "_stats/bloom_filter.100/1") + require.NoError(t, packed.WriteFile(storageConfig, statPath, []byte("original stats"))) + manifest := packed.MarshalManifestPath(legacyBase, packed.ManifestEarliest) + for version := int64(0); version < 7; version++ { + var err error + manifest, err = packed.CommitManifestUpdates(legacyBase, version, storageConfig, &packed.ManifestUpdates{ + Stats: []packed.StatEntry{{Key: "bloom_filter.100", Files: []string{statPath}}}, + }) + require.NoError(t, err) + } + segment := metacache.NewSegmentInfo(&datapb.SegmentInfo{ + ID: 1, CollectionID: 3, PartitionID: 2, StorageVersion: storage.StorageV3, ManifestPath: manifest, + }, pkoracle.NewBloomFilterSet(), nil, nil) + task := NewGrowingSourceSyncTask().WithCollectionID(3).WithPartitionID(2).WithSegmentID(1). + WithChunkManager(storage.NewLocalChunkManager(objectstorage.RootPath(root))). + WithSchema(&schemapb.CollectionSchema{Fields: []*schemapb.FieldSchema{{FieldID: 101, DataType: schemapb.DataType_Text}}}) + config, err := task.buildFlushConfig(segment, nil) + require.NoError(t, err) + require.Equal(t, legacyBase, config.SegmentBasePath) + require.EqualValues(t, 7, config.ReadVersion) + require.Equal(t, path.Dir(legacyBase), config.PartitionBasePath) + require.Equal(t, []string{path.Join(path.Dir(legacyBase), "lobs/101")}, config.TextLobPaths) + + // Reopen the actual loon manifest, then append through the same base/version + // that growing flush will use. A canonical base paired with legacy version 7 + // cannot even reopen the old manifest, let alone preserve its existing stats. + stats, err := packed.GetManifestStats(packed.MarshalManifestPath(config.SegmentBasePath, config.ReadVersion), storageConfig) + require.NoError(t, err) + require.Equal(t, []string{statPath}, stats["bloom_filter.100"].Paths) + newStatPath := path.Join(config.SegmentBasePath, "_stats/bm25.102/2") + require.NoError(t, packed.WriteFile(storageConfig, newStatPath, []byte("new stats"))) + updatedManifest, err := packed.CommitManifestUpdates(config.SegmentBasePath, config.ReadVersion, storageConfig, &packed.ManifestUpdates{ + Stats: []packed.StatEntry{{Key: "bm25.102", Files: []string{newStatPath}}}, + }) + require.NoError(t, err) + require.Equal(t, packed.MarshalManifestPath(legacyBase, 8), updatedManifest) + stats, err = packed.GetManifestStats(updatedManifest, storageConfig) + require.NoError(t, err) + require.Len(t, stats, 2) + require.Equal(t, []string{statPath}, stats["bloom_filter.100"].Paths) + require.Equal(t, []string{newStatPath}, stats["bm25.102"].Paths) + content, err := packed.ReadFile(storageConfig, statPath) + require.NoError(t, err) + require.Equal(t, "original stats", string(content)) +} + func TestGrowingSourceSyncTaskBuildFlushConfigBM25AllocatorError(t *testing.T) { schema := &schemapb.CollectionSchema{ Name: "bm25", diff --git a/internal/flushcommon/syncmgr/meta_writer_test.go b/internal/flushcommon/syncmgr/meta_writer_test.go index 816abc6c27..ed7e4b8017 100644 --- a/internal/flushcommon/syncmgr/meta_writer_test.go +++ b/internal/flushcommon/syncmgr/meta_writer_test.go @@ -287,9 +287,9 @@ func (s *MetaWriterSuite) TestGrowingSourceSyncShipsStats() { ctx := context.Background() cfg := &indexpb.StorageConfig{StorageType: "local", RootPath: s.T().TempDir()} - basePath := "files/growing_stats/seg1" - bloomPath := path.Join(cfg.RootPath, basePath, "_stats/bloom_filter.100/1") - bm25Path := path.Join(cfg.RootPath, basePath, "_stats/bm25.102/1") + basePath := path.Join(cfg.RootPath, "growing_stats/seg1") + bloomPath := path.Join(basePath, "_stats/bloom_filter.100/1") + bm25Path := path.Join(basePath, "_stats/bm25.102/1") s.Require().NoError(packed.WriteFile(cfg, bloomPath, []byte("bloom"))) s.Require().NoError(packed.WriteFile(cfg, bm25Path, []byte("bm25"))) manifestPath, err := packed.CommitManifestUpdates(basePath, packed.ManifestEarliest, cfg, &packed.ManifestUpdates{ diff --git a/internal/flushcommon/syncmgr/pack_writer_v3_test.go b/internal/flushcommon/syncmgr/pack_writer_v3_test.go index ea4d10f8f5..d9dc72beb8 100644 --- a/internal/flushcommon/syncmgr/pack_writer_v3_test.go +++ b/internal/flushcommon/syncmgr/pack_writer_v3_test.go @@ -145,7 +145,7 @@ func (s *PackWriterV3Suite) TestPackWriterV3_Write() { bfs := pkoracle.NewBloomFilterSet() k := metautil.JoinIDPath(collectionID, partitionID, segmentID) - basePath := path.Join(common.SegmentInsertLogPath, k) + basePath := path.Join(s.rootPath, common.SegmentInsertLogPath, k) manifestPath := packed.MarshalManifestPath(basePath, packed.ManifestEarliest) seg := metacache.NewSegmentInfo(&datapb.SegmentInfo{ @@ -197,7 +197,7 @@ func (s *PackWriterV3Suite) TestPackWriterV3_UsesManifestFormatAfterConfigSwitch bfs := pkoracle.NewBloomFilterSet() k := metautil.JoinIDPath(collectionID, partitionID, segmentID) - basePath := path.Join(common.SegmentInsertLogPath, k) + basePath := path.Join(s.rootPath, common.SegmentInsertLogPath, k) manifestPath := packed.MarshalManifestPath(basePath, packed.ManifestEarliest) seg := metacache.NewSegmentInfo(&datapb.SegmentInfo{ @@ -266,7 +266,7 @@ func (s *PackWriterV3Suite) TestResolveInsertWriterFormatsUsesColumnGroupFormats } bw := NewBulkPackWriterV3(nil, s.schema, s.cm, s.logIDAlloc, packed.DefaultWriteBufferSize, 0, s.storageConfig, columnGroups, - packed.MarshalManifestPath("files/missing-manifest-segment", 42)) + packed.MarshalManifestPath(path.Join(s.rootPath, "missing-manifest-segment"), 42)) bw.initialManifestPath = bw.manifestPath writerFormat, schemaBasedFormats, err := bw.resolveInsertWriterFormats() @@ -287,7 +287,7 @@ func (s *PackWriterV3Suite) TestResolveInsertWriterFormatsRequiresFormatsForExis } bw := NewBulkPackWriterV3(nil, s.schema, s.cm, s.logIDAlloc, packed.DefaultWriteBufferSize, 0, s.storageConfig, columnGroups, - packed.MarshalManifestPath("files/existing-manifest-segment", 42)) + packed.MarshalManifestPath(path.Join(s.rootPath, "existing-manifest-segment"), 42)) bw.initialManifestPath = bw.manifestPath _, _, err := bw.resolveInsertWriterFormats() @@ -303,7 +303,7 @@ func (s *PackWriterV3Suite) TestWriteEmptyInsertData() { channelName := fmt.Sprintf("by-dev-rootcoord-dml_0_%dv0", collectionID) k := metautil.JoinIDPath(collectionID, partitionID, segmentID) - basePath := path.Join(common.SegmentInsertLogPath, k) + basePath := path.Join(s.rootPath, common.SegmentInsertLogPath, k) manifestPath := packed.MarshalManifestPath(basePath, packed.ManifestEarliest) bfs := pkoracle.NewBloomFilterSet() @@ -333,7 +333,7 @@ func (s *PackWriterV3Suite) TestNoPkField() { segmentID := int64(789) k := metautil.JoinIDPath(collectionID, partitionID, segmentID) - basePath := path.Join(common.SegmentInsertLogPath, k) + basePath := path.Join(s.rootPath, common.SegmentInsertLogPath, k) manifestPath := packed.MarshalManifestPath(basePath, packed.ManifestEarliest) channelName := fmt.Sprintf("by-dev-rootcoord-dml_0_%dv0", collectionID) @@ -361,7 +361,7 @@ func (s *PackWriterV3Suite) TestWriteInsertDataError() { channelName := fmt.Sprintf("by-dev-rootcoord-dml_0_%dv0", collectionID) k := metautil.JoinIDPath(collectionID, partitionID, segmentID) - basePath := path.Join(common.SegmentInsertLogPath, k) + basePath := path.Join(s.rootPath, common.SegmentInsertLogPath, k) manifestPath := packed.MarshalManifestPath(basePath, packed.ManifestEarliest) mc := metacache.NewMockMetaCache(s.T()) @@ -410,7 +410,7 @@ func (s *PackWriterV3Suite) TestWriteWithDeleteData() { bfs := pkoracle.NewBloomFilterSet() k := metautil.JoinIDPath(collectionID, partitionID, segmentID) - basePath := path.Join(common.SegmentInsertLogPath, k) + basePath := path.Join(s.rootPath, common.SegmentInsertLogPath, k) manifestPath := packed.MarshalManifestPath(basePath, packed.ManifestEarliest) seg := metacache.NewSegmentInfo(&datapb.SegmentInfo{ @@ -459,7 +459,7 @@ func (s *PackWriterV3Suite) TestV3InheritsV2Fields() { segmentID := int64(789) k := metautil.JoinIDPath(collectionID, partitionID, segmentID) - basePath := path.Join(common.SegmentInsertLogPath, k) + basePath := path.Join(s.rootPath, common.SegmentInsertLogPath, k) manifestPath := packed.MarshalManifestPath(basePath, packed.ManifestEarliest) mc := metacache.NewMockMetaCache(s.T()) @@ -523,7 +523,7 @@ func (s *PackWriterV3Suite) TestMultiBatchStatsAccumulation() { bfs := pkoracle.NewBloomFilterSet() k := metautil.JoinIDPath(collectionID, partitionID, segmentID) - basePath := path.Join(common.SegmentInsertLogPath, k) + basePath := path.Join(s.rootPath, common.SegmentInsertLogPath, k) manifestPath := packed.MarshalManifestPath(basePath, packed.ManifestEarliest) seg := metacache.NewSegmentInfo(&datapb.SegmentInfo{ @@ -613,7 +613,7 @@ func (s *PackWriterV3Suite) TestWrite_PropagatesPriorStatsReadError() { bfs := pkoracle.NewBloomFilterSet() k := metautil.JoinIDPath(collectionID, partitionID, segmentID) - basePath := path.Join(common.SegmentInsertLogPath, k) + basePath := path.Join(s.rootPath, common.SegmentInsertLogPath, k) manifestPath := packed.MarshalManifestPath(basePath, packed.ManifestEarliest) seg := metacache.NewSegmentInfo(&datapb.SegmentInfo{ManifestPath: manifestPath}, bfs, nil, metacache.NewEmptySegmentStats()) @@ -692,7 +692,7 @@ func (s *PackWriterV3Suite) TestMultiBatchBM25StatsAccumulation() { bfs := pkoracle.NewBloomFilterSet() k := metautil.JoinIDPath(collectionID, partitionID, segmentID) - basePath := path.Join(common.SegmentInsertLogPath, k) + basePath := path.Join(s.rootPath, common.SegmentInsertLogPath, k) manifestPath := packed.MarshalManifestPath(basePath, packed.ManifestEarliest) seg := metacache.NewSegmentInfo(&datapb.SegmentInfo{ @@ -770,7 +770,7 @@ func (s *PackWriterV3Suite) TestWrite_SingleVersionBumpAcrossSections() { bfs := pkoracle.NewBloomFilterSet() k := metautil.JoinIDPath(collectionID, partitionID, segmentID) - basePath := path.Join(common.SegmentInsertLogPath, k) + basePath := path.Join(s.rootPath, common.SegmentInsertLogPath, k) manifestPath := packed.MarshalManifestPath(basePath, packed.ManifestEarliest) seg := metacache.NewSegmentInfo(&datapb.SegmentInfo{ManifestPath: manifestPath}, bfs, nil, metacache.NewEmptySegmentStats()) @@ -846,7 +846,7 @@ func (s *PackWriterV3Suite) TestWrite_RetryDoesNotLeakVersionBumps() { bfs := pkoracle.NewBloomFilterSet() k := metautil.JoinIDPath(collectionID, partitionID, segmentID) - basePath := path.Join(common.SegmentInsertLogPath, k) + basePath := path.Join(s.rootPath, common.SegmentInsertLogPath, k) manifestPath := packed.MarshalManifestPath(basePath, packed.ManifestEarliest) seg := metacache.NewSegmentInfo(&datapb.SegmentInfo{ManifestPath: manifestPath}, bfs, nil, metacache.NewEmptySegmentStats()) metacache.UpdateNumOfRows(1000)(seg) diff --git a/internal/flushcommon/syncmgr/task_test.go b/internal/flushcommon/syncmgr/task_test.go index 675858ce8f..899405381b 100644 --- a/internal/flushcommon/syncmgr/task_test.go +++ b/internal/flushcommon/syncmgr/task_test.go @@ -220,7 +220,8 @@ func (s *SyncTaskSuite) createSegment(storageVersion int64) *metacache.SegmentIn // For StorageV3, set up a manifest path if storageVersion == storage.StorageV3 { k := fmt.Sprintf("%d/%d/%d", s.collectionID, s.partitionID, s.segmentID) - basePath := fmt.Sprintf("insert_log/%s", k) + // Complete key under the suite's local storage root ("/tmp"). + basePath := fmt.Sprintf("/tmp/insert_log/%s", k) // Use JSON format for manifest path: {"ver": 0, "base_path": "..."} segInfo.ManifestPath = packed.MarshalManifestPath(basePath, packed.ManifestEarliest) } diff --git a/internal/flushcommon/writebuffer/write_buffer.go b/internal/flushcommon/writebuffer/write_buffer.go index 925463994a..9e52248aa3 100644 --- a/internal/flushcommon/writebuffer/write_buffer.go +++ b/internal/flushcommon/writebuffer/write_buffer.go @@ -3,7 +3,6 @@ package writebuffer import ( "context" "fmt" - "path" "strings" "sync" "time" @@ -23,14 +22,12 @@ import ( "github.com/milvus-io/milvus/internal/storage" "github.com/milvus-io/milvus/internal/storagev2/packed" "github.com/milvus-io/milvus/internal/util/streamingutil" - "github.com/milvus-io/milvus/pkg/v3/common" "github.com/milvus-io/milvus/pkg/v3/metrics" "github.com/milvus-io/milvus/pkg/v3/mlog" "github.com/milvus-io/milvus/pkg/v3/mq/msgstream" "github.com/milvus-io/milvus/pkg/v3/proto/datapb" "github.com/milvus-io/milvus/pkg/v3/util/conc" "github.com/milvus-io/milvus/pkg/v3/util/merr" - "github.com/milvus-io/milvus/pkg/v3/util/metautil" "github.com/milvus-io/milvus/pkg/v3/util/paramtable" "github.com/milvus-io/milvus/pkg/v3/util/retry" "github.com/milvus-io/milvus/pkg/v3/util/tsoutil" @@ -1411,8 +1408,7 @@ func (wb *writeBufferBase) newGrowingSegmentManifestPath(partitionID int64, segm if storageVersion != storage.StorageV3 { return "" } - k := metautil.JoinIDPath(wb.collectionID, partitionID, segmentID) - basePath := path.Join(paramtable.Get().MinioCfg.RootPath.GetValue(), common.SegmentInsertLogPath, k) + basePath := storage.SegmentManifestBasePath(packed.CreateStorageConfig().GetRootPath(), wb.collectionID, partitionID, segmentID) return packed.MarshalManifestPath(basePath, packed.ManifestEarliest) } diff --git a/internal/querynodev2/segments/segment_loader.go b/internal/querynodev2/segments/segment_loader.go index e1522db22e..a084b356bc 100644 --- a/internal/querynodev2/segments/segment_loader.go +++ b/internal/querynodev2/segments/segment_loader.go @@ -43,6 +43,7 @@ import ( "github.com/milvus-io/milvus-proto/go-api/v3/commonpb" "github.com/milvus-io/milvus-proto/go-api/v3/schemapb" + "github.com/milvus-io/milvus/internal/metastore/kv/binlog" "github.com/milvus-io/milvus/internal/querynodev2/pkoracle" "github.com/milvus-io/milvus/internal/storage" "github.com/milvus-io/milvus/internal/storagecommon" @@ -1012,7 +1013,8 @@ func separateLoadInfoV2(loadInfo *querypb.SegmentLoadInfo, schema *schemapb.Coll // For V2 (non-manifest) segments, compute basePaths from metadata. // The resolver returns empty basePaths for V2; we compute them here. - rootPath := paramtable.Get().MinioCfg.RootPath.GetValue() + // Match the writer's primary storage root; local stats do not use MinIO's prefix. + rootPath := binlog.GetRootPath() for fieldID, stats := range textIndexedInfo { if _, ok := textBasePaths[fieldID]; !ok { textBasePaths[fieldID] = metautil.BuildTextIndexPrefix(rootPath, diff --git a/internal/querynodev2/segments/segment_loader_test.go b/internal/querynodev2/segments/segment_loader_test.go index 52ed0dedcf..f83ffaebf6 100644 --- a/internal/querynodev2/segments/segment_loader_test.go +++ b/internal/querynodev2/segments/segment_loader_test.go @@ -27,6 +27,7 @@ import ( "github.com/bytedance/mockey" "github.com/cockroachdb/errors" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" "go.uber.org/atomic" @@ -49,6 +50,39 @@ import ( "github.com/milvus-io/milvus/pkg/v3/util/typeutil" ) +func TestSeparateLoadInfoV2LegacyStatsRoot(t *testing.T) { + paramtable.Init() + params := paramtable.Get() + for _, item := range []*paramtable.ParamItem{¶ms.CommonCfg.StorageType, ¶ms.LocalStorageCfg.Path, ¶ms.MinioCfg.RootPath} { + key, value := item.Key, item.GetValue() + t.Cleanup(func() { require.NoError(t, params.Save(key, value)) }) + } + const localRoot = "/var/lib/milvus/data" + require.NoError(t, params.Save(params.LocalStorageCfg.Path.Key, localRoot)) + require.NoError(t, params.Save(params.MinioCfg.RootPath.Key, "files")) + for _, backend := range []struct{ name, root string }{{"local", localRoot}, {"remote", "files"}} { + t.Run(backend.name, func(t *testing.T) { + require.NoError(t, params.Save(params.CommonCfg.StorageType.Key, backend.name)) + for _, version := range []int64{storage.StorageV1, storage.StorageV2} { + loadInfo := &querypb.SegmentLoadInfo{ + StorageVersion: version, CollectionID: 2, PartitionID: 3, SegmentID: 4, + TextStatsLogs: map[int64]*datapb.TextIndexStats{ + 106: {FieldID: 106, BuildID: 10, Version: 1, Files: []string{"index.v3"}}, + }, + JsonKeyStatsLogs: map[int64]*datapb.JsonKeyStats{ + 107: {FieldID: 107, BuildID: 11, Version: 1, JsonKeyStatsDataFormat: 3, Files: []string{"meta.json", "shared_key_index/0", "shredding_data/0/0"}}, + }, + } + _, _, textStats, _, jsonStats, textBases, jsonBases := separateLoadInfoV2(loadInfo, &schemapb.CollectionSchema{}) + assert.Equal(t, backend.root+"/text_log/10/1/2/3/4/106", textBases[106]) + assert.Equal(t, backend.root+"/json_stats/3/11/1/2/3/4/107", jsonBases[107]) + assert.Equal(t, loadInfo.TextStatsLogs, textStats) + assert.Equal(t, loadInfo.JsonKeyStatsLogs, jsonStats) + } + }) + } +} + type SegmentLoaderSuite struct { suite.Suite loader Loader diff --git a/internal/snapshotio/storage/snapshot_data_test.go b/internal/snapshotio/storage/snapshot_data_test.go index 7be0049f1f..9c5e0d38e6 100644 --- a/internal/snapshotio/storage/snapshot_data_test.go +++ b/internal/snapshotio/storage/snapshot_data_test.go @@ -1525,6 +1525,10 @@ func TestListSnapshotDataFiles_StorageV3IncludesManifestRootObjectsAndLobs(t *te basePath := path.Join(tempDir, "files/insert_log/100/20/1001") require.NoError(t, cm.Write(ctx, path.Join(basePath, "manifest"), []byte("manifest"))) require.NoError(t, cm.Write(ctx, path.Join(basePath, "_data/cg0.parquet"), []byte("data"))) + deltaPath := path.Join(basePath, "_delta/7") + require.NoError(t, cm.Write(ctx, deltaPath, []byte("delta"))) + pathlessDeltaPath := path.Join(basePath, "_delta/8") + require.NoError(t, cm.Write(ctx, pathlessDeltaPath, []byte("pathless delta"))) siblingPath := basePath + "0/manifest" require.NoError(t, cm.Write(ctx, siblingPath, []byte("sibling"))) @@ -1552,7 +1556,15 @@ func TestListSnapshotDataFiles_StorageV3IncludesManifestRootObjectsAndLobs(t *te }, } segment.Statslogs = nil - segment.Deltalogs = nil + // StorageV3 delta metadata: one entry with a complete key and one + // compressed summary placeholder without a path (the manifest owns the + // physical file, which the manifest walk lists). + segment.Deltalogs = []*datapb.FieldBinlog{{ + Binlogs: []*datapb.Binlog{ + {LogPath: deltaPath}, + {LogID: 8}, + }, + }} segment.Bm25Statslogs = nil segment.IndexFiles = nil @@ -1560,12 +1572,20 @@ func TestListSnapshotDataFiles_StorageV3IncludesManifestRootObjectsAndLobs(t *te BucketName: "foreign-bucket", RootPath: "foreign-root", } + getLobCalls := 0 mockGetLobFiles := mockey.Mock(packed.GetManifestLobFiles).To( func(gotManifestPath string, gotStorageConfig *indexpb.StorageConfig) ([]packed.LobFileInfo, error) { assert.Equal(t, segment.GetManifestPath(), gotManifestPath) - assert.Same(t, foreignStorageConfig, gotStorageConfig) + // Listing passes the caller's storage config through; the rewrite + // below calls again with its own config. + if getLobCalls == 0 { + assert.Same(t, foreignStorageConfig, gotStorageConfig) + } + getLobCalls++ + // Manifest-expanded LOB paths are complete keys, so on local + // storage they are absolute like the manifest base. return []packed.LobFileInfo{ - {Path: "files/insert_log/100/20/lobs/30/_data/lob.vx", FieldID: 30}, + {Path: path.Join(tempDir, "files/insert_log/100/20/lobs/30/_data/lob.vx"), FieldID: 30}, }, nil }).Build() defer mockGetLobFiles.UnPatch() @@ -1581,11 +1601,32 @@ func TestListSnapshotDataFiles_StorageV3IncludesManifestRootObjectsAndLobs(t *te assert.Equal(t, snapshotstorage.SnapshotFileTypeStorageV3ManifestRoot, byPath[basePath].Type) assert.Equal(t, snapshotstorage.SnapshotFileTypeStorageV3ManifestObject, byPath[path.Join(basePath, "manifest")].Type) assert.Equal(t, snapshotstorage.SnapshotFileTypeStorageV3ManifestObject, byPath[path.Join(basePath, "_data/cg0.parquet")].Type) - assert.Equal(t, snapshotstorage.SnapshotFileTypeStorageV3LOBFile, byPath["files/insert_log/100/20/lobs/30/_data/lob.vx"].Type) + assert.Equal(t, snapshotstorage.SnapshotFileTypeStorageV3LOBFile, byPath[path.Join(tempDir, "files/insert_log/100/20/lobs/30/_data/lob.vx")].Type) assert.NotContains(t, byPath, "files/text_index/100/12/1001/7002/posting") assert.NotContains(t, byPath, "files/json_index/100/13/1001/7003/key") assert.NotContains(t, byPath, siblingPath) - assert.Len(t, byPath, 4) + assert.Contains(t, byPath, deltaPath) + assert.Contains(t, byPath, pathlessDeltaPath) + assert.Len(t, byPath, 6) + + // Rewriting keeps the pathless summary pathless instead of failing on a + // missing mapping for "". + mappings := make(map[string]string, len(refs)*2) + for _, ref := range refs { + destination := snapshotstorage.ExportedSnapshotPath(cm, ref.NormalizedPath, "bundle") + mappings[ref.Path] = destination + mappings[ref.NormalizedPath] = destination + } + rewritten, err := snapshotstorage.RewriteSnapshotWithMapping( + snapshot, + mappings, + "bundle", + "s3://bucket/bundle/snapshots/100/metadata/1.json", + ) + require.NoError(t, err) + assert.Equal(t, mappings[deltaPath], rewritten.Segments[0].GetDeltalogs()[0].GetBinlogs()[0].GetLogPath()) + assert.Empty(t, rewritten.Segments[0].GetDeltalogs()[0].GetBinlogs()[1].GetLogPath()) + assert.Equal(t, int64(8), rewritten.Segments[0].GetDeltalogs()[0].GetBinlogs()[1].GetLogID()) } func validateSnapshotDataFiles( diff --git a/internal/storage/binlog_record_writer.go b/internal/storage/binlog_record_writer.go index e23ea58425..8abd130ea8 100644 --- a/internal/storage/binlog_record_writer.go +++ b/internal/storage/binlog_record_writer.go @@ -59,6 +59,32 @@ type BinlogRecordWriter interface { Schema() *schemapb.CollectionSchema } +// SegmentManifestBasePath returns the loon manifest base path for a segment's +// insert log: /insert_log///. +// +// The rule is the same for every backend: a key is the storage prefix +// (minio.rootPath for remote, localStorage.path for local) followed by the +// layout, and the manifest stores exactly that key. The loon filesystem is +// rooted at the namespace root (the bucket, or "/" for local; see +// storagev2.LoonFSRootPath), so the manifest path, the ChunkManager key and the +// physical location are one string. Every primary-storage V3 writer (flush, +// compaction, import) must derive the path through this helper; building it +// from minio.rootPath under local storage was #53052. +func SegmentManifestBasePath(rootPath string, collectionID, partitionID, segmentID UniqueID) string { + k := metautil.JoinIDPath(collectionID, partitionID, segmentID) + return path.Join(rootPath, common.SegmentInsertLogPath, k) +} + +// SegmentPartitionBasePath returns the partition-level base path used for TEXT +// LOB files: /insert_log//. +// It shares the rule of SegmentManifestBasePath so LOB references written by +// mix/clustering compaction resolve to the same files the reader reconstructs +// from the manifest base. +func SegmentPartitionBasePath(rootPath string, collectionID, partitionID UniqueID) string { + k := metautil.JoinIDPath(collectionID, partitionID) + return path.Join(rootPath, common.SegmentInsertLogPath, k) +} + type packedBinlogRecordWriterBase struct { // attributes collectionID UniqueID @@ -477,8 +503,7 @@ func (pw *PackedManifestRecordWriter) initWriters(r Record) error { writerFormat, schemaBasedFormats := pw.fillV3ColumnGroupFormats() var err error - k := metautil.JoinIDPath(pw.collectionID, pw.partitionID, pw.segmentID) - pw.basePath = path.Join(pw.storageConfig.GetRootPath(), common.SegmentInsertLogPath, k) + pw.basePath = SegmentManifestBasePath(pw.storageConfig.GetRootPath(), pw.collectionID, pw.partitionID, pw.segmentID) pw.writer, err = newPackedRecordBatchWriter(pw.basePath, pw.schema, pw.bufferSize, pw.multiPartUploadSize, pw.columnGroups, pw.storageConfig, pw.storagePluginContext, true, pw.textRefsAsBinary, writerFormat, schemaBasedFormats) if err != nil { return merr.WrapErrStorage(err, "can not new packed record writer") @@ -710,8 +735,7 @@ func (pw *PackedTextManifestRecordWriter) initWriters(r Record) error { writerFormat, schemaBasedFormats := pw.fillV3ColumnGroupFormats() var err error - k := metautil.JoinIDPath(pw.collectionID, pw.partitionID, pw.segmentID) - pw.basePath = path.Join(pw.storageConfig.GetRootPath(), common.SegmentInsertLogPath, k) + pw.basePath = SegmentManifestBasePath(pw.storageConfig.GetRootPath(), pw.collectionID, pw.partitionID, pw.segmentID) pw.writer, err = NewPackedTextBatchWriter(pw.storageConfig.GetBucketName(), pw.basePath, pw.schema, pw.bufferSize, pw.multiPartUploadSize, pw.columnGroups, pw.storageConfig, pw.textColumnConfigs, writerFormat, schemaBasedFormats) if err != nil { return merr.WrapErrStorage(err, "can not new packed text writer") diff --git a/internal/storage/binlog_record_writer_v3_test.go b/internal/storage/binlog_record_writer_v3_test.go index 453f54e390..4d6c4828fc 100644 --- a/internal/storage/binlog_record_writer_v3_test.go +++ b/internal/storage/binlog_record_writer_v3_test.go @@ -355,3 +355,67 @@ func TestPackedTextManifestRecordWriter_FillsV3ColumnGroupFormats(t *testing.T) assert.Equal(t, "parquet", gotColumnGroups[1].Format) assert.Equal(t, gotColumnGroups, w.columnGroups) } + +// TestSegmentManifestBasePath verifies that the V3 manifest base path stays +// consistent with the loon filesystem root for both storage types. For local +// storage the filesystem is rooted at localStorage.path, so the key must stay +// relative to it and minio.rootPath must NOT leak in (see #53051, #53052). For +// remote storage the logical minio.rootPath is a key prefix and must be kept. +func TestSegmentManifestBasePath(t *testing.T) { + // One rule for every backend: /insert_log///. + // The prefix is localStorage.path for local and minio.rootPath for remote; + // the manifest stores the complete key and never a root-relative path. + localCfg := &indexpb.StorageConfig{StorageType: "local", RootPath: "/var/lib/milvus/data"} + assert.Equal(t, "/var/lib/milvus/data/insert_log/1/2/3", + SegmentManifestBasePath(localCfg.GetRootPath(), 1, 2, 3)) + assert.Equal(t, "/var/lib/milvus/data/insert_log/1/2", + SegmentPartitionBasePath(localCfg.GetRootPath(), 1, 2)) + + remoteCfg := &indexpb.StorageConfig{StorageType: "minio", RootPath: "files"} + assert.Equal(t, "files/insert_log/1/2/3", + SegmentManifestBasePath(remoteCfg.GetRootPath(), 1, 2, 3)) + assert.Equal(t, "files/insert_log/1/2", + SegmentPartitionBasePath(remoteCfg.GetRootPath(), 1, 2)) + + // Bucket-root remote layouts keep working: an empty prefix adds nothing. + bucketRootCfg := &indexpb.StorageConfig{StorageType: "remote", RootPath: ""} + assert.Equal(t, "insert_log/1/2/3", + SegmentManifestBasePath(bucketRootCfg.GetRootPath(), 1, 2, 3)) +} + +// TestPackedManifestRecordWriter_LocalStorageBasePath verifies that initWriters +// derives the complete key /insert_log//

/ for local +// storage, exactly as it derives /insert_log/... for remote. +func TestPackedManifestRecordWriter_LocalStorageBasePath(t *testing.T) { + schema := &schemapb.CollectionSchema{Fields: []*schemapb.FieldSchema{ + {FieldID: common.TimeStampField, DataType: schemapb.DataType_Int64}, + {FieldID: common.RowIDField, DataType: schemapb.DataType_Int64, IsPrimaryKey: true}, + }} + columnGroups := []storagecommon.ColumnGroup{ + {GroupID: 0, Columns: []int{0, 1}, Fields: []int64{common.TimeStampField, common.RowIDField}}, + } + + var gotBasePath string + patch := mockey.Mock(newPackedRecordBatchWriter).To( + func(basePath string, _ *schemapb.CollectionSchema, _, _ int64, _ []storagecommon.ColumnGroup, + _ *indexpb.StorageConfig, _ *indexcgopb.StoragePluginContext, _ bool, _ bool, + _ string, _ []string, + ) (*packedRecordBatchWriter, error) { + gotBasePath = basePath + return &packedRecordBatchWriter{}, nil + }).Build() + defer patch.UnPatch() + + root := t.TempDir() + cfg := &indexpb.StorageConfig{StorageType: "local", RootPath: root} + + w, err := newPackedManifestRecordWriter(1, 2, 3, schema, + ChunkedBlobsWriter(func(_ []*Blob) error { return nil }), + allocator.NewLocalAllocator(1, 1<<20), + 1024, 0, 0, columnGroups, cfg, nil, false, "") + require.NoError(t, err) + require.NoError(t, w.initWriters(nil)) + + assert.Equal(t, root+"/insert_log/1/2/3", w.basePath) + assert.Equal(t, root+"/insert_log/1/2/3", gotBasePath) +} diff --git a/internal/storage/local_chunk_manager.go b/internal/storage/local_chunk_manager.go index e3860ed46c..9d86b33ff5 100644 --- a/internal/storage/local_chunk_manager.go +++ b/internal/storage/local_chunk_manager.go @@ -172,11 +172,23 @@ func (lcm *LocalChunkManager) WalkWithPrefix(ctx context.Context, prefix string, if ctx.Err() != nil { return ctx.Err() } + // A missing scan root is an empty listing, including repeated GC. + if filePath == dir && os.IsNotExist(err) { + return nil + } + matches := strings.HasPrefix(filePath, prefix) + // Prune unrelated directories before handling their read errors. + // Keep matches (including insert_logExtra) and ancestors of prefix. + if f != nil && f.IsDir() && filePath != dir && + !matches && + !strings.HasPrefix(prefix, filePath+string(filepath.Separator)) { + return filepath.SkipDir + } if err != nil { return err } - if strings.HasPrefix(filePath, prefix) && !f.IsDir() { + if matches && !f.IsDir() { if !walkFunc(&ChunkObjectInfo{FilePath: filePath, ModifyTime: f.ModTime()}) { return nil } @@ -251,6 +263,16 @@ func (lcm *LocalChunkManager) Size(ctx context.Context, filePath string) (int64, } func (lcm *LocalChunkManager) Remove(ctx context.Context, filePath string) error { + // Keep an empty key as a no-op and never allow this API to remove the + // configured storage root. Some existing local-only cleanup paths remove a + // derived job subdirectory recursively, so retain that behavior below the + // root while making the dangerous root cases explicit. + if filePath == "" { + return nil + } + if lcm.localPath != "" && filepath.Clean(filePath) == filepath.Clean(lcm.localPath) { + return merr.WrapErrParameterInvalidMsg("refuse to remove local storage root") + } err := os.RemoveAll(filePath) return merr.WrapErrIoFailed(filePath, err) } diff --git a/internal/storage/local_chunk_manager_test.go b/internal/storage/local_chunk_manager_test.go index ed54255f22..b32cd23424 100644 --- a/internal/storage/local_chunk_manager_test.go +++ b/internal/storage/local_chunk_manager_test.go @@ -18,6 +18,7 @@ package storage import ( "context" + "os" "path" "path/filepath" "testing" @@ -30,6 +31,122 @@ import ( var localPath string +func TestLocalChunkManagerWalkWithPrefixPrunesUnrelatedDirectories(t *testing.T) { + for _, readOnlyParent := range []bool{false, true} { + name := "unreadable backup" + if readOnlyParent { + name = "read-only backup with unreadable child" + } + t.Run(name, func(t *testing.T) { + root := t.TempDir() + cm := NewLocalChunkManager(objectstorage.RootPath(root)) + key := filepath.Join(root, "insert_log/1/2/3/data") + backup := filepath.Join(root, ".backup") + blocked := backup + if readOnlyParent { + blocked = filepath.Join(backup, "nested") + } + require.NoError(t, cm.Write(t.Context(), key, []byte("live"))) + require.NoError(t, cm.Write(t.Context(), filepath.Join(blocked, "retained"), []byte("backup"))) + t.Cleanup(func() { + require.NoError(t, os.Chmod(backup, 0o700)) + require.NoError(t, os.Chmod(blocked, 0o700)) + }) + if readOnlyParent { + require.NoError(t, os.Chmod(backup, 0o500)) + } + require.NoError(t, os.Chmod(blocked, 0)) + if _, err := os.ReadDir(blocked); err == nil { + t.Skip("filesystem permissions do not prevent this user from reading the directory") + } else { + require.ErrorIs(t, err, os.ErrPermission) + } + var found []string + require.NoError(t, cm.WalkWithPrefix(t.Context(), filepath.Join(root, "insert_log"), true, func(info *ChunkObjectInfo) bool { + found = append(found, info.FilePath) + return true + })) + require.Equal(t, []string{key}, found) + + // Pruning unrelated directories must not hide a real error under + // the requested prefix itself. + err := cm.WalkWithPrefix(t.Context(), blocked+string(filepath.Separator), true, func(*ChunkObjectInfo) bool { return true }) + require.ErrorIs(t, err, os.ErrPermission) + }) + } +} + +func TestLocalChunkManagerWalkWithPrefixKeepsStringPrefixSemantics(t *testing.T) { + root := t.TempDir() + cm := NewLocalChunkManager(objectstorage.RootPath(root)) + keys := []string{"insert_log/1/data", "insert_log/10/data", "insert_logExtra/1/data", "insert_log.file", "unrelated/data"} + for _, key := range keys { + require.NoError(t, cm.Write(t.Context(), filepath.Join(root, key), []byte(key))) + } + for _, test := range []struct { + prefix string + want []string + }{ + {prefix: "insert_log", want: keys[:4]}, + {prefix: "insert_log/", want: keys[:2]}, + {prefix: "insert_log/1", want: keys[:2]}, + {prefix: "insert_log/1/", want: keys[:1]}, + {prefix: "insert_logExtra", want: keys[2:3]}, + {prefix: "missing"}, + } { + t.Run(test.prefix, func(t *testing.T) { + var found []string + // Concatenate rather than Join so a caller's trailing slash survives. + prefix := root + string(filepath.Separator) + filepath.FromSlash(test.prefix) + require.NoError(t, cm.WalkWithPrefix(t.Context(), prefix, true, func(info *ChunkObjectInfo) bool { + rel, err := filepath.Rel(root, info.FilePath) + require.NoError(t, err) + found = append(found, filepath.ToSlash(rel)) + return true + })) + require.ElementsMatch(t, test.want, found) + }) + } + t.Run("relative prefix", func(t *testing.T) { + t.Chdir(root) + var found []string + require.NoError(t, cm.WalkWithPrefix(t.Context(), "insert_log", true, func(info *ChunkObjectInfo) bool { + found = append(found, filepath.ToSlash(info.FilePath)) + return true + })) + require.ElementsMatch(t, keys[:4], found) + }) +} + +func TestLocalChunkManagerEmptyKeyDoesNotDeleteRoot(t *testing.T) { + ctx := context.Background() + root := t.TempDir() + testCM := NewLocalChunkManager(objectstorage.RootPath(root)) + sentinelPath := filepath.Join(root, "sentinel") + removedPath := filepath.Join(root, "remove-me") + require.NoError(t, testCM.Write(ctx, sentinelPath, []byte("keep"))) + require.NoError(t, testCM.Write(ctx, removedPath, []byte("remove"))) + + require.NoError(t, testCM.Remove(ctx, "")) + content, err := testCM.Read(ctx, sentinelPath) + require.NoError(t, err) + assert.Equal(t, []byte("keep"), content) + + err = testCM.Remove(ctx, root) + require.Error(t, err) + content, err = testCM.Read(ctx, sentinelPath) + require.NoError(t, err) + assert.Equal(t, []byte("keep"), content) + + require.NoError(t, testCM.MultiRemove(ctx, []string{"", removedPath})) + content, err = testCM.Read(ctx, sentinelPath) + require.NoError(t, err) + assert.Equal(t, []byte("keep"), content) + exists, err := testCM.Exist(ctx, removedPath) + require.NoError(t, err) + assert.False(t, exists) +} + func TestLocalCM(t *testing.T) { ctx := context.Background() @@ -731,3 +848,45 @@ func readAllChunkWithPrefix(ctx context.Context, manager ChunkManager, prefix st } return paths, contents, nil } + +// An object store returns an empty listing for a prefix that holds nothing, so +// the local manager must not report ENOENT for a directory that was never +// created or was already cleaned. Callers (segment GC) rely on prefix removal +// being idempotent across retries. +func TestLocalChunkManagerWalkMissingPrefixIsEmpty(t *testing.T) { + ctx := context.Background() + root := t.TempDir() + lcm := NewLocalChunkManager(objectstorage.RootPath(root)) + + for _, prefix := range []string{ + path.Join(root, "insert_log", "1", "2", "3"), + path.Join(root, "insert_log", "1", "2", "3") + "/", + path.Join(root, "never", "existed"), + } { + walked := 0 + require.NoError(t, lcm.WalkWithPrefix(ctx, prefix, true, func(*ChunkObjectInfo) bool { + walked++ + return true + })) + assert.Zero(t, walked) + require.NoError(t, lcm.RemoveWithPrefix(ctx, prefix)) + } + + // Removing a prefix twice must succeed both times. + key := path.Join(root, "insert_log", "1", "2", "4", "a.parquet") + require.NoError(t, lcm.Write(ctx, key, []byte("payload"))) + prefix := path.Join(root, "insert_log", "1", "2", "4") + "/" + require.NoError(t, lcm.RemoveWithPrefix(ctx, prefix)) + require.NoError(t, lcm.RemoveWithPrefix(ctx, prefix)) + exist, err := lcm.Exist(ctx, key) + require.NoError(t, err) + assert.False(t, exist) +} + +// An empty prefix matches every object in the bucket, so this API must refuse it +// rather than erase the instance. LocalChunkManager already did; the remote one +// is the backend where the blast radius is the whole bucket. +func TestRemoteChunkManagerRemoveWithPrefixRejectsEmptyPrefix(t *testing.T) { + rcm := &RemoteChunkManager{} + require.Error(t, rcm.RemoveWithPrefix(context.Background(), "")) +} diff --git a/internal/storage/localmigrate/displaced_empty_test.go b/internal/storage/localmigrate/displaced_empty_test.go new file mode 100644 index 0000000000..b57d1df18c --- /dev/null +++ b/internal/storage/localmigrate/displaced_empty_test.go @@ -0,0 +1,112 @@ +// Licensed to the LF AI & Data foundation under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 + +package localmigrate + +import ( + "io/fs" + "os" + "path/filepath" + "testing" + + "github.com/bytedance/mockey" + "github.com/stretchr/testify/require" + "golang.org/x/sys/unix" +) + +func TestMigrateDisplacedEmptySkeleton(t *testing.T) { + for _, layout := range LayoutDirs { + t.Run(layout, func(t *testing.T) { + root := t.TempDir() + t.Chdir(t.TempDir()) + skeleton := filepath.Join(layout, "1/2/3/empty") + require.NoError(t, os.MkdirAll(filepath.Join(displaced(root), skeleton), 0o755)) + canonical := filepath.Join(root, skeleton) + require.NoError(t, os.MkdirAll(canonical, 0o755)) + unrelated := filepath.Join(displaced(root), "unrelated/empty") + require.NoError(t, os.MkdirAll(unrelated, 0o755)) + + report, err := Migrate(t.Context(), root, Options{LegacyPrefix: "files"}) + require.NoError(t, err) + require.Empty(t, report.Candidates) + require.Zero(t, report.Renamed) + require.Zero(t, report.Copied) + require.NoDirExists(t, filepath.Join(displaced(root), layout)) + require.DirExists(t, canonical) + require.DirExists(t, unrelated) + require.DirExists(t, root) + }) + } +} + +func TestMigrateDisplacedEmptyRecoveryAfterLastFile(t *testing.T) { + root := t.TempDir() + t.Chdir(t.TempDir()) + key := "insert_log/100/10/2001/_data/data.parquet" + source := filepath.Join(displaced(root), key) + writeFile(t, source, "complete-data") + // Existing canonical files force a recursive merge instead of a subtree rename. + writeFile(t, filepath.Join(root, "insert_log/100/10/2001/_data/other.parquet"), "canonical-data") + var originalRemove func(*os.Root, string) error + patch := mockey.Mock((*os.Root).Remove).Origin(&originalRemove).To(func(opened *os.Root, name string) error { + if filepath.Join(opened.Name(), name) == filepath.Dir(source) { + return unix.EIO + } + return originalRemove(opened, name) + }).Build() + t.Cleanup(func() { patch.UnPatch() }) + report, err := Migrate(t.Context(), root, Options{LegacyPrefix: "files"}) + require.ErrorIs(t, err, unix.EIO) + require.Equal(t, 1, report.Renamed) + require.NoFileExists(t, source) + require.Equal(t, "complete-data", readFile(t, filepath.Join(root, key))) + patch.UnPatch() + + report, err = Migrate(t.Context(), root, Options{LegacyPrefix: "files"}) + require.NoError(t, err) + require.Empty(t, report.Candidates) + require.Zero(t, report.Renamed) + require.NoDirExists(t, filepath.Join(displaced(root), "insert_log")) + require.Equal(t, "canonical-data", readFile(t, filepath.Join(root, "insert_log/100/10/2001/_data/other.parquet"))) + + // Once repaired, another startup must not traverse the old empty skeleton. + walk := mockey.Mock(fs.WalkDir).To(func(fs.FS, string, fs.WalkDirFunc) error { + t.Fatal("repaired startup must not walk missing legacy layouts") + return nil + }).Build() + t.Cleanup(func() { walk.UnPatch() }) + _, err = Migrate(t.Context(), root, Options{LegacyPrefix: "files"}) + require.NoError(t, err) +} + +func TestMigrateDisplacedEmptyPreservesActiveNamespace(t *testing.T) { + root := t.TempDir() + t.Chdir(t.TempDir()) + empty := filepath.Join(displaced(root), "insert_log/1/2/3") + require.NoError(t, os.MkdirAll(empty, 0o755)) + legacyPrefix, err := filepath.Rel(root, displaced(root)) + require.NoError(t, err) + _, err = Migrate(t.Context(), root, Options{LegacyPrefix: legacyPrefix}) + require.NoError(t, err) + require.DirExists(t, empty) +} + +func TestMigrateDisplacedEmptyWaitsForSuccessfulMigration(t *testing.T) { + root := t.TempDir() + t.Chdir(t.TempDir()) + empty := filepath.Join(displaced(root), "text_log/1/2/3") + require.NoError(t, os.MkdirAll(empty, 0o755)) + key := "insert_log/1/2/3/data.parquet" + writeFile(t, filepath.Join(displaced(root), key), "source") + writeFile(t, filepath.Join(root, key), "conflicting-target") + _, err := Migrate(t.Context(), root, Options{LegacyPrefix: "files"}) + require.Error(t, err) + require.DirExists(t, empty) +} diff --git a/internal/storage/localmigrate/durability_test.go b/internal/storage/localmigrate/durability_test.go new file mode 100644 index 0000000000..c61cae34d1 --- /dev/null +++ b/internal/storage/localmigrate/durability_test.go @@ -0,0 +1,158 @@ +// Licensed to the LF AI & Data foundation under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 + +package localmigrate + +import ( + "io" + "os" + "path/filepath" + "testing" + + "github.com/bytedance/mockey" + "github.com/stretchr/testify/require" + "golang.org/x/sys/unix" +) + +func TestMigrateCopySyncsDestinationAncestorsBeforeSourceRemoval(t *testing.T) { + for _, publication := range []string{"fresh", "resumed"} { + t.Run(publication, func(t *testing.T) { + root, cwd, key := setupDurabilityCopy(t, publication == "resumed") + expected := destinationAncestorPaths(root, key) + var synced []string + var originalSync func(*os.File) error + syncPatch := mockey.Mock((*os.File).Sync).Origin(&originalSync).To(func(file *os.File) error { + if err := originalSync(file); err != nil { + return err + } + for _, directory := range expected { + if filepath.Clean(file.Name()) == directory { + synced = append(synced, directory) + } + } + return nil + }).Build() + defer syncPatch.UnPatch() + var originalRemove func(*os.Root, string) error + removed := 0 + removePatch := mockey.Mock((*os.Root).Remove).Origin(&originalRemove).To(func(opened *os.Root, name string) error { + if filepath.Join(opened.Name(), name) == filepath.Join(cwd, key) { + require.GreaterOrEqual(t, len(synced), len(expected)) + require.Equal(t, expected, synced[len(synced)-len(expected):], + "persist the complete destination directory chain before unlinking the source") + removed++ + } + return originalRemove(opened, name) + }).Build() + defer removePatch.UnPatch() + report, err := Migrate(t.Context(), root, Options{LegacyPrefix: "files"}) + require.NoError(t, err) + require.Equal(t, 1, removed) + require.Equal(t, 1, report.Copied) + require.Equal(t, "complete-index", readFile(t, filepath.Join(root, key))) + require.NoFileExists(t, filepath.Join(cwd, key)) + }) + } +} + +func TestMigrateCopyAncestorSyncFailureRetainsSourceAndRetries(t *testing.T) { + for _, publication := range []string{"fresh", "resumed"} { + for _, ancestor := range []string{"index_files/8/0/2/3", "index_files/8/0/2", "index_files/8/0", "index_files/8", "index_files", "."} { + t.Run(publication+"/"+ancestor, func(t *testing.T) { + root, cwd, key := setupDurabilityCopy(t, publication == "resumed") + failedDirectory := filepath.Join(root, ancestor) + failures := 0 + matchedSyncs := 0 + failOnSync := 1 + if publication == "fresh" && ancestor == filepath.Dir(key) { + failOnSync = 2 + } + var originalSync func(*os.File) error + syncPatch := mockey.Mock((*os.File).Sync).Origin(&originalSync).To(func(file *os.File) error { + if filepath.Clean(file.Name()) == failedDirectory { + matchedSyncs++ + // The leaf is synced once before the temporary file is + // renamed and again to persist publication. Fail the + // latter so every case exercises restart after publish. + if matchedSyncs == failOnSync { + failures++ + return unix.EIO + } + } + return originalSync(file) + }).Build() + defer syncPatch.UnPatch() + _, err := Migrate(t.Context(), root, Options{LegacyPrefix: "files"}) + require.ErrorIs(t, err, unix.EIO) + require.Equal(t, 1, failures) + require.Equal(t, "complete-index", readFile(t, filepath.Join(cwd, key))) + target := filepath.Join(root, key) + require.Equal(t, "complete-index", readFile(t, target)) + markers, err := filepath.Glob(filepath.Join(filepath.Dir(target), ".milvus-local-copy-*")) + require.NoError(t, err) + require.Len(t, markers, 1) + markerInfo, err := os.Stat(markers[0]) + require.NoError(t, err) + targetInfo, err := os.Stat(target) + require.NoError(t, err) + require.True(t, markerInfo.Mode().IsRegular()) + require.False(t, os.SameFile(markerInfo, targetInfo), "publication proof must not be a hard link") + require.Contains(t, readFile(t, markers[0]), "milvus-local-copy-v1") + syncPatch.UnPatch() + + copyCalls := 0 + copyPatch := mockey.Mock(io.Copy).To(func(io.Writer, io.Reader) (int64, error) { + copyCalls++ + return 0, unix.EIO + }).Build() + defer copyPatch.UnPatch() + report, err := Migrate(t.Context(), root, Options{LegacyPrefix: "files"}) + require.NoError(t, err, "retry should finish the already-published copy") + require.Zero(t, copyCalls) + require.Equal(t, 1, report.Copied) + require.Equal(t, "complete-index", readFile(t, target)) + require.NoFileExists(t, filepath.Join(cwd, key)) + require.NoFileExists(t, markers[0]) + }) + } + } +} + +func setupDurabilityCopy(t *testing.T, resumed bool) (root, cwd, key string) { + t.Helper() + root, cwd = t.TempDir(), t.TempDir() + key = "index_files/8/0/2/3/milvus_packed_inverted_index.v3" + t.Chdir(cwd) + writeFile(t, filepath.Join(cwd, key), "complete-index") + injectSourceRenameError(t, filepath.Join(cwd, filepath.Dir(filepath.Dir(key))), unix.EXDEV) + if resumed { + var originalRemove func(*os.Root, string) error + patch := mockey.Mock((*os.Root).Remove).Origin(&originalRemove).To(func(opened *os.Root, name string) error { + if filepath.Join(opened.Name(), name) == filepath.Join(cwd, key) { + return unix.EACCES + } + return originalRemove(opened, name) + }).Build() + defer patch.UnPatch() + _, err := Migrate(t.Context(), root, Options{LegacyPrefix: "files"}) + require.ErrorIs(t, err, unix.EACCES) + } + return root, cwd, key +} + +func destinationAncestorPaths(root, key string) []string { + var directories []string + for relative := filepath.Dir(key); ; relative = filepath.Dir(relative) { + directories = append(directories, filepath.Join(root, relative)) + if relative == "." { + return directories + } + } +} diff --git a/internal/storage/localmigrate/empty_dirs_test.go b/internal/storage/localmigrate/empty_dirs_test.go new file mode 100644 index 0000000000..8b475dee93 --- /dev/null +++ b/internal/storage/localmigrate/empty_dirs_test.go @@ -0,0 +1,191 @@ +// Licensed to the LF AI & Data foundation under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 + +package localmigrate + +import ( + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestMigrateLegacyCWDRemovesEmptyAncestors(t *testing.T) { + for _, key := range []string{ + "index_files/7/0/2/3/milvus_packed_inverted_index.v3", + "text_log/7/0/1/2/3/100/milvus_packed_text_index.v3", + } { + layout := strings.Split(key, "/")[0] + t.Run(layout, func(t *testing.T) { + root, cwd := t.TempDir(), t.TempDir() + t.Chdir(cwd) + writeFile(t, filepath.Join(cwd, key), "legacy-index") + _, err := Migrate(t.Context(), root, Options{LegacyPrefix: "files"}) + require.NoError(t, err) + require.Equal(t, "legacy-index", readFile(t, filepath.Join(root, key))) + require.NoDirExists(t, filepath.Join(cwd, layout), "remove the empty layout root, not just the packed leaf") + require.DirExists(t, cwd) + report, err := Migrate(t.Context(), root, Options{LegacyPrefix: "files"}) + require.NoError(t, err) + require.Empty(t, report.Candidates) + require.Zero(t, report.Renamed) + require.Zero(t, report.Copied) + }) + } +} + +func TestMigrateLegacyCWDCleansEmptySkeletonOnRestart(t *testing.T) { + for _, skeleton := range []string{ + "index_files/7/0/2/3", + "text_log/7/0/1/2/3/100", + } { + layout := strings.Split(skeleton, "/")[0] + t.Run(layout, func(t *testing.T) { + root, cwd := t.TempDir(), t.TempDir() + t.Chdir(cwd) + // Simulate interruption after the final packed file was moved: + // only eligible empty directories remain, so no file selects a leaf. + require.NoError(t, os.MkdirAll(filepath.Join(cwd, skeleton), 0o755)) + canonicalEmpty := filepath.Join(root, skeleton) + require.NoError(t, os.MkdirAll(canonicalEmpty, 0o755)) + report, err := Migrate(t.Context(), root, Options{LegacyPrefix: "files"}) + require.NoError(t, err) + require.Zero(t, report.Renamed) + require.Zero(t, report.Copied) + require.NoDirExists(t, filepath.Join(cwd, layout)) + require.DirExists(t, cwd) + require.DirExists(t, canonicalEmpty, "canonical empty directories are not legacy cleanup candidates") + }) + } +} + +func TestMigrateLegacyCWDEmptyCleanupPreservesUnrelatedEntries(t *testing.T) { + for _, key := range []string{ + "index_files/7/0/2/3/milvus_packed_inverted_index.v3", + "text_log/7/0/1/2/3/100/milvus_packed_text_index.v3", + } { + layout := strings.Split(key, "/")[0] + t.Run(layout, func(t *testing.T) { + root, cwd := t.TempDir(), t.TempDir() + t.Chdir(cwd) + writeFile(t, filepath.Join(cwd, key), "legacy-index") + unrelatedFile := filepath.Join(cwd, filepath.Dir(filepath.Dir(key)), "999", "legacy-index") + writeFile(t, unrelatedFile, "unrelated-data") + unknownEmpty := filepath.Join(cwd, layout, "notes", "empty") + tooDeepEmpty := filepath.Join(cwd, strings.Replace(filepath.Dir(key), "/7/", "/9/", 1), "111") + outsideEmpty := filepath.Join(cwd, "unrelated-empty") + for _, directory := range []string{unknownEmpty, tooDeepEmpty, outsideEmpty} { + require.NoError(t, os.MkdirAll(directory, 0o755)) + } + _, err := Migrate(t.Context(), root, Options{LegacyPrefix: "files"}) + require.NoError(t, err) + require.Equal(t, "legacy-index", readFile(t, filepath.Join(root, key))) + require.Equal(t, "unrelated-data", readFile(t, unrelatedFile)) + for _, directory := range []string{unknownEmpty, tooDeepEmpty, outsideEmpty, filepath.Join(cwd, layout)} { + require.DirExists(t, directory, "cleanup must stop at nonempty or unrecognized directories") + } + }) + } +} + +func TestMigrateLegacyCWDEmptyCleanupProtectsCanonicalRoot(t *testing.T) { + for _, nested := range []bool{false, true} { + name := "cwd-is-root" + if nested { + name = "root-inside-legacy-layout" + } + t.Run(name, func(t *testing.T) { + cwd := t.TempDir() + t.Chdir(cwd) + root := cwd + if nested { + root = filepath.Join(cwd, "index_files/7/0/2") + } + canonicalEmpty := filepath.Join(root, "3") + if !nested { + canonicalEmpty = filepath.Join(root, "index_files/7/0/2/3") + } + require.NoError(t, os.MkdirAll(canonicalEmpty, 0o755)) + _, err := Migrate(t.Context(), root, Options{LegacyPrefix: "files"}) + require.NoError(t, err) + require.DirExists(t, cwd) + require.DirExists(t, root) + require.DirExists(t, canonicalEmpty) + }) + } +} + +func TestMigrateLegacyCWDEmptyCleanupDoesNotDeleteReplacementFile(t *testing.T) { + for _, kind := range []string{"leaf", "ancestor"} { + t.Run(kind, func(t *testing.T) { + root, cwd := t.TempDir(), t.TempDir() + t.Chdir(cwd) + key := "index_files/7/0/2/3/milvus_packed_inverted_index.v3" + writeFile(t, filepath.Join(cwd, key), "legacy-index") + emptyLeaf := filepath.Join(cwd, "index_files/8/0/2/3") + require.NoError(t, os.MkdirAll(emptyLeaf, 0o755)) + replacement := emptyLeaf + if kind == "ancestor" { + replacement = filepath.Dir(emptyLeaf) + } + replaced := false + _, err := Migrate(t.Context(), root, Options{LegacyPrefix: "files", OnDirDone: func(Dir, time.Duration) { + if !replaced { + // Discovery already saw directories here. Change the leaf + // or its parent before cleanup; neither may unlink a file. + if kind == "ancestor" { + require.NoError(t, os.Remove(emptyLeaf)) + } + require.NoError(t, os.Remove(replacement)) + writeFile(t, replacement, "keep-replacement") + replaced = true + } + }}) + require.NoError(t, err) + require.True(t, replaced) + require.Equal(t, "keep-replacement", readFile(t, replacement)) + }) + } +} + +func TestMigrateLegacyCWDEmptyCleanupWaitsForSuccessfulMigration(t *testing.T) { + root, cwd := t.TempDir(), t.TempDir() + t.Chdir(cwd) + key := "index_files/7/0/2/3/milvus_packed_inverted_index.v3" + writeFile(t, filepath.Join(cwd, key), "legacy-index") + writeFile(t, filepath.Join(root, key), "conflicting-target") + empty := filepath.Join(cwd, "index_files/8/0/2/3") + require.NoError(t, os.MkdirAll(empty, 0o755)) + report, err := Migrate(t.Context(), root, Options{LegacyPrefix: "files"}) + require.Error(t, err) + require.NotEmpty(t, report.Conflicts) + require.DirExists(t, empty, "a failed migration must not proceed to empty-directory cleanup") + require.Equal(t, "legacy-index", readFile(t, filepath.Join(cwd, key))) + require.Equal(t, "conflicting-target", readFile(t, filepath.Join(root, key))) +} + +func TestMigrateLegacyCWDEmptyCleanupProtectsActiveAndReservedNamespaces(t *testing.T) { + for _, prefix := range []string{"files/insert_log/1/2/3", ".milvus-local-layout"} { + t.Run(prefix, func(t *testing.T) { + root := t.TempDir() + cwd := filepath.Join(root, prefix) + empty := filepath.Join(cwd, "index_files/7/0/2/3") + require.NoError(t, os.MkdirAll(empty, 0o755)) + t.Chdir(cwd) + _, err := Migrate(t.Context(), root, Options{LegacyPrefix: "files"}) + require.NoError(t, err) + require.DirExists(t, empty) + require.DirExists(t, cwd) + }) + } +} diff --git a/internal/storage/localmigrate/legacy_cwd_test.go b/internal/storage/localmigrate/legacy_cwd_test.go new file mode 100644 index 0000000000..976f98e868 --- /dev/null +++ b/internal/storage/localmigrate/legacy_cwd_test.go @@ -0,0 +1,155 @@ +// Licensed to the LF AI & Data foundation under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 + +package localmigrate + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestMigrateLegacyCWDUnifiedIndexesRenamesDeepestDirectories(t *testing.T) { + root, cwd := t.TempDir(), t.TempDir() + t.Chdir(cwd) + files := map[string]string{ + "text_log/7/0/1/2/3/100/milvus_packed_text_index.v3": "text-index", + "index_files/8/0/2/3/milvus_packed_inverted_index.v3": "scalar-index", + } + for key, content := range files { + writeFile(t, filepath.Join(cwd, key), content) + } + writeFile(t, filepath.Join(cwd, "notes.txt"), "unrelated") + report, err := Migrate(t.Context(), root, Options{LegacyPrefix: "files"}) + require.NoError(t, err) + assert.Equal(t, 2, report.Renamed) + assert.EqualValues(t, len("text-index")+len("scalar-index"), report.Bytes) + for key, content := range files { + assert.Equal(t, content, readFile(t, filepath.Join(root, key))) + assert.NoFileExists(t, filepath.Join(cwd, key)) + } + assert.Equal(t, "unrelated", readFile(t, filepath.Join(cwd, "notes.txt"))) + second, err := Migrate(t.Context(), root, Options{LegacyPrefix: "files"}) + require.NoError(t, err) + assert.Zero(t, second.Renamed) +} + +func TestMigrateLegacyCWDUnknownTargetConflicts(t *testing.T) { + root, cwd := t.TempDir(), t.TempDir() + t.Chdir(cwd) + key := "index_files/7/0/2/3/milvus_packed_inverted_index.v3" + writeFile(t, filepath.Join(cwd, key), "source") + writeFile(t, filepath.Join(root, key), "target") + report, err := Migrate(t.Context(), root, Options{LegacyPrefix: "files"}) + require.Error(t, err) + assert.NotEmpty(t, report.Conflicts) + assert.Equal(t, "source", readFile(t, filepath.Join(cwd, key))) +} + +func TestMigrateLegacyCWDMatchesDisplacedRoot(t *testing.T) { + for _, name := range []string{"direct", "symlink_alias"} { + t.Run(name, func(t *testing.T) { + root := t.TempDir() + cwd := displaced(root) + files := map[string]string{ + "text_log/7/0/1/2/3/100/milvus_packed_text_index.v3": "text-index", + "index_files/8/0/2/3/milvus_packed_inverted_index.v3": "scalar-index", + } + for key, content := range files { + writeFile(t, filepath.Join(cwd, key), content) + } + if name == "symlink_alias" { + alias := filepath.Join(t.TempDir(), "legacy") + require.NoError(t, os.Symlink(cwd, alias)) + t.Chdir(alias) + } else { + t.Chdir(cwd) + } + + report, err := Migrate(t.Context(), root, Options{LegacyPrefix: "files"}) + require.NoError(t, err) + assert.Empty(t, report.Conflicts) + assert.Len(t, report.Candidates, 1) + assert.Equal(t, 2, report.Renamed) + assert.EqualValues(t, len("text-index")+len("scalar-index"), report.Bytes) + for key, content := range files { + assert.Equal(t, content, readFile(t, filepath.Join(root, key))) + assert.NoFileExists(t, filepath.Join(cwd, key)) + } + + second, err := Migrate(t.Context(), root, Options{LegacyPrefix: "files"}) + require.NoError(t, err) + assert.Zero(t, second.Renamed) + }) + } +} + +func TestMigrateLegacyCWDDistinctFromDisplacedRootStillConflicts(t *testing.T) { + root, cwd := t.TempDir(), t.TempDir() + t.Chdir(cwd) + key := "index_files/8/0/2/3/milvus_packed_inverted_index.v3" + writeFile(t, filepath.Join(displaced(root), key), "displaced") + writeFile(t, filepath.Join(cwd, key), "cwd") + + report, err := Migrate(t.Context(), root, Options{LegacyPrefix: "files"}) + require.Error(t, err) + assert.NotEmpty(t, report.Conflicts) + assert.Zero(t, report.Renamed) + assert.NoFileExists(t, filepath.Join(root, key)) + assert.Equal(t, "displaced", readFile(t, filepath.Join(displaced(root), key))) + assert.Equal(t, "cwd", readFile(t, filepath.Join(cwd, key))) +} + +func TestMigrateLegacyCWDSkipsUnrelatedFiles(t *testing.T) { + root, cwd := t.TempDir(), t.TempDir() + t.Chdir(cwd) + for _, key := range []string{ + "notes.txt", + "index_files/7/0/2/3/legacy-index", + "index_files/7/0/2/3/milvus_packed_inverted_index.v2", + "text_log/7/0/1/2/3/100/index_meta.json", + } { + writeFile(t, filepath.Join(cwd, key), "unrelated") + } + report, err := Migrate(t.Context(), root, Options{LegacyPrefix: "files"}) + require.NoError(t, err) + assert.Zero(t, report.Renamed) + assert.Equal(t, "unrelated", readFile(t, filepath.Join(cwd, "notes.txt"))) + assert.NoDirExists(t, filepath.Join(root, "index_files")) +} + +func TestMigrateLegacyCWDNestedCanonicalRootIsSkipped(t *testing.T) { + cwd := t.TempDir() + t.Chdir(cwd) + root := filepath.Join(cwd, "data") + key := "index_files/7/0/2/3/milvus_packed_inverted_index.v3" + writeFile(t, filepath.Join(root, key), "canonical") + writeFile(t, filepath.Join(cwd, "index_files/8/0/2/3/milvus_packed_inverted_index.v3"), "legacy") + report, err := Migrate(t.Context(), root, Options{LegacyPrefix: "files"}) + require.NoError(t, err) + assert.Equal(t, 1, report.Renamed) + assert.Equal(t, "canonical", readFile(t, filepath.Join(root, key))) + assert.Equal(t, "legacy", readFile(t, filepath.Join(root, "index_files/8/0/2/3/milvus_packed_inverted_index.v3"))) +} + +func TestMigrateUnavailableCurrentDirectoryFailsBeforeMigration(t *testing.T) { + root, cwd := t.TempDir(), t.TempDir() + t.Chdir(cwd) + source := filepath.Join(displaced(root), "insert_log/1/2/3/data") + writeFile(t, source, "old-data") + require.NoError(t, os.Remove(cwd)) + _, err := Migrate(t.Context(), root, Options{LegacyPrefix: "files"}) + require.Error(t, err) + assert.NoFileExists(t, filepath.Join(root, "insert_log")) + assert.Equal(t, "old-data", readFile(t, source)) +} diff --git a/internal/storage/localmigrate/merge_copy_test.go b/internal/storage/localmigrate/merge_copy_test.go new file mode 100644 index 0000000000..6bca590377 --- /dev/null +++ b/internal/storage/localmigrate/merge_copy_test.go @@ -0,0 +1,339 @@ +// Licensed to the LF AI & Data foundation under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 + +package localmigrate + +import ( + "context" + "io" + "io/fs" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/bytedance/mockey" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "golang.org/x/sys/unix" + + "github.com/milvus-io/milvus/pkg/v3/util/merr" +) + +func TestMigrateMergeSplitJSONShreddingSegment(t *testing.T) { + t.Chdir(t.TempDir()) + root := t.TempDir() + segment := "insert_log/100/10/2001" + canonical := map[string]string{ + segment + "/_stats/json_stats.103/meta.json": "json-metadata", + segment + "/_stats/json_stats.103/shared_key_index/keys": "shared-keys", + } + legacy := map[string]string{ + segment + "/_data/data.parquet": "segment-data", + segment + "/_metadata/manifest-3.avro": "manifest", + segment + "/_stats/text_index.102/packed": "text-index", + segment + "/_stats/json_stats.103/shredding_data/0.parquet": "shredded-json", + } + for key, content := range canonical { + writeFile(t, filepath.Join(root, key), content) + } + for key, content := range legacy { + writeFile(t, filepath.Join(displaced(root), key), content) + } + _, err := Migrate(t.Context(), root, Options{LegacyPrefix: "files"}) + require.NoError(t, err) + for key, content := range canonical { + assert.Equal(t, content, readFile(t, filepath.Join(root, key))) + } + for key, content := range legacy { + assert.Equal(t, content, readFile(t, filepath.Join(root, key))) + assert.NoFileExists(t, filepath.Join(displaced(root), key)) + } + assert.NoDirExists(t, filepath.Join(displaced(root), "insert_log")) + assertMigrationRestartIsIdle(t, root) +} + +func TestMigrateMergePreservesCanonicalV1Segment(t *testing.T) { + t.Chdir(t.TempDir()) + root := t.TempDir() + v1 := "insert_log/100/10/2000/101/1234" + v3 := "insert_log/100/10/2001/_data/data.parquet" + writeFile(t, filepath.Join(root, v1), "v1-binlog") + writeFile(t, filepath.Join(displaced(root), v3), "v3-data") + _, err := Migrate(t.Context(), root, Options{LegacyPrefix: "files"}) + require.NoError(t, err) + assert.Equal(t, "v1-binlog", readFile(t, filepath.Join(root, v1))) + assert.Equal(t, "v3-data", readFile(t, filepath.Join(root, v3))) + assert.NoDirExists(t, filepath.Join(displaced(root), "insert_log")) + assertMigrationRestartIsIdle(t, root) +} + +func TestMigrateMergeDisplacedAndCWDIndexes(t *testing.T) { + root, cwd := t.TempDir(), t.TempDir() + t.Chdir(cwd) + displacedKey := "index_files/8/0/2/3/milvus_packed_inverted_index.v3" + cwdKey := "index_files/9/0/2/3/milvus_packed_inverted_index.v3" + writeFile(t, filepath.Join(displaced(root), displacedKey), "displaced-index") + writeFile(t, filepath.Join(cwd, cwdKey), "cwd-index") + _, err := Migrate(t.Context(), root, Options{LegacyPrefix: "files"}) + require.NoError(t, err) + assert.Equal(t, "displaced-index", readFile(t, filepath.Join(root, displacedKey))) + assert.Equal(t, "cwd-index", readFile(t, filepath.Join(root, cwdKey))) + assert.NoDirExists(t, filepath.Join(displaced(root), "index_files")) + assert.NoFileExists(t, filepath.Join(cwd, cwdKey)) + assertMigrationRestartIsIdle(t, root) +} + +func TestMigrateCopyFallbackCrossDevice(t *testing.T) { + for _, sourceKind := range []string{"displaced", "cwd"} { + t.Run(sourceKind, func(t *testing.T) { + root, cwd := t.TempDir(), t.TempDir() + t.Chdir(cwd) + key := "index_files/8/0/2/3/milvus_packed_inverted_index.v3" + sourceRoot, sourceDirectory := displaced(root), "index_files" + if sourceKind == "cwd" { + sourceRoot, sourceDirectory = cwd, filepath.Dir(key) + } + sourceFile := filepath.Join(sourceRoot, key) + writeFile(t, sourceFile, "complete-index") + injected := injectSourceRenameError(t, filepath.Join(sourceRoot, filepath.Dir(sourceDirectory)), unix.EXDEV) + _, err := Migrate(t.Context(), root, Options{LegacyPrefix: "files"}) + require.NoError(t, err) + require.Positive(t, *injected, "must exercise the cross-device fallback") + assert.Equal(t, "complete-index", readFile(t, filepath.Join(root, key))) + assert.NoDirExists(t, filepath.Join(sourceRoot, sourceDirectory)) + assertMigrationRestartIsIdle(t, root) + }) + } +} + +func TestMigrateRenamePermissionErrorDoesNotCopy(t *testing.T) { + root, cwd := t.TempDir(), t.TempDir() + t.Chdir(cwd) + key := "index_files/8/0/2/3/milvus_packed_inverted_index.v3" + writeFile(t, filepath.Join(cwd, key), "source-index") + injected := injectSourceRenameError(t, filepath.Join(cwd, filepath.Dir(filepath.Dir(key))), unix.EACCES) + _, err := Migrate(t.Context(), root, Options{LegacyPrefix: "files"}) + require.ErrorIs(t, err, unix.EACCES) + require.Positive(t, *injected) + assert.Equal(t, "source-index", readFile(t, filepath.Join(cwd, key))) + assert.NoFileExists(t, filepath.Join(root, key)) +} + +func TestMigrateCopyFailureLeavesSourceAndRetries(t *testing.T) { + root, cwd := t.TempDir(), t.TempDir() + t.Chdir(cwd) + key := "index_files/8/0/2/3/milvus_packed_inverted_index.v3" + source, target := filepath.Join(cwd, key), filepath.Join(root, key) + writeFile(t, source, "complete-original-index") + injected := injectSourceRenameError(t, filepath.Join(cwd, filepath.Dir(filepath.Dir(key))), unix.EXDEV) + copyCalls := 0 + copyPatch := mockey.Mock(io.Copy).To(func(dst io.Writer, src io.Reader) (int64, error) { + copyCalls++ + assert.Equal(t, "complete-original-index", readFile(t, source)) + assert.NoFileExists(t, target, "an incomplete copy must not become a canonical file") + n, err := dst.Write([]byte("partial")) + require.NoError(t, err) + return int64(n), io.ErrShortWrite + }).Build() + t.Cleanup(func() { copyPatch.UnPatch() }) + _, err := Migrate(t.Context(), root, Options{LegacyPrefix: "files"}) + require.ErrorIs(t, err, io.ErrShortWrite) + require.Positive(t, *injected) + require.Equal(t, 1, copyCalls) + assert.Equal(t, "complete-original-index", readFile(t, source)) + assert.NoFileExists(t, target) + copyPatch.UnPatch() + _, err = Migrate(t.Context(), root, Options{LegacyPrefix: "files"}) + require.NoError(t, err) + assert.Equal(t, "complete-original-index", readFile(t, target)) + assert.NoFileExists(t, source) + assertMigrationRestartIsIdle(t, root) +} + +func TestCopyFilePublicationRenameFailureRetainsSourceAndRetries(t *testing.T) { + sourceDir, targetDir := t.TempDir(), t.TempDir() + key := "index_files/8/0/2/3/milvus_packed_inverted_index.v3" + writeFile(t, filepath.Join(sourceDir, key), "complete-index") + require.NoError(t, os.MkdirAll(filepath.Join(targetDir, filepath.Dir(key)), 0o755)) + + source, err := os.OpenRoot(sourceDir) + require.NoError(t, err) + defer source.Close() + target, err := os.OpenRoot(targetDir) + require.NoError(t, err) + defer target.Close() + + var originalRename func(int, string, int, string) error + renameCalls := 0 + renamePatch := mockey.Mock(unix.Renameat).To(func(oldfd int, oldname string, newfd int, newname string) error { + if strings.HasSuffix(oldname, ".tmp") && newname == filepath.Base(key) { + renameCalls++ + return unix.EIO + } + return originalRename(oldfd, oldname, newfd, newname) + }).Origin(&originalRename).Build() + defer renamePatch.UnPatch() + + err = copyFile(t.Context(), source, key, target, key, &Report{}) + require.ErrorIs(t, err, unix.EIO) + require.Equal(t, 1, renameCalls) + require.Equal(t, "complete-index", readFile(t, filepath.Join(sourceDir, key))) + require.NoFileExists(t, filepath.Join(targetDir, key)) + artifacts, err := filepath.Glob(filepath.Join(targetDir, filepath.Dir(key), ".milvus-local-copy-*")) + require.NoError(t, err) + require.Empty(t, artifacts) + + renamePatch.UnPatch() + report := &Report{} + require.NoError(t, copyFile(t.Context(), source, key, target, key, report)) + require.Equal(t, 1, report.Copied) + require.Equal(t, "complete-index", readFile(t, filepath.Join(targetDir, key))) + require.NoFileExists(t, filepath.Join(sourceDir, key)) +} + +func TestCopyFileCanceledBeforeReadRetainsSource(t *testing.T) { + sourceDir, targetDir := t.TempDir(), t.TempDir() + key := "index_files/8/0/2/3/milvus_packed_inverted_index.v3" + writeFile(t, filepath.Join(sourceDir, key), "complete-index") + require.NoError(t, os.MkdirAll(filepath.Join(targetDir, filepath.Dir(key)), 0o755)) + + source, err := os.OpenRoot(sourceDir) + require.NoError(t, err) + defer source.Close() + target, err := os.OpenRoot(targetDir) + require.NoError(t, err) + defer target.Close() + + ctx, cancel := context.WithCancel(t.Context()) + cancel() + err = copyFile(ctx, source, key, target, key, &Report{}) + require.ErrorIs(t, err, context.Canceled) + require.Equal(t, "complete-index", readFile(t, filepath.Join(sourceDir, key))) + require.NoFileExists(t, filepath.Join(targetDir, key)) + artifacts, err := filepath.Glob(filepath.Join(targetDir, filepath.Dir(key), ".milvus-local-copy-*")) + require.NoError(t, err) + require.Empty(t, artifacts) +} + +func TestContextReaderStopsAfterCancellation(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + reader := &contextReader{ctx: ctx, reader: strings.NewReader("payload")} + buffer := make([]byte, 1) + + n, err := reader.Read(buffer) + require.NoError(t, err) + require.Equal(t, 1, n) + cancel() + + n, err = reader.Read(buffer) + require.ErrorIs(t, err, context.Canceled) + require.Zero(t, n) +} + +func TestMigrateCopyPublishedBeforeSourceRemovalRetries(t *testing.T) { + root, cwd := t.TempDir(), t.TempDir() + t.Chdir(cwd) + key := "index_files/8/0/2/3/milvus_packed_inverted_index.v3" + source, target := filepath.Join(cwd, key), filepath.Join(root, key) + writeFile(t, source, "complete-index") + injectSourceRenameError(t, filepath.Join(cwd, filepath.Dir(filepath.Dir(key))), unix.EXDEV) + var originalRemove func(*os.Root, string) error + removeCalls := 0 + removePatch := mockey.Mock((*os.Root).Remove).To(func(openedRoot *os.Root, name string) error { + if filepath.Join(openedRoot.Name(), name) == source { + removeCalls++ + assert.Equal(t, "complete-index", readFile(t, target), "publish a complete target before removing its source") + return unix.EACCES + } + return originalRemove(openedRoot, name) + }).Origin(&originalRemove).Build() + t.Cleanup(func() { removePatch.UnPatch() }) + _, err := Migrate(t.Context(), root, Options{LegacyPrefix: "files"}) + require.ErrorIs(t, err, unix.EACCES) + require.Equal(t, 1, removeCalls) + assert.Equal(t, "complete-index", readFile(t, source)) + assert.Equal(t, "complete-index", readFile(t, target)) + removePatch.UnPatch() + _, err = Migrate(t.Context(), root, Options{LegacyPrefix: "files"}) + require.NoError(t, err, "an already-published copy must not become a same-name conflict on retry") + assert.Equal(t, "complete-index", readFile(t, target)) + assert.NoFileExists(t, source) + assertMigrationRestartIsIdle(t, root) +} + +func TestMigrateMergeRejectsSameNameConflicts(t *testing.T) { + for _, conflictKind := range []string{"different-files", "source-file-target-directory", "source-directory-target-file"} { + t.Run(conflictKind, func(t *testing.T) { + t.Chdir(t.TempDir()) + root := t.TempDir() + key := "insert_log/100/10/2001/_data/conflict" + source, target := filepath.Join(displaced(root), key), filepath.Join(root, key) + sourceFile, targetFile := source, target + if conflictKind == "source-file-target-directory" { + targetFile = filepath.Join(target, "data") + } + if conflictKind == "source-directory-target-file" { + sourceFile = filepath.Join(source, "data") + } + writeFile(t, sourceFile, "legacy") + writeFile(t, targetFile, "canonical") + report, err := Migrate(t.Context(), root, Options{LegacyPrefix: "files"}) + require.ErrorIs(t, err, merr.ErrDataIntegrity) + assert.NotEmpty(t, report.Conflicts) + assert.Equal(t, "legacy", readFile(t, sourceFile)) + assert.Equal(t, "canonical", readFile(t, targetFile)) + }) + } +} + +func assertMigrationRestartIsIdle(t *testing.T, root string) { + t.Helper() + report, err := Migrate(t.Context(), root, Options{LegacyPrefix: "files"}) + require.NoError(t, err) + assert.Zero(t, report.Renamed) + assert.Zero(t, report.Bytes) +} + +// Only the old source filesystem rejects rename. Temporary-file publication +// inside the target filesystem must still execute the real system call. +func injectSourceRenameError(t *testing.T, sourceParent string, injectedErr error) *int { + t.Helper() + var parents []unix.Stat_t + require.NoError(t, filepath.WalkDir(sourceParent, func(name string, entry fs.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if entry.IsDir() { + var stat unix.Stat_t + if err := unix.Stat(name, &stat); err != nil { + return err + } + parents = append(parents, stat) + } + return nil + })) + var original func(int, string, int, string) error + injected := 0 + patch := mockey.Mock(unix.Renameat).To(func(oldfd int, oldname string, newfd int, newname string) error { + var actual unix.Stat_t + if err := unix.Fstat(oldfd, &actual); err == nil { + for _, expected := range parents { + if actual.Dev == expected.Dev && actual.Ino == expected.Ino { + injected++ + return injectedErr + } + } + } + return original(oldfd, oldname, newfd, newname) + }).Origin(&original).Build() + t.Cleanup(func() { patch.UnPatch() }) + return &injected +} diff --git a/internal/storage/localmigrate/migrate.go b/internal/storage/localmigrate/migrate.go new file mode 100644 index 0000000000..9e6dd0269d --- /dev/null +++ b/internal/storage/localmigrate/migrate.go @@ -0,0 +1,556 @@ +// Licensed to the LF AI & Data foundation under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package localmigrate repairs the double-root layout written by Milvus 3.0.0 +// and 3.0.1 and late 2.6 unified indexes written in the working directory. +// Migration merges directories without overwriting files, using rename where +// possible and copying across filesystems. No backup or migration journal is created. +// +// This package is unix-only: it needs a cross-root renameat and a +// directory-only unlinkat, neither of which os.Root offers (os.Root.Rename +// cannot cross roots, and os.Root.Remove would unlink a replacement file). +package localmigrate + +import ( + "context" + "fmt" + "io/fs" + "os" + "path/filepath" + "sort" + "strings" + "time" + + "github.com/gofrs/flock" + + "github.com/milvus-io/milvus/pkg/v3/util/merr" +) + +// LayoutDirs are layouts written through the formerly double-rooted Arrow +// filesystem. ChunkManager layouts such as delta_log and stats_log are not moved. +var LayoutDirs = []string{"insert_log", "text_log", "json_stats", "index_files", "index_v1"} + +const lockFileName = ".milvus-local-layout.lock" + +// reservedNamespace is a directory name this package keeps for itself under the +// storage root. It is distinct from lockFileName and is never a move source or +// a cleanup target, so a user directory that happens to use the name survives. +const reservedNamespace = ".milvus-local-layout" + +// maxReportedConflicts bounds the conflict list embedded in the returned error. +// Report.Conflicts keeps them all; an operator-facing message must not grow to +// the size of the storage root. +const maxReportedConflicts = 20 + +// Options controls automatic migration before the instance starts. +type Options struct { + // LegacyPrefix is minio.rootPath. Its insert_log namespace is still read in + // place by legacy manifests and must never be moved. + LegacyPrefix string + // OnPlan reports every directory this run decided to move, once, after + // discovery and conflict checking and before anything has moved. OnDirStart + // and OnDirDone then report each of those directories as it is moved, so a + // migration that takes minutes is visible while it runs instead of only in + // hindsight. All three are optional. + OnPlan func(dirs []Dir) + OnDirStart func(dir Dir) + OnDirDone func(dir Dir, elapsed time.Duration) +} + +// Dir describes one legacy directory the migration moves. Source and Target are +// absolute paths; Files and Bytes count the regular files below Source. +type Dir struct { + Source string + Target string + Files int + Bytes int64 +} + +// Report describes legacy directories moved in this run. +type Report struct { + Root string + Candidates []string + Conflicts []string + Renamed int + Copied int + Bytes int64 + Elapsed time.Duration +} + +func (r *Report) String() string { + return fmt.Sprintf("root=%s candidates=%v renamed=%d copied=%d bytes=%d conflicts=%d elapsed=%s", + r.Root, r.Candidates, r.Renamed, r.Copied, r.Bytes, len(r.Conflicts), r.Elapsed) +} + +// DisplacedRoots describes the one double-joined root; "/" has no displacement. +func DisplacedRoots(absRoot string) []string { + absRoot = filepath.Clean(absRoot) + if !filepath.IsAbs(absRoot) || absRoot == string(filepath.Separator) { + return nil + } + return []string{filepath.Join(absRoot, strings.TrimLeft(absRoot, string(filepath.Separator)))} +} + +type migrationSource struct { + kind string + root *os.Root + base string + physical string + cleanupDirs []string // Validated legacy directories in WalkDir order. +} + +type migrationDir struct { + source *migrationSource + path string + target string + files int + bytes int64 + dirs []string // Inspected directories, reused if only an empty skeleton remains. +} + +// describe renders the absolute source and target of d for reporting. +func (d migrationDir) describe(root *os.Root) Dir { + return Dir{ + Source: filepath.Join(d.source.root.Name(), d.path), + Target: filepath.Join(root.Name(), d.target), + Files: d.files, + Bytes: d.bytes, + } +} + +// Migrate merges old local layouts into the canonical storage root. Missing +// subtrees are renamed directly; cross-filesystem files are copied before their +// sources are removed. Existing directories are merged, never overwritten. +// A later startup resumes from the remaining sources without a migration journal. +// +// Canceling ctx stops the migration between filesystem reads and directories, which is +// safe precisely because it is resumable: the sources that have not moved yet +// are rediscovered on the next run. A partially copied temporary file is never +// published and is recopied from the intact source on the next run. +func Migrate(ctx context.Context, absRoot string, opts Options) (*Report, error) { + if !filepath.IsAbs(absRoot) { + return nil, merr.WrapErrParameterInvalidMsg("local storage root %q must be absolute", absRoot) + } + absRoot = filepath.Clean(absRoot) + startedAt := time.Now() + report := &Report{Root: absRoot} + defer func() { report.Elapsed = time.Since(startedAt) }() + if info, err := os.Stat(absRoot); err == nil && !info.IsDir() { + return report, merr.WrapErrParameterInvalidMsg("local storage root %q is not a directory", absRoot) + } + root, err := os.OpenRoot(absRoot) + if err != nil { + if os.IsNotExist(err) { + return report, nil + } + return report, merr.Wrapf(err, "open local storage root %s", absRoot) + } + defer root.Close() + cwd, err := openLegacyWorkingDirectory() + if err != nil { + return report, err + } + defer cwd.Close() + physicalRoot, err := filepath.EvalSymlinks(absRoot) + if err != nil { + return report, err + } + activeNamespace, err := legacyNamespace(physicalRoot, opts.LegacyPrefix) + if err != nil { + return report, err + } + if err := rejectSymlinks(root, lockFileName); err != nil { + return report, err + } + lock := flock.New(filepath.Join(root.Name(), lockFileName)) + locked, err := lock.TryLock() + if err != nil { + return report, merr.Wrap(err, "lock local layout migration") + } + if !locked { + return report, merr.WrapErrServiceUnavailableMsg("another local layout migration holds %s", lock.Path()) + } + defer lock.Unlock() + sources, err := migrationSources(root, cwd, physicalRoot, absRoot) + if err != nil { + return report, err + } + dirs, err := discoverDirectories(ctx, sources, physicalRoot, activeNamespace, report) + if err != nil { + return report, err + } + // CWD leaves are discovered through a map, so fix a stable order before + // anything is reported or moved: the plan, the move order and the log then + // agree across runs and are comparable after an interruption. + sort.SliceStable(dirs, func(i, j int) bool { + if dirs[i].source.root.Name() != dirs[j].source.root.Name() { + return dirs[i].source.root.Name() < dirs[j].source.root.Name() + } + return dirs[i].path < dirs[j].path + }) + if err := checkTargets(ctx, root, dirs, report); err != nil { + return report, err + } + if len(report.Conflicts) != 0 { + sort.Strings(report.Conflicts) + return report, merr.WrapErrDataIntegrityMsg("local layout files conflict (%d total): %v", + len(report.Conflicts), report.Conflicts[:min(len(report.Conflicts), maxReportedConflicts)]) + } + plan := make([]Dir, 0, len(dirs)) + for _, directory := range dirs { + plan = append(plan, directory.describe(root)) + } + if opts.OnPlan != nil { + opts.OnPlan(plan) + } + for index, directory := range dirs { + if err := ctx.Err(); err != nil { + return report, err + } + if opts.OnDirStart != nil { + opts.OnDirStart(plan[index]) + } + directoryStartedAt := time.Now() + if err := moveEntry(ctx, directory.source.root, directory.path, root, directory.target, report); err != nil { + return report, err + } + report.Bytes += directory.bytes + if opts.OnDirDone != nil { + opts.OnDirDone(plan[index], time.Since(directoryStartedAt)) + } + } + for _, source := range sources { + if err := removeEmptyLegacyDirectories(ctx, source); err != nil { + return report, err + } + } + return report, nil +} + +func openLegacyWorkingDirectory() (*os.Root, error) { + directory, err := os.Getwd() + if err != nil { + return nil, merr.Wrap(err, "resolve current working directory for local layout migration") + } + root, err := os.OpenRoot(filepath.Clean(directory)) + if err != nil { + return nil, merr.Wrapf(err, "open legacy working directory %s", directory) + } + return root, nil +} + +func migrationSources(root, cwd *os.Root, physicalRoot, configuredRoot string) ([]*migrationSource, error) { + var sources []*migrationSource + // R=/ has no displaced root, but may still have indexes under an old CWD. + if candidates := DisplacedRoots(configuredRoot); len(candidates) != 0 { + base, err := filepath.Rel(root.Name(), candidates[0]) + if err != nil { + return nil, err + } + sources = append(sources, &migrationSource{kind: "displaced", root: root, base: base, physical: physicalRoot}) + } + rootInfo, err := root.Stat(".") + if err != nil { + return nil, err + } + cwdInfo, err := cwd.Stat(".") + if err != nil { + return nil, err + } + if !os.SameFile(rootInfo, cwdInfo) { + // The displaced source already includes its text_log and index_files. + // Do not discover them twice when CWD names the same physical directory. + for _, source := range sources { + if info, err := source.root.Stat(source.base); err == nil && os.SameFile(info, cwdInfo) { + return sources, nil + } + // Leave missing or unsafe source validation to discovery below. + } + physicalCWD, err := filepath.EvalSymlinks(cwd.Name()) + if err != nil { + return nil, err + } + sources = append(sources, &migrationSource{kind: "cwd", root: cwd, base: ".", physical: physicalCWD}) + } + return sources, nil +} + +func legacyNamespace(root, prefix string) (string, error) { + prefix = filepath.Clean(prefix) + if !filepath.IsLocal(prefix) { + return "", merr.WrapErrParameterInvalidMsg("unsafe legacy minio.rootPath %q", prefix) + } + return filepath.Join(root, prefix, "insert_log"), nil +} + +func discoverDirectories(ctx context.Context, sources []*migrationSource, root, active string, report *Report) ([]migrationDir, error) { + var directories []migrationDir + for _, source := range sources { + if source.kind == "displaced" { + start := len(directories) + for _, layout := range LayoutDirs { + path := filepath.Join(source.base, layout) + if err := addDirectory(ctx, source, root, active, path, layout, &directories); err != nil { + return nil, err + } + } + if len(directories) != start { + report.Candidates = append(report.Candidates, filepath.Join(source.root.Name(), source.base)) + } + continue + } + // CWD indexes are moved at their deepest eligible directory. This keeps + // unrelated files in the old working directory untouched. + groups := make(map[string]*migrationDir) + for _, layout := range []string{"text_log", "index_files"} { + if err := collectLegacyIndexDirs(ctx, source, root, active, layout, groups); err != nil { + return nil, err + } + } + for _, directory := range groups { + directories = append(directories, *directory) + } + if len(groups) != 0 { + report.Candidates = append(report.Candidates, source.root.Name()) + } + } + return directories, nil +} + +func addDirectory(ctx context.Context, source *migrationSource, root, active, path, target string, directories *[]migrationDir) error { + if err := rejectSymlinks(source.root, path); err != nil { + return err + } + info, err := source.root.Lstat(path) + if os.IsNotExist(err) { + return nil + } + if err != nil { + return err + } + if !info.IsDir() { + return merr.WrapErrDataIntegrityMsg("legacy layout %s is not a directory", filepath.Join(source.root.Name(), path)) + } + directory := migrationDir{source: source, path: path, target: target} + if err := inspectDirectory(ctx, source.root, path, &directory); err != nil { + return err + } + if directory.files == 0 { + // An interrupted merge may have moved the last file but left its empty + // parents. Reuse discovery for cleanup after all migration work succeeds. + // Empty live/reserved namespaces must still be left untouched. + if protectNamespace(source, path, active, root) == nil { + source.cleanupDirs = append(source.cleanupDirs, directory.dirs...) + } + return nil + } + if err := protectNamespace(source, path, active, root); err != nil { + return err + } + *directories = append(*directories, directory) + return nil +} + +func collectLegacyIndexDirs(ctx context.Context, source *migrationSource, root, active, layout string, groups map[string]*migrationDir) error { + if err := rejectSymlinks(source.root, layout); err != nil { + return err + } + info, err := source.root.Lstat(layout) + if os.IsNotExist(err) { + return nil + } + if err != nil { + return err + } + if !info.IsDir() { + return merr.WrapErrDataIntegrityMsg("legacy index layout %s is not a directory", filepath.Join(source.root.Name(), layout)) + } + return fs.WalkDir(source.root.FS(), layout, func(name string, entry fs.DirEntry, walkErr error) error { + if err := ctx.Err(); err != nil { + return err + } + if walkErr != nil { + return walkErr + } + if entry.Type()&fs.ModeSymlink != 0 { + return merr.WrapErrDataIntegrityMsg("legacy source %s is a symlink; resolve manually", filepath.Join(source.root.Name(), name)) + } + if entry.IsDir() { + // The CWD is already resolved and WalkDir never follows symlinks. + physical := filepath.Join(source.physical, name) + if entersCanonicalRoot(source.physical, root, physical) { + return fs.SkipDir + } + // Include empty prefixes left by an interrupted migration, even when + // no packed file remains to identify a leaf. Never collect live paths + // or unrelated directories for cleanup. + if legacyIndexPath(name, true) && !pathsOverlap(physical, active) && + !pathsOverlap(physical, filepath.Join(root, reservedNamespace)) { + source.cleanupDirs = append(source.cleanupDirs, name) + } + return nil + } + if !legacyIndexPath(name, false) { + return nil + } + parent := filepath.Dir(name) + if directory := groups[parent]; directory != nil { + info, err := entry.Info() + if err != nil { + return err + } + directory.files++ + directory.bytes += info.Size() + return nil + } + directory := &migrationDir{source: source, path: parent, target: parent} + if err := protectNamespace(source, parent, active, root); err != nil { + return err + } + groups[parent] = directory + info, err := entry.Info() + if err != nil { + return err + } + if !info.Mode().IsRegular() { + return merr.WrapErrDataIntegrityMsg("legacy source %s is not a regular file", filepath.Join(source.root.Name(), name)) + } + directory.files = 1 + directory.bytes = info.Size() + return nil + }) +} + +func inspectDirectory(ctx context.Context, root *os.Root, path string, directory *migrationDir) error { + return fs.WalkDir(root.FS(), path, func(name string, entry fs.DirEntry, walkErr error) error { + if err := ctx.Err(); err != nil { + return err + } + if walkErr != nil { + return walkErr + } + if entry.Type()&fs.ModeSymlink != 0 { + return merr.WrapErrDataIntegrityMsg("legacy source %s is a symlink; resolve manually", filepath.Join(root.Name(), name)) + } + if entry.IsDir() { + if directory.files == 0 { + directory.dirs = append(directory.dirs, name) + } + return nil + } + info, err := entry.Info() + if err != nil { + return err + } + if !info.Mode().IsRegular() { + return merr.WrapErrDataIntegrityMsg("legacy source %s is not a regular file", filepath.Join(root.Name(), name)) + } + // Nonempty trees are moved normally; retain directory names only while + // this could still be an empty skeleton requiring cleanup. + directory.dirs = nil + directory.files++ + directory.bytes += info.Size() + return nil + }) +} + +func protectNamespace(source *migrationSource, path, active, root string) error { + original := filepath.Join(source.physical, path) + if source.kind == "displaced" { + original = filepath.Join(source.physical, source.base, filepath.Base(path)) + } + if pathsOverlap(original, active) { + return merr.WrapErrDataIntegrityMsg("legacy source %s overlaps the live manifest namespace %s; refusing to move", original, active) + } + if pathsOverlap(original, filepath.Join(root, reservedNamespace)) { + return merr.WrapErrDataIntegrityMsg("legacy source %s overlaps the reserved migration namespace", original) + } + return nil +} + +func withinDirectory(directory, name string) bool { + rel, err := filepath.Rel(directory, name) + return err == nil && filepath.IsLocal(rel) +} + +func pathsOverlap(a, b string) bool { + return withinDirectory(a, b) || withinDirectory(b, a) +} + +func entersCanonicalRoot(cwd, root, name string) bool { + return withinDirectory(cwd, root) && withinDirectory(root, name) +} + +// legacyIndexPath admits only the directory prefixes of eligible unified index files. +func legacyIndexPath(name string, directory bool) bool { + parts := strings.Split(filepath.ToSlash(name), "/") + ids := 4 + if parts[0] == "text_log" { + ids = 6 + } else if parts[0] != "index_files" { + return false + } + for i := 1; i < len(parts) && i <= ids; i++ { + if !isLegacyIndexID(parts[i]) { + return false + } + } + if directory { + return len(parts) <= ids+1 + } + return len(parts) == ids+2 && isLegacyUnifiedIndexFile(parts[len(parts)-1]) +} + +func isLegacyIndexID(value string) bool { + if value == "" { + return false + } + for _, ch := range value { + if ch < '0' || ch > '9' { + return false + } + } + return true +} + +func isLegacyUnifiedIndexFile(name string) bool { + // A discovery marker, also moved last so interrupted CWD leaves stay recognizable. + const prefix, suffix = "milvus_packed_", "_index.v3" + return len(name) > len(prefix)+len(suffix) && strings.HasPrefix(name, prefix) && strings.HasSuffix(name, suffix) +} + +// os.Root confines access but permits in-root symlinks. Migration must not +// follow those aliases into a different live directory, including at mutation. +func rejectSymlinks(root *os.Root, name string) error { + if !filepath.IsLocal(name) { + return merr.WrapErrDataIntegrityMsg("migration path %q is outside its opened root", name) + } + current := "" + for _, component := range strings.Split(filepath.Clean(name), string(filepath.Separator)) { + current = filepath.Join(current, component) + info, err := root.Lstat(current) + if os.IsNotExist(err) { + return nil + } + if err != nil { + return err + } + if info.Mode()&fs.ModeSymlink != 0 { + return merr.WrapErrDataIntegrityMsg("migration path %s is a symlink; resolve manually", filepath.Join(root.Name(), current)) + } + } + return nil +} diff --git a/internal/storage/localmigrate/migrate_test.go b/internal/storage/localmigrate/migrate_test.go new file mode 100644 index 0000000000..75ab3d043d --- /dev/null +++ b/internal/storage/localmigrate/migrate_test.go @@ -0,0 +1,189 @@ +// Licensed to the LF AI & Data foundation under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 + +package localmigrate + +import ( + "fmt" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/bytedance/mockey" + "github.com/gofrs/flock" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/milvus-io/milvus/pkg/v3/util/merr" +) + +func writeFile(t *testing.T, p, content string) { + t.Helper() + require.NoError(t, os.MkdirAll(filepath.Dir(p), 0o755)) + require.NoError(t, os.WriteFile(p, []byte(content), 0o600)) +} + +func readFile(t *testing.T, p string) string { + t.Helper() + b, err := os.ReadFile(p) + require.NoError(t, err) + return string(b) +} + +func displaced(root string) string { + return filepath.Join(root, strings.TrimPrefix(root, string(filepath.Separator))) +} + +func TestDisplacedRoots(t *testing.T) { + assert.Equal(t, []string{"/var/lib/milvus/data/var/lib/milvus/data"}, DisplacedRoots("/var/lib/milvus/data/")) + assert.Equal(t, []string{"/r/r"}, DisplacedRoots("/r")) + assert.Empty(t, DisplacedRoots("data")) + assert.Empty(t, DisplacedRoots("")) + assert.Empty(t, DisplacedRoots("/")) +} + +func TestMigrateRenamesDisplacedLayouts(t *testing.T) { + t.Chdir(t.TempDir()) + root := t.TempDir() + for _, layout := range []string{"insert_log", "text_log", "json_stats", "index_files", "index_v1"} { + writeFile(t, filepath.Join(displaced(root), layout, "1", "data"), layout) + } + writeFile(t, filepath.Join(displaced(root), "delta_log", "1", "data"), "untouched") + report, err := Migrate(t.Context(), root, Options{LegacyPrefix: "files"}) + require.NoError(t, err) + assert.Equal(t, 5, report.Renamed) + assert.EqualValues(t, 10+8+10+11+8, report.Bytes) + for _, layout := range []string{"insert_log", "text_log", "json_stats", "index_files", "index_v1"} { + assert.Equal(t, layout, readFile(t, filepath.Join(root, layout, "1", "data"))) + assert.NoDirExists(t, filepath.Join(displaced(root), layout)) + } + assert.Equal(t, "untouched", readFile(t, filepath.Join(displaced(root), "delta_log", "1", "data"))) + second, err := Migrate(t.Context(), root, Options{LegacyPrefix: "files"}) + require.NoError(t, err) + assert.Zero(t, second.Renamed) +} + +func TestMigrateMergesEmptyTargetDirectory(t *testing.T) { + t.Chdir(t.TempDir()) + root := t.TempDir() + key := "insert_log/1/2/3/data" + source := filepath.Join(displaced(root), key) + target := filepath.Join(root, "insert_log") + writeFile(t, source, "source") + require.NoError(t, os.MkdirAll(target, 0o755)) + report, err := Migrate(t.Context(), root, Options{LegacyPrefix: "files"}) + require.NoError(t, err) + assert.Empty(t, report.Conflicts) + assert.Equal(t, "source", readFile(t, filepath.Join(root, key))) + assert.NoFileExists(t, source) +} + +func TestMigrateLeavesLegacyMinioNamespaceUntouched(t *testing.T) { + for _, prefix := range []string{"files", "./files", "files//", "files/tmp/.."} { + t.Run(prefix, func(t *testing.T) { + testMigrateLeavesLegacyMinioNamespaceUntouched(t, prefix) + }) + } +} + +func testMigrateLeavesLegacyMinioNamespaceUntouched(t *testing.T, prefix string) { + t.Chdir(t.TempDir()) + root := t.TempDir() + key := "insert_log/1/2/5/_manifest/1.avro" + writeFile(t, filepath.Join(root, "files", key), "relative-manifest-data") + source := filepath.Join(displaced(root), key) + writeFile(t, source, "complete-key-data") + report, err := Migrate(t.Context(), root, Options{LegacyPrefix: prefix}) + require.NoError(t, err) + assert.Equal(t, 1, report.Renamed) + assert.Equal(t, "complete-key-data", readFile(t, filepath.Join(root, key))) + assert.Equal(t, "relative-manifest-data", readFile(t, filepath.Join(root, "files", key))) +} + +func TestMigrateValidatesRootAndLegacyPrefix(t *testing.T) { + t.Chdir(t.TempDir()) + for _, root := range []string{"", "relative/root"} { + _, err := Migrate(t.Context(), root, Options{}) + require.ErrorIs(t, err, merr.ErrParameterInvalid) + } + file := filepath.Join(t.TempDir(), "file") + writeFile(t, file, "data") + _, err := Migrate(t.Context(), file, Options{}) + require.ErrorIs(t, err, merr.ErrParameterInvalid) + for _, prefix := range []string{"../outside", "/absolute"} { + _, err := Migrate(t.Context(), t.TempDir(), Options{LegacyPrefix: prefix}) + require.Error(t, err) + } +} + +func TestMigrateFilesystemRootStillMigratesCWD(t *testing.T) { + for _, withIndex := range []bool{false, true} { + t.Run(fmt.Sprintf("index=%t", withIndex), func(t *testing.T) { + cwd, sandbox := t.TempDir(), t.TempDir() + t.Chdir(cwd) + key := "index_files/7/0/2/3/milvus_packed_inverted_index.v3" + if withIndex { + writeFile(t, filepath.Join(cwd, key), "old-index") + } + // Exercise the configured filesystem root without writing to the + // machine's actual root directory. + var original func(string) (*os.Root, error) + mock := mockey.Mock(os.OpenRoot).Origin(&original).To(func(name string) (*os.Root, error) { + if name == "/" { + return original(sandbox) + } + return original(name) + }).Build() + defer mock.UnPatch() + report, err := Migrate(t.Context(), "/", Options{LegacyPrefix: "files"}) + require.NoError(t, err) + if withIndex { + assert.Equal(t, "old-index", readFile(t, filepath.Join(sandbox, key))) + assert.NoFileExists(t, filepath.Join(cwd, key)) + } else { + assert.Zero(t, report.Renamed) + } + }) + } +} + +func TestMigrateLockContention(t *testing.T) { + t.Chdir(t.TempDir()) + root := t.TempDir() + writeFile(t, filepath.Join(displaced(root), "insert_log/1/data"), "data") + lock := flock.New(filepath.Join(root, lockFileName)) + require.NoError(t, lock.Lock()) + defer lock.Unlock() + _, err := Migrate(t.Context(), root, Options{LegacyPrefix: "files"}) + require.ErrorIs(t, err, merr.ErrServiceUnavailable) +} + +func TestMigrateRejectsSymlinks(t *testing.T) { + t.Chdir(t.TempDir()) + root, outside := t.TempDir(), t.TempDir() + writeFile(t, filepath.Join(outside, "sentinel"), "outside") + require.NoError(t, os.MkdirAll(filepath.Dir(displaced(root)), 0o755)) + require.NoError(t, os.Symlink(outside, displaced(root))) + _, err := Migrate(t.Context(), root, Options{LegacyPrefix: "files"}) + require.Error(t, err) + assert.Equal(t, "outside", readFile(t, filepath.Join(outside, "sentinel"))) +} + +func TestMigrateProtectsLiveManifestNamespace(t *testing.T) { + t.Chdir(t.TempDir()) + root := t.TempDir() + prefix := strings.TrimPrefix(root, string(filepath.Separator)) + source := filepath.Join(displaced(root), "insert_log/1/2/3/data") + writeFile(t, source, "live") + _, err := Migrate(t.Context(), root, Options{LegacyPrefix: prefix}) + require.ErrorIs(t, err, merr.ErrDataIntegrity) + assert.Equal(t, "live", readFile(t, source)) +} diff --git a/internal/storage/localmigrate/move.go b/internal/storage/localmigrate/move.go new file mode 100644 index 0000000000..ae6bd96288 --- /dev/null +++ b/internal/storage/localmigrate/move.go @@ -0,0 +1,412 @@ +// Licensed to the LF AI & Data foundation under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 + +package localmigrate + +import ( + "bytes" + "context" + "crypto/sha256" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "sort" + + "github.com/cockroachdb/errors" + "golang.org/x/sys/unix" + + "github.com/milvus-io/milvus/pkg/v3/util/merr" +) + +// Check actual entries, not overlapping directory prefixes: legacy JSON stats +// can split even a single segment's _stats directory between the two roots. +func checkTargets(ctx context.Context, root *os.Root, dirs []migrationDir, report *Report) error { + planned := make(map[string]bool) + for _, dir := range dirs { + err := fs.WalkDir(dir.source.root.FS(), dir.path, func(name string, entry fs.DirEntry, err error) error { + if ctxErr := ctx.Err(); ctxErr != nil { + return ctxErr + } + if err != nil { + return err + } + if !entry.IsDir() && !entry.Type().IsRegular() { + return merr.WrapErrDataIntegrityMsg("legacy source %s is not a regular file", name) + } + rel, err := filepath.Rel(dir.path, name) + if err != nil { + return err + } + target := filepath.Join(dir.target, rel) + isDir, seen := planned[target] + conflict := seen && (!isDir || !entry.IsDir()) + planned[target] = entry.IsDir() + info, err := root.Lstat(target) + switch { + case err == nil: + if !entry.IsDir() || !info.IsDir() { + published, err := publishedCopy(dir.source.root, name, root, target) + if err != nil { + return err + } + conflict = conflict || !published + } + case errors.Is(err, unix.ENOTDIR): + conflict = true + case !os.IsNotExist(err): + return err + } + if conflict { + report.Conflicts = append(report.Conflicts, filepath.Join(root.Name(), target)) + if entry.IsDir() { + return fs.SkipDir + } + } + return nil + }) + if err != nil { + return err + } + } + return nil +} + +func moveEntry(ctx context.Context, source *os.Root, name string, root *os.Root, target string, report *Report) error { + if err := ctx.Err(); err != nil { + return err + } + if err := rejectSymlinks(source, name); err != nil { + return err + } + if err := rejectSymlinks(root, target); err != nil { + return err + } + info, err := source.Lstat(name) + if err != nil { + return err + } + dest, err := root.Lstat(target) + if os.IsNotExist(err) { + if err := root.MkdirAll(filepath.Dir(target), 0o755); err != nil { + return err + } + if err := renameEntry(source, name, root, target); err == nil { + report.Renamed++ + return nil + } else if !errors.Is(err, unix.EXDEV) { + return merr.Wrapf(err, "rename local layout %s to %s", name, target) + } + if !info.IsDir() { + return copyFile(ctx, source, name, root, target, report) + } + if err := root.Mkdir(target, info.Mode().Perm()); err != nil { + return err + } + } else if err != nil { + return err + } else if !info.IsDir() || !dest.IsDir() { + published, err := publishedCopy(source, name, root, target) + if err != nil { + return err + } + if published { + return finishCopy(source, name, root, target, report) + } + return merr.WrapErrDataIntegrityMsg("local layout file already exists: %s", filepath.Join(root.Name(), target)) + } + entries, err := fs.ReadDir(source.FS(), name) + if err != nil { + return err + } + // Keep a CWD leaf recognizable after an interruption while moving auxiliary + // files: its packed index files are the last entries removed from the source. + sort.SliceStable(entries, func(i, j int) bool { + return !isLegacyUnifiedIndexFile(entries[i].Name()) && isLegacyUnifiedIndexFile(entries[j].Name()) + }) + for _, entry := range entries { + if err := moveEntry(ctx, source, filepath.Join(name, entry.Name()), root, filepath.Join(target, entry.Name()), report); err != nil { + return err + } + } + return source.Remove(name) // Only the now-empty source directory is removed. +} + +// removeEmptyLegacyDirectories prunes children before parents using the existing +// discovery order. Rmdir leaves nonempty directories and replacement files alone; +// it never removes the working directory itself or recursively deletes content. +func removeEmptyLegacyDirectories(ctx context.Context, source *migrationSource) error { + for i := len(source.cleanupDirs) - 1; i >= 0; i-- { + if err := ctx.Err(); err != nil { + return err + } + name := source.cleanupDirs[i] + if err := rejectSymlinks(source.root, name); err != nil { + if errors.Is(err, unix.ENOTDIR) { + continue + } + return err + } + parent, err := source.root.Open(filepath.Dir(name)) + if os.IsNotExist(err) || errors.Is(err, unix.ENOTDIR) { + continue + } + if err != nil { + return merr.Wrapf(err, "open legacy parent for cleanup %s", name) + } + err = unix.Unlinkat(int(parent.Fd()), filepath.Base(name), unix.AT_REMOVEDIR) + closeErr := parent.Close() + if err != nil && !os.IsNotExist(err) && !errors.Is(err, unix.ENOTEMPTY) && + !errors.Is(err, unix.EEXIST) && !errors.Is(err, unix.ENOTDIR) { + return merr.Wrapf(err, "remove empty legacy directory %s", name) + } + if closeErr != nil { + return merr.Wrapf(closeErr, "close legacy parent for cleanup %s", name) + } + } + return nil +} + +func renameEntry(source *os.Root, name string, root *os.Root, target string) error { + from, err := source.Open(filepath.Dir(name)) + if err != nil { + return err + } + defer from.Close() + to, err := root.Open(filepath.Dir(target)) + if err != nil { + return err + } + defer to.Close() + return unix.Renameat(int(from.Fd()), filepath.Base(name), int(to.Fd()), filepath.Base(target)) +} + +// The hash identifies one source-to-target publication, not its contents. +// The data temporary file is renamed into place; a small sidecar marker remains +// until the source removal is durable so a restart can recognize that target as +// this migration's own completed copy. +func copyArtifactBase(source *os.Root, name, target string) string { + sum := sha256.Sum256([]byte(source.Name() + "\x00" + name + "\x00" + target)) + return filepath.Join(filepath.Dir(target), fmt.Sprintf(".milvus-local-copy-%x", sum)) +} + +func copyTempPath(source *os.Root, name, target string) string { + return copyArtifactBase(source, name, target) + ".tmp" +} + +func copyMarkerPath(source *os.Root, name, target string) string { + return copyArtifactBase(source, name, target) + ".published" +} + +func copyMarkerPayload(source *os.Root, name, target string, size int64) []byte { + return []byte(fmt.Sprintf("milvus-local-copy-v1\n%s\n%d\n", copyArtifactBase(source, name, target), size)) +} + +func publishedCopy(source *os.Root, name string, root *os.Root, target string) (bool, error) { + markerPath := copyMarkerPath(source, name, target) + marker, err := root.Lstat(markerPath) + if os.IsNotExist(err) { + return false, nil + } + if err != nil { + return false, err + } + if !marker.Mode().IsRegular() { + return false, merr.WrapErrDataIntegrityMsg("invalid local migration publication marker for %s", target) + } + sourceInfo, err := source.Lstat(name) + if err != nil { + return false, err + } + expectedMarker := copyMarkerPayload(source, name, target, sourceInfo.Size()) + actualMarker, err := root.ReadFile(markerPath) + if err != nil { + return false, err + } + if !bytes.Equal(actualMarker, expectedMarker) { + return false, merr.WrapErrDataIntegrityMsg("invalid local migration publication marker for %s", target) + } + dest, err := root.Lstat(target) + if os.IsNotExist(err) { + return false, nil + } + return err == nil && dest.Mode().IsRegular() && dest.Size() == sourceInfo.Size(), err +} + +func copyFile(ctx context.Context, source *os.Root, name string, root *os.Root, target string, report *Report) error { + if err := ctx.Err(); err != nil { + return err + } + in, err := source.Open(name) + if err != nil { + return err + } + defer in.Close() + info, err := in.Stat() + if err != nil { + return err + } + if !info.Mode().IsRegular() { + return merr.WrapErrDataIntegrityMsg("legacy source %s is not a regular file", name) + } + temp := copyTempPath(source, name, target) + marker := copyMarkerPath(source, name, target) + for _, artifact := range []string{temp, marker} { + if err := rejectSymlinks(root, artifact); err != nil { + return err + } + if previous, err := root.Lstat(artifact); err == nil { + if !previous.Mode().IsRegular() { + return merr.WrapErrDataIntegrityMsg("invalid local migration copy artifact for %s", target) + } + } else if !os.IsNotExist(err) { + return err + } + // A previous interrupted copy was never published. Recopy from the + // intact source; never expose a partially written file under the final + // name. + if err := root.Remove(artifact); err != nil && !os.IsNotExist(err) { + return err + } + } + out, err := root.OpenFile(temp, os.O_CREATE|os.O_EXCL|os.O_WRONLY, info.Mode().Perm()) + if err != nil { + return err + } + published := false + defer func() { + out.Close() + if !published { + _ = root.Remove(temp) + _ = root.Remove(marker) + } + }() + if _, err := io.Copy(out, &contextReader{ctx: ctx, reader: in}); err != nil { + return merr.Wrapf(err, "copy local layout %s to %s", name, target) + } + if err := ctx.Err(); err != nil { + return err + } + if err := out.Sync(); err != nil { + return err + } + if err := out.Close(); err != nil { + return err + } + markerFile, err := root.OpenFile(marker, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600) + if err != nil { + return err + } + markerOpen := true + defer func() { + if markerOpen { + markerFile.Close() + } + }() + if _, err := markerFile.Write(copyMarkerPayload(source, name, target, info.Size())); err != nil { + return err + } + if err := markerFile.Sync(); err != nil { + return err + } + if err := markerFile.Close(); err != nil { + return err + } + markerOpen = false + // Persist the proof and complete temporary file before publishing it. If + // the process stops after the rename, the next startup can finish source + // removal instead of treating its own target as an unrelated conflict. + if err := syncDirectory(root, filepath.Dir(target)); err != nil { + return err + } + if err := renameEntry(root, temp, root, target); err != nil { + return err + } + published = true + if err := ctx.Err(); err != nil { + return err + } + return finishCopy(source, name, root, target, report) +} + +type contextReader struct { + ctx context.Context + reader io.Reader +} + +func (r *contextReader) Read(buffer []byte) (int, error) { + if err := r.ctx.Err(); err != nil { + return 0, err + } + return r.reader.Read(buffer) +} + +func syncDirectory(root *os.Root, directory string) error { + opened, err := root.Open(directory) + if err != nil { + return err + } + err = opened.Sync() + closeErr := opened.Close() + if err != nil { + return err + } + return closeErr +} + +// Persist the target entry and every ancestor link that MkdirAll may have +// created. Syncing only the leaf directory does not make new parents durable. +// Repeat this on recovery too: an existing directory may be from an interrupted +// migration that never finished syncing its parents. +func syncCopyTargetDirectories(root *os.Root, directory string) error { + for { + parent, err := root.Open(directory) + if err != nil { + return err + } + err = parent.Sync() + closeErr := parent.Close() + if err != nil { + return err + } + if closeErr != nil { + return closeErr + } + if directory == "." { + return nil + } + directory = filepath.Dir(directory) + } +} + +func finishCopy(source *os.Root, name string, root *os.Root, target string, report *Report) error { + if err := syncCopyTargetDirectories(root, filepath.Dir(target)); err != nil { + return err + } + sourceParent, err := source.Open(filepath.Dir(name)) + if err != nil { + return err + } + defer sourceParent.Close() + if err := source.Remove(name); err != nil { + return err + } + // Persist source removal before dropping the publication marker. Otherwise + // a power loss could resurrect the source without proof that target is ours. + if err := sourceParent.Sync(); err != nil { + return err + } + // Best-effort: the copy is already published and the source is gone, so the + // migration is complete. The sidecar is only retry provenance, not data. + _ = root.Remove(copyMarkerPath(source, name, target)) + report.Copied++ + return nil +} diff --git a/internal/storage/localmigrate/progress_test.go b/internal/storage/localmigrate/progress_test.go new file mode 100644 index 0000000000..edfa40ed2c --- /dev/null +++ b/internal/storage/localmigrate/progress_test.go @@ -0,0 +1,140 @@ +// Licensed to the LF AI & Data foundation under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package localmigrate + +import ( + "context" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// The plan is what a slow or interrupted migration has to be reconciled against, +// so every directory must be named before anything moves, and again as it moves +// and once it is done. +func TestMigrateReportsPlanAndEveryDirectory(t *testing.T) { + t.Chdir(t.TempDir()) + root := t.TempDir() + layouts := []string{"index_files", "insert_log", "json_stats"} + for _, layout := range layouts { + writeFile(t, filepath.Join(displaced(root), layout, "1", "data"), layout) + } + + var plan, starts, dones []Dir + var elapsed []time.Duration + report, err := Migrate(t.Context(), root, Options{ + LegacyPrefix: "files", + OnPlan: func(dirs []Dir) { plan = append(plan, dirs...) }, + OnDirStart: func(dir Dir) { starts = append(starts, dir) }, + OnDirDone: func(dir Dir, took time.Duration) { + dones = append(dones, dir) + elapsed = append(elapsed, took) + }, + }) + require.NoError(t, err) + + // Discovery walks a map for CWD leaves, so the order is fixed explicitly. + sources := make([]string, 0, len(plan)) + targets := make([]string, 0, len(plan)) + for _, dir := range plan { + sources = append(sources, dir.Source) + targets = append(targets, dir.Target) + assert.Equal(t, 1, dir.Files) + assert.Positive(t, dir.Bytes) + } + wantSources := make([]string, 0, len(layouts)) + wantTargets := make([]string, 0, len(layouts)) + for _, layout := range layouts { + wantSources = append(wantSources, filepath.Join(displaced(root), layout)) + wantTargets = append(wantTargets, filepath.Join(root, layout)) + } + assert.Equal(t, wantSources, sources) + assert.Equal(t, wantTargets, targets) + + // Every planned directory is reported starting and finishing, in plan order. + assert.Equal(t, plan, starts) + assert.Equal(t, plan, dones) + assert.Len(t, elapsed, len(layouts)) + assert.Positive(t, report.Elapsed) +} + +func TestMigrateWithoutLegacyDirectoriesReportsAnEmptyPlan(t *testing.T) { + t.Chdir(t.TempDir()) + root := t.TempDir() + + planned := 0 + moved := 0 + report, err := Migrate(t.Context(), root, Options{ + LegacyPrefix: "files", + OnPlan: func(dirs []Dir) { planned = len(dirs) }, + OnDirStart: func(Dir) { moved++ }, + }) + + require.NoError(t, err) + assert.Zero(t, planned) + assert.Zero(t, moved) + assert.Zero(t, report.Renamed+report.Copied) +} + +// An expired deadline must surface as a failure rather than a partially applied +// layout that looks finished. +func TestMigrateExpiredDeadlineFailsBeforeMoving(t *testing.T) { + t.Chdir(t.TempDir()) + root := t.TempDir() + key := filepath.Join("insert_log", "1", "data") + writeFile(t, filepath.Join(displaced(root), key), "payload") + + ctx, cancel := context.WithDeadline(t.Context(), time.Now().Add(-time.Second)) + defer cancel() + _, err := Migrate(ctx, root, Options{LegacyPrefix: "files"}) + + require.ErrorIs(t, err, context.DeadlineExceeded) + // The source is untouched, so a later run still has everything to move. + assert.Equal(t, "payload", readFile(t, filepath.Join(displaced(root), key))) +} + +// Cancellation is safe because the migration is resumable: whatever has not +// moved is rediscovered, so a restart with more time finishes the job. +func TestMigrateCancelledMidwayResumesOnRestart(t *testing.T) { + t.Chdir(t.TempDir()) + root := t.TempDir() + layouts := []string{"index_files", "insert_log", "json_stats"} + for _, layout := range layouts { + writeFile(t, filepath.Join(displaced(root), layout, "1", "data"), layout) + } + + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + first, err := Migrate(ctx, root, Options{ + LegacyPrefix: "files", + // Stop after the first directory is fully in place. + OnDirDone: func(Dir, time.Duration) { cancel() }, + }) + require.ErrorIs(t, err, context.Canceled) + require.Equal(t, 1, first.Renamed+first.Copied) + assert.Equal(t, "index_files", readFile(t, filepath.Join(root, "index_files", "1", "data"))) + + second, err := Migrate(t.Context(), root, Options{LegacyPrefix: "files"}) + require.NoError(t, err) + assert.Equal(t, len(layouts)-1, second.Renamed+second.Copied) + for _, layout := range layouts { + assert.Equal(t, layout, readFile(t, filepath.Join(root, layout, "1", "data"))) + } +} diff --git a/internal/storage/record_reader.go b/internal/storage/record_reader.go index 93752bdb95..5c09cc28d8 100644 --- a/internal/storage/record_reader.go +++ b/internal/storage/record_reader.go @@ -221,7 +221,7 @@ func newIterativePackedRecordReader( type ManifestReader struct { fieldBinlogs []*datapb.FieldBinlog manifest string - reader *packed.FFIPackedReader + reader manifestArrowReader bufferSize int64 arrowSchema *arrow.Schema @@ -231,10 +231,16 @@ type ManifestReader struct { storageConfig *indexpb.StorageConfig storagePluginContext *indexcgopb.StoragePluginContext externalSpecContext packed.ExternalSpecContext + textColumnConfigs []packed.TextColumnConfig neededColumns []string } +type manifestArrowReader interface { + ReadNext() (arrow.Record, error) + Close() error +} + // NewManifestReaderFromBinlogs creates a ManifestReader from binlogs func NewManifestReaderFromBinlogs(fieldBinlogs []*datapb.FieldBinlog, schema *schemapb.CollectionSchema, @@ -309,6 +315,31 @@ func NewManifestReaderWithExtfs( storageConfig *indexpb.StorageConfig, storagePluginContext *indexcgopb.StoragePluginContext, extfs packed.ExternalSpecContext, +) (*ManifestReader, error) { + return newManifestReader(manifest, schema, bufferSize, storageConfig, storagePluginContext, extfs, nil) +} + +// NewTextDecodedManifestReader opens a manifest through milvus-storage's +// SegmentReader so TEXT reference columns are resolved through source-specific +// LOB paths and returned as logical UTF8 strings. +func NewTextDecodedManifestReader( + manifest string, + schema *schemapb.CollectionSchema, + bufferSize int64, + storageConfig *indexpb.StorageConfig, + textColumnConfigs []packed.TextColumnConfig, +) (*ManifestReader, error) { + return newManifestReader(manifest, schema, bufferSize, storageConfig, nil, packed.ExternalSpecContext{}, textColumnConfigs) +} + +func newManifestReader( + manifest string, + schema *schemapb.CollectionSchema, + bufferSize int64, + storageConfig *indexpb.StorageConfig, + storagePluginContext *indexcgopb.StoragePluginContext, + extfs packed.ExternalSpecContext, + textColumnConfigs []packed.TextColumnConfig, ) (*ManifestReader, error) { columnResolver := typeutil.NewStorageColumnResolver(schema, typeutil.WithStorageColumnExternalSpec(extfs.Spec)) arrowSchema, err := ConvertToArrowSchemaWithNameResolver( @@ -328,7 +359,7 @@ func NewManifestReaderWithExtfs( // RecordToInsertData conversion does not accidentally decode internal LOB // references as user text. Any source type coercion must stay in the // external-source normalization path, not in the internal manifest path. - if !typeutil.IsExternalCollection(schema) || columnResolver.IsMilvusTable() { + if len(textColumnConfigs) == 0 && (!typeutil.IsExternalCollection(schema) || columnResolver.IsMilvusTable()) { arrowSchema = overrideTextFieldsToBinaryByFields( columnResolver.ManifestStoredFields(), arrowSchema, @@ -360,6 +391,7 @@ func NewManifestReaderWithExtfs( storageConfig: storageConfig, storagePluginContext: storagePluginContext, externalSpecContext: extfs, + textColumnConfigs: textColumnConfigs, neededColumns: neededColumns, } @@ -373,6 +405,15 @@ func NewManifestReaderWithExtfs( } func (mr *ManifestReader) init() error { + if len(mr.textColumnConfigs) > 0 { + reader, err := packed.NewFFISegmentReader(mr.manifest, mr.arrowSchema, mr.neededColumns, + mr.bufferSize, mr.storageConfig, mr.textColumnConfigs) + if err != nil { + return err + } + mr.reader = reader + return nil + } reader, err := packed.NewFFIPackedReader(mr.manifest, mr.arrowSchema, mr.neededColumns, mr.bufferSize, mr.storageConfig, diff --git a/internal/storage/remote_chunk_manager.go b/internal/storage/remote_chunk_manager.go index 156f432c23..113133e44e 100644 --- a/internal/storage/remote_chunk_manager.go +++ b/internal/storage/remote_chunk_manager.go @@ -321,6 +321,14 @@ func (mcm *RemoteChunkManager) MultiRemove(ctx context.Context, keys []string) e // RemoveWithPrefix removes all objects with the same prefix @prefix from minio. func (mcm *RemoteChunkManager) RemoveWithPrefix(ctx context.Context, prefix string) error { + // An empty prefix matches every object in the bucket, so this API would + // erase the whole instance. LocalChunkManager refuses it for the same + // reason; the guard belongs here rather than in each caller. + if len(prefix) == 0 { + errMsg := "empty prefix is not allowed for ChunkManager remove operation" + mlog.Warn(ctx, errMsg) + return merr.WrapErrStorageMsg("%s", errMsg) + } // removeObject in parallel. runningGroup, _ := errgroup.WithContext(ctx) runningGroup.SetLimit(10) diff --git a/internal/storage/rw.go b/internal/storage/rw.go index 0f8fcc4639..7cba024356 100644 --- a/internal/storage/rw.go +++ b/internal/storage/rw.go @@ -443,6 +443,48 @@ func NewManifestRecordReader(ctx context.Context, manifestPath string, neededSch return NewAbsentFieldFillRecordReader(inner, neededSchema, present), nil } +// NewTextDecodedManifestRecordReader resolves persisted TEXT references through +// source-specific LOB paths and returns logical UTF8 columns. It is intended for +// compaction that must rewrite LOB payloads into a different partition namespace. +func NewTextDecodedManifestRecordReader( + _ context.Context, + manifestPath string, + neededSchema *schemapb.CollectionSchema, + textColumnConfigs []packed.TextColumnConfig, + option ...RwOption, +) (RecordReader, error) { + rwOptions := DefaultReaderOptions() + for _, opt := range option { + opt(rwOptions) + } + if err := rwOptions.validate(); err != nil { + return nil, err + } + present := rwOptions.presentFields + if present == nil { + var err error + present, err = packed.GetManifestFieldIDs(manifestPath, rwOptions.storageConfig) + if err != nil { + return nil, err + } + } + presentSchema, err := filterSchemaToPresentFields(neededSchema, present) + if err != nil { + return nil, err + } + inner, err := NewTextDecodedManifestReader( + manifestPath, + presentSchema, + rwOptions.bufferSize, + rwOptions.storageConfig, + textColumnConfigs, + ) + if err != nil { + return nil, err + } + return NewAbsentFieldFillRecordReader(inner, neededSchema, present), nil +} + // filterSchemaToPresentFields returns a copy of schema keeping only the fields // (and struct sub-fields) whose FieldID is physically present, so the packed // reader is asked to read only stored columns; absent fields are then filled by diff --git a/internal/storage/rw_test.go b/internal/storage/rw_test.go index ebdf055f69..f03d79f626 100644 --- a/internal/storage/rw_test.go +++ b/internal/storage/rw_test.go @@ -41,8 +41,10 @@ import ( "github.com/milvus-io/milvus/internal/storagecommon" "github.com/milvus-io/milvus/internal/storagev2/packed" "github.com/milvus-io/milvus/pkg/v3/common" + "github.com/milvus-io/milvus/pkg/v3/objectstorage" "github.com/milvus-io/milvus/pkg/v3/proto/datapb" "github.com/milvus-io/milvus/pkg/v3/proto/indexpb" + "github.com/milvus-io/milvus/pkg/v3/proto/querypb" "github.com/milvus-io/milvus/pkg/v3/util/merr" "github.com/milvus-io/milvus/pkg/v3/util/metautil" "github.com/milvus-io/milvus/pkg/v3/util/paramtable" @@ -812,6 +814,20 @@ func (s *PackedBinlogRecordSuite) TestV3StatsWrittenUnderBasePath() { assert.NotContains(s.T(), p, "stats_log", "bloom filter stat path must not use legacy stats_log/ layout") } + + // QueryNode resolves V3 bloom-filter paths from SegmentLoadInfo and passes + // them unchanged to ChunkManager.MultiRead. Exercise that exact boundary so + // a regression to bucket-relative local paths cannot be hidden by the test + // joining localStorage.path itself. + loadInfo := &querypb.SegmentLoadInfo{ManifestPath: manifestPath} + resolvedPaths, err := packed.NewStatsResolverFromLoadInfo(loadInfo).BloomFilterPaths(pkField.GetFieldID()) + require.NoError(s.T(), err) + require.Equal(s.T(), bfStat.Paths, resolvedPaths) + + chunkManager := NewLocalChunkManager(objectstorage.RootPath(dir)) + values, err := chunkManager.MultiRead(s.ctx, resolvedPaths) + require.NoError(s.T(), err) + require.Len(s.T(), values, len(resolvedPaths)) } func genRowWithBM25(magic int64) map[int64]interface{} { diff --git a/internal/storage/types.go b/internal/storage/types.go index 20214b9ae9..e563070f6c 100644 --- a/internal/storage/types.go +++ b/internal/storage/types.go @@ -51,7 +51,20 @@ type ChunkObjectInfo struct { // ChunkManager is to manager chunks. // Include Read, Write, Remove chunks. type ChunkManager interface { - // RootPath returns current root path. + // Paths passed to ChunkManager are complete keys in the backend namespace, + // and implementations never prepend RootPath themselves: + // - remote keys are bucket-relative object keys that already include the + // storage prefix minio.rootPath (for example, files/insert_log/...); + // they are not URIs and do not start with a slash + // - local keys are complete filesystem paths that already include the + // storage prefix localStorage.path (for example, + // /var/lib/milvus/data/insert_log/...) + // Storage V3 manifests and the loon filesystem use exactly the same keys + // (the loon filesystem is rooted at the bucket, or at "/" for local; see + // storagev2.LoonFSRootPath), so a path never needs conversion between the + // manifest, the ChunkManager and the disk. WalkWithPrefix returns FilePath + // values in the same namespace. + // RootPath returns the storage prefix (minio.rootPath or localStorage.path). RootPath() string // Path returns path of @filePath. Path(ctx context.Context, filePath string) (string, error) diff --git a/internal/storagev2/filesystem_metrics.go b/internal/storagev2/filesystem_metrics.go index 76e4e69daf..ac9ca47d22 100644 --- a/internal/storagev2/filesystem_metrics.go +++ b/internal/storagev2/filesystem_metrics.go @@ -151,7 +151,9 @@ func makePropertiesFromConfig(storageConfig *indexpb.StorageConfig) (C.LoonPrope keys = append(keys, propAccessKeyValue) values = append(values, v) } - if v := storageConfig.GetRootPath(); v != "" { + // Same root rule as packed.MakePropertiesFromStorageConfig, otherwise the + // metrics lookup addresses a filesystem that was never created. + if v := LoonFSRootPath(storageConfig); v != "" { keys = append(keys, propRootPath) values = append(values, v) } diff --git a/internal/storagev2/filesystem_metrics_test.go b/internal/storagev2/filesystem_metrics_test.go index 8e3bcdb088..77cb18d2de 100644 --- a/internal/storagev2/filesystem_metrics_test.go +++ b/internal/storagev2/filesystem_metrics_test.go @@ -60,46 +60,29 @@ func TestGetFilesystemMetricsWithConfig(t *testing.T) { } } +// Every local storage config maps to the single loon filesystem rooted at "/" +// (see LoonFSRootPath), so two local configs with different root paths share +// one metrics entry whose display key names that root. func TestListFilesystemMetrics(t *testing.T) { - metricsBefore, err := ListFilesystemMetrics() - require.NoError(t, err) - existingDisplayKeys := make(map[string]struct{}, len(metricsBefore)) - for _, fsMetrics := range metricsBefore { - existingDisplayKeys[fsMetrics.DisplayKey] = struct{}{} - } - - dirA := t.TempDir() - dirB := t.TempDir() - expectedDisplayPrefixes := map[string]struct{}{ - "file://" + dirA + "#fs:": {}, - "file://" + dirB + "#fs:": {}, - } - for _, dir := range []string{dirA, dirB} { - _, err := GetFilesystemMetricsWithConfig(&indexpb.StorageConfig{ + for _, dir := range []string{t.TempDir(), t.TempDir()} { + fsMetrics, err := GetFilesystemMetricsWithConfig(&indexpb.StorageConfig{ StorageType: "local", RootPath: dir, }) require.NoError(t, err) + require.NotNil(t, fsMetrics) } metricsList, err := ListFilesystemMetrics() require.NoError(t, err) - newFilesystemCount := 0 + localPrefix := "file://" + LoonLocalFSRootPath + "#fs:" + localEntries := 0 for _, fsMetrics := range metricsList { - if _, ok := existingDisplayKeys[fsMetrics.DisplayKey]; ok { + if !strings.HasPrefix(fsMetrics.DisplayKey, localPrefix) { continue } - newFilesystemCount++ - matched := false - for prefix := range expectedDisplayPrefixes { - if strings.HasPrefix(fsMetrics.DisplayKey, prefix) { - delete(expectedDisplayPrefixes, prefix) - matched = true - break - } - } - assert.True(t, matched, fsMetrics.DisplayKey) + localEntries++ assert.GreaterOrEqual(t, fsMetrics.ReadCount, int64(0)) assert.GreaterOrEqual(t, fsMetrics.WriteCount, int64(0)) assert.GreaterOrEqual(t, fsMetrics.ReadBytes, int64(0)) @@ -109,8 +92,7 @@ func TestListFilesystemMetrics(t *testing.T) { assert.GreaterOrEqual(t, fsMetrics.MultiPartUploadCreated, int64(0)) assert.GreaterOrEqual(t, fsMetrics.MultiPartUploadFinished, int64(0)) } - require.Equal(t, 2, newFilesystemCount) - require.Empty(t, expectedDisplayPrefixes) + require.Equal(t, 1, localEntries, "local storage shares one loon filesystem rooted at /") } func TestNilConfig(t *testing.T) { diff --git a/internal/storagev2/loon_root.go b/internal/storagev2/loon_root.go new file mode 100644 index 0000000000..62f2e3dc23 --- /dev/null +++ b/internal/storagev2/loon_root.go @@ -0,0 +1,42 @@ +// Licensed to the LF AI & Data foundation under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storagev2 + +import "github.com/milvus-io/milvus/pkg/v3/proto/indexpb" + +// LoonLocalFSRootPath is the fs.root_path handed to loon for storageType=local. +// +// Milvus keys already carry the storage prefix: localStorage.path for local +// (/var/lib/milvus/data/insert_log/...) and minio.rootPath for remote +// (files/insert_log/...). The loon filesystem therefore has to be rooted at the +// namespace root, which is the bucket for remote (milvus-storage never applies +// root_path there) and "/" for local. Rooting the local SubTreeFileSystem at +// localStorage.path would join the prefix twice (milvus-storage #351, #53051) +// and make complete keys differ from their physical locations. Every Go site +// that turns a StorageConfig into loon properties must go through +// LoonFSRootPath; the C++ counterpart is +// LoonFSRootPath in storage/loon_ffi/util.h. +const LoonLocalFSRootPath = "/" + +// LoonFSRootPath returns the fs.root_path for storageConfig: "/" for local, +// the configured root path otherwise. +func LoonFSRootPath(storageConfig *indexpb.StorageConfig) string { + if storageConfig.GetStorageType() == "local" { + return LoonLocalFSRootPath + } + return storageConfig.GetRootPath() +} diff --git a/internal/storagev2/loon_root_test.go b/internal/storagev2/loon_root_test.go new file mode 100644 index 0000000000..02ed3b6b3f --- /dev/null +++ b/internal/storagev2/loon_root_test.go @@ -0,0 +1,38 @@ +// Licensed to the LF AI & Data foundation under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storagev2 + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/milvus-io/milvus/pkg/v3/proto/indexpb" +) + +// Milvus keys already carry the storage prefix, so the loon filesystem must be +// rooted at the namespace root: "/" for local, and the configured root path +// (which milvus-storage does not apply for object storage) for remote. +func TestLoonFSRootPath(t *testing.T) { + assert.Equal(t, "/", LoonFSRootPath(&indexpb.StorageConfig{StorageType: "local", RootPath: "/var/lib/milvus/data"})) + assert.Equal(t, "/", LoonFSRootPath(&indexpb.StorageConfig{StorageType: "local", RootPath: "data"})) + assert.Equal(t, "/", LoonFSRootPath(&indexpb.StorageConfig{StorageType: "local"})) + assert.Equal(t, "files", LoonFSRootPath(&indexpb.StorageConfig{StorageType: "remote", RootPath: "files"})) + assert.Equal(t, "files", LoonFSRootPath(&indexpb.StorageConfig{StorageType: "minio", RootPath: "files"})) + assert.Equal(t, "", LoonFSRootPath(&indexpb.StorageConfig{StorageType: "remote"})) + assert.Equal(t, "", LoonFSRootPath(nil)) +} diff --git a/internal/storagev2/packed/ffi_common.go b/internal/storagev2/packed/ffi_common.go index 8322bdbde1..a767a1d1e9 100644 --- a/internal/storagev2/packed/ffi_common.go +++ b/internal/storagev2/packed/ffi_common.go @@ -18,6 +18,7 @@ import ( "github.com/cockroachdb/errors" + "github.com/milvus-io/milvus/internal/storagev2" _ "github.com/milvus-io/milvus/internal/util/cgo" "github.com/milvus-io/milvus/pkg/v3/proto/indexpb" "github.com/milvus-io/milvus/pkg/v3/util/merr" @@ -82,10 +83,9 @@ func ExtfsPrefixForCollection(collectionID int64) string { return fmt.Sprintf("extfs.%d.", collectionID) } -// MakePropertiesFromStorageConfig creates a Properties object from StorageConfig -// This function converts a StorageConfig structure into a Properties object by -// calling the FFI properties_create function. All configuration fields from -// StorageConfig are mapped to corresponding key-value pairs in Properties. +// MakePropertiesFromStorageConfig creates a Properties object from StorageConfig. +// All configuration fields are mapped to corresponding property key-value pairs, +// with the local filesystem root normalized separately from the key prefix. func MakePropertiesFromStorageConfig(storageConfig *indexpb.StorageConfig, extraKVs map[string]string) (*C.LoonProperties, error) { if storageConfig == nil { return nil, merr.WrapErrStorageMsg("storageConfig is required") @@ -112,9 +112,9 @@ func MakePropertiesFromStorageConfig(storageConfig *indexpb.StorageConfig, extra keys = append(keys, PropertyFSAccessKeyValue) values = append(values, storageConfig.GetSecretAccessKey()) } - if storageConfig.GetRootPath() != "" { + if fsRoot := storagev2.LoonFSRootPath(storageConfig); fsRoot != "" { keys = append(keys, PropertyFSRootPath) - values = append(values, storageConfig.GetRootPath()) + values = append(values, fsRoot) } if storageConfig.GetStorageType() != "" { keys = append(keys, PropertyFSStorageType) diff --git a/internal/storagev2/packed/ffi_root_path_test.go b/internal/storagev2/packed/ffi_root_path_test.go new file mode 100644 index 0000000000..5405ca5662 --- /dev/null +++ b/internal/storagev2/packed/ffi_root_path_test.go @@ -0,0 +1,47 @@ +// Licensed to the LF AI & Data foundation under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package packed + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/milvus-io/milvus/pkg/v3/proto/indexpb" +) + +func TestMakePropertiesFromStorageConfig_RootPath(t *testing.T) { + for _, tc := range []struct { + storageType string + rootPath string + expected string + }{ + {storageType: "local", rootPath: "/var/lib/milvus/data", expected: "/"}, + {storageType: "remote", rootPath: "files", expected: "files"}, + } { + t.Run(tc.storageType, func(t *testing.T) { + props, err := MakePropertiesFromStorageConfig(&indexpb.StorageConfig{ + StorageType: tc.storageType, + RootPath: tc.rootPath, + }, nil) + require.NoError(t, err) + defer FreeProperties(props) + assert.Equal(t, tc.expected, loonPropertyString(props, PropertyFSRootPath)) + }) + } +} diff --git a/internal/storagev2/packed/manifest_commit_test.go b/internal/storagev2/packed/manifest_commit_test.go index b017def041..a29c9e38cb 100644 --- a/internal/storagev2/packed/manifest_commit_test.go +++ b/internal/storagev2/packed/manifest_commit_test.go @@ -46,7 +46,7 @@ func manifestTestStorageConfig(t *testing.T) *indexpb.StorageConfig { // loon transaction (and therefore without bumping the version). func TestCommitManifestUpdates_EmptyShortCircuit(t *testing.T) { cfg := manifestTestStorageConfig(t) - basePath := "files/commit_empty_test/seg1" + basePath := path.Join(cfg.RootPath, "commit_empty_test/seg1") cases := []struct { name string @@ -71,9 +71,9 @@ func TestCommitManifestUpdates_EmptyShortCircuit(t *testing.T) { // via a single CommitManifestUpdates call. func TestCommitManifestUpdates_StatsOnly(t *testing.T) { cfg := manifestTestStorageConfig(t) - basePath := "files/commit_stats_only/seg1" + basePath := path.Join(cfg.RootPath, "commit_stats_only/seg1") - statPath := path.Join(cfg.RootPath, basePath, "_stats/bloom_filter.100/1") + statPath := path.Join(basePath, "_stats/bloom_filter.100/1") require.NoError(t, WriteFile(cfg, statPath, []byte("bloom-blob"))) got, err := CommitManifestUpdates(basePath, ManifestEarliest, cfg, &ManifestUpdates{ @@ -98,11 +98,11 @@ func TestCommitManifestUpdates_StatsOnly(t *testing.T) { // BM25 memory_size while excluding text/JSON index stats. func TestStatsBinlogSizeFromManifest(t *testing.T) { cfg := manifestTestStorageConfig(t) - basePath := "files/stats_binlog_size/seg1" + basePath := path.Join(cfg.RootPath, "stats_binlog_size/seg1") - bloomPath := path.Join(cfg.RootPath, basePath, "_stats/bloom_filter.100/1") - bm25Path := path.Join(cfg.RootPath, basePath, "_stats/bm25.101/1") - textPath := path.Join(cfg.RootPath, basePath, "_stats/text.102/1") + bloomPath := path.Join(basePath, "_stats/bloom_filter.100/1") + bm25Path := path.Join(basePath, "_stats/bm25.101/1") + textPath := path.Join(basePath, "_stats/text.102/1") require.NoError(t, WriteFile(cfg, bloomPath, []byte("bloom-blob"))) require.NoError(t, WriteFile(cfg, bm25Path, []byte("bm25-blob"))) require.NoError(t, WriteFile(cfg, textPath, []byte("text-blob"))) @@ -132,9 +132,9 @@ func TestStatsBinlogSizeFromManifest_ReadError(t *testing.T) { // TestCommitManifestUpdates_DeltaOnly exercises the delta-only path. func TestCommitManifestUpdates_DeltaOnly(t *testing.T) { cfg := manifestTestStorageConfig(t) - basePath := "files/commit_delta_only/seg1" + basePath := path.Join(cfg.RootPath, "commit_delta_only/seg1") - deltaPath := path.Join(cfg.RootPath, basePath, "_delta/delta-1") + deltaPath := path.Join(basePath, "_delta/delta-1") require.NoError(t, WriteFile(cfg, deltaPath, []byte("delta-payload"))) got, err := CommitManifestUpdates(basePath, ManifestEarliest, cfg, &ManifestUpdates{ @@ -156,7 +156,7 @@ func TestCommitManifestUpdates_DeltaOnly(t *testing.T) { // bundle. func TestCommitManifestUpdates_AllSections(t *testing.T) { cfg := manifestTestStorageConfig(t) - basePath := "files/commit_all_sections/seg1" + basePath := path.Join(cfg.RootPath, "commit_all_sections/seg1") schema := arrow.NewSchema([]arrow.Field{ { @@ -186,9 +186,9 @@ func TestCommitManifestUpdates_AllSections(t *testing.T) { require.NoError(t, err) defer out.Destroy() - statPath := path.Join(cfg.RootPath, basePath, "_stats/bloom_filter.100/1") + statPath := path.Join(basePath, "_stats/bloom_filter.100/1") require.NoError(t, WriteFile(cfg, statPath, []byte("bloom-blob"))) - deltaPath := path.Join(cfg.RootPath, basePath, "_delta/delta-1") + deltaPath := path.Join(basePath, "_delta/delta-1") require.NoError(t, WriteFile(cfg, deltaPath, []byte("delta-payload"))) got, err := CommitManifestUpdates(basePath, ManifestEarliest, cfg, &ManifestUpdates{ @@ -212,7 +212,7 @@ func TestCommitManifestUpdates_AllSections(t *testing.T) { // add_column_group (function-backfill) branch in applyManifestUpdates. func TestCommitManifestUpdates_AddNewColumnGroups(t *testing.T) { cfg := manifestTestStorageConfig(t) - basePath := "files/commit_new_cgs/seg1" + basePath := path.Join(cfg.RootPath, "commit_new_cgs/seg1") schema := arrow.NewSchema([]arrow.Field{ { @@ -254,7 +254,7 @@ func TestCommitManifestUpdates_AddNewColumnGroups(t *testing.T) { func TestCommitManifestUpdates_AddEmptyColumnGroup(t *testing.T) { cfg := manifestTestStorageConfig(t) - basePath := "files/commit_empty_cg/seg1" + basePath := path.Join(cfg.RootPath, "commit_empty_cg/seg1") schema := arrow.NewSchema([]arrow.Field{{ Name: "101", @@ -288,7 +288,7 @@ func TestCommitManifestUpdates_AddEmptyColumnGroup(t *testing.T) { // ColumnGroups handle multiple times (and on a nil receiver) is safe. func TestColumnGroups_DestroyIdempotent(t *testing.T) { cfg := manifestTestStorageConfig(t) - basePath := "files/cg_destroy_test/seg1" + basePath := path.Join(cfg.RootPath, "cg_destroy_test/seg1") schema := arrow.NewSchema([]arrow.Field{ { @@ -330,7 +330,7 @@ func TestColumnGroups_DestroyIdempotent(t *testing.T) { // the close FFI on an exhausted handle. func TestFFIPackedWriter_DoubleCloseRejected(t *testing.T) { cfg := manifestTestStorageConfig(t) - basePath := "files/ffi_double_close/seg1" + basePath := path.Join(cfg.RootPath, "ffi_double_close/seg1") schema := arrow.NewSchema([]arrow.Field{ { diff --git a/internal/storagev2/packed/manifest_index_test.go b/internal/storagev2/packed/manifest_index_test.go index 317fd48a65..27abd313cb 100644 --- a/internal/storagev2/packed/manifest_index_test.go +++ b/internal/storagev2/packed/manifest_index_test.go @@ -52,8 +52,8 @@ func indexEntry(t *testing.T, basePath, indexPrefix string, indexID, buildID int // absolute legacy prefix, with every scalar field intact. func TestCommitManifestUpdates_IndexRoundTrip(t *testing.T) { cfg := manifestTestStorageConfig(t) - basePath := "files/commit_index_roundtrip/seg1" - indexPrefix := "files/index_v1/1/2/3/500/1" + basePath := path.Join(cfg.GetRootPath(), "commit_index_roundtrip/seg1") + indexPrefix := path.Join(cfg.GetRootPath(), "index_v1/1/2/3/500/1") committed, err := CommitManifestUpdates(basePath, ManifestEarliest, cfg, &ManifestUpdates{ Indexes: []ManifestIndexInfo{indexEntry(t, basePath, indexPrefix, 10, 500)}, @@ -90,17 +90,19 @@ func TestCommitManifestUpdates_IndexRoundTrip(t *testing.T) { // DataCoord publish a rebuild without an explicit drop. func TestCommitManifestUpdates_IndexRepublishReplaces(t *testing.T) { cfg := manifestTestStorageConfig(t) - basePath := "files/commit_index_republish/seg1" + basePath := path.Join(cfg.GetRootPath(), "commit_index_republish/seg1") + firstPrefix := path.Join(cfg.GetRootPath(), "index_v1/1/2/3/500/1") + secondPrefix := path.Join(cfg.GetRootPath(), "index_v1/1/2/3/501/2") first, err := CommitManifestUpdates(basePath, ManifestEarliest, cfg, &ManifestUpdates{ - Indexes: []ManifestIndexInfo{indexEntry(t, basePath, "files/index_v1/1/2/3/500/1", 10, 500)}, + Indexes: []ManifestIndexInfo{indexEntry(t, basePath, firstPrefix, 10, 500)}, }) require.NoError(t, err) _, version, err := UnmarshalManifestPath(first) require.NoError(t, err) second, err := CommitManifestUpdates(basePath, version, cfg, &ManifestUpdates{ - Indexes: []ManifestIndexInfo{indexEntry(t, basePath, "files/index_v1/1/2/3/501/2", 10, 501)}, + Indexes: []ManifestIndexInfo{indexEntry(t, basePath, secondPrefix, 10, 501)}, }) require.NoError(t, err) @@ -108,17 +110,17 @@ func TestCommitManifestUpdates_IndexRepublishReplaces(t *testing.T) { require.NoError(t, err) require.Len(t, indexes, 1) require.EqualValues(t, 501, indexes[0].BuildID) - require.Equal(t, "files/index_v1/1/2/3/501/2", indexes[0].Path) + require.Equal(t, secondPrefix, indexes[0].Path) } func TestCommitManifestUpdates_DropIndex(t *testing.T) { cfg := manifestTestStorageConfig(t) - basePath := "files/commit_index_drop/seg1" + basePath := path.Join(cfg.GetRootPath(), "commit_index_drop/seg1") published, err := CommitManifestUpdates(basePath, ManifestEarliest, cfg, &ManifestUpdates{ Indexes: []ManifestIndexInfo{ - indexEntry(t, basePath, "files/index_v1/1/2/3/500/1", 10, 500), - indexEntry(t, basePath, "files/index_v1/1/2/3/600/1", 11, 600), + indexEntry(t, basePath, path.Join(cfg.GetRootPath(), "index_v1/1/2/3/500/1"), 10, 500), + indexEntry(t, basePath, path.Join(cfg.GetRootPath(), "index_v1/1/2/3/600/1"), 11, 600), }, }) require.NoError(t, err) diff --git a/internal/storagev2/packed/packed_reader_ffi_test.go b/internal/storagev2/packed/packed_reader_ffi_test.go index 5cc56abd0f..0be193983b 100644 --- a/internal/storagev2/packed/packed_reader_ffi_test.go +++ b/internal/storagev2/packed/packed_reader_ffi_test.go @@ -4,6 +4,7 @@ import ( "fmt" "io" "math" + "path" "testing" "github.com/apache/arrow/go/v17/arrow" @@ -53,7 +54,7 @@ func TestPackedFFIReader(t *testing.T) { }, }, nil) - basePath := "files/packed_reader_test/1" + basePath := path.Join(dir, "packed_reader_test/1") version := int64(0) // Build record batch @@ -204,7 +205,7 @@ func TestPackedFFIReaderPartialColumns(t *testing.T) { }, }, nil) - basePath := "files/packed_reader_partial_test/1" + basePath := path.Join(dir, "packed_reader_partial_test/1") version := int64(0) // Build record batch @@ -342,7 +343,7 @@ func TestPackedFFIReaderMultipleBatches(t *testing.T) { }, }, nil) - basePath := "files/packed_reader_multi_batch_test/1" + basePath := path.Join(dir, "packed_reader_multi_batch_test/1") version := int64(0) columnGroups := []storagecommon.ColumnGroup{ diff --git a/internal/storagev2/packed/packed_writer_ffi_test.go b/internal/storagev2/packed/packed_writer_ffi_test.go index 72f17ad86b..35e03a5ac6 100644 --- a/internal/storagev2/packed/packed_writer_ffi_test.go +++ b/internal/storagev2/packed/packed_writer_ffi_test.go @@ -2,6 +2,7 @@ package packed import ( "math" + "path" "testing" "github.com/apache/arrow/go/v17/arrow" @@ -112,7 +113,7 @@ func TestGetManifestFieldIDs_InvalidColumnName(t *testing.T) { }, }, nil) columnGroups := []storagecommon.ColumnGroup{{Columns: []int{0}, GroupID: storagecommon.DefaultShortColumnGroupID}} - basePath := "files/packed_writer_invalid_column/1" + basePath := path.Join(CreateStorageConfig().GetRootPath(), "packed_writer_invalid_column/1") cfg := CreateStorageConfig() writer, err := NewFFIPackedWriter(basePath, schema, columnGroups, cfg, nil) require.NoError(t, err) @@ -159,7 +160,7 @@ func TestGetManifestFieldIDs_FromPackedWriterManifest(t *testing.T) { }, }, nil) columnGroups := []storagecommon.ColumnGroup{{Columns: []int{0, 1}, GroupID: storagecommon.DefaultShortColumnGroupID}} - basePath := "files/packed_writer_field_ids/1" + basePath := path.Join(CreateStorageConfig().GetRootPath(), "packed_writer_field_ids/1") cfg := CreateStorageConfig() writer, err := NewFFIPackedWriter(basePath, schema, columnGroups, cfg, nil) require.NoError(t, err) @@ -214,7 +215,7 @@ func TestResolveManifestSingleWriterFormat_FromPackedWriterManifest(t *testing.T }, }, nil) columnGroups := []storagecommon.ColumnGroup{{Columns: []int{0}, GroupID: storagecommon.DefaultShortColumnGroupID, Fields: []int64{100}}} - basePath := "files/packed_writer_format/1" + basePath := path.Join(CreateStorageConfig().GetRootPath(), "packed_writer_format/1") cfg := CreateStorageConfig() writer, err := NewFFIPackedWriter(basePath, schema, columnGroups, cfg, nil) require.NoError(t, err) @@ -250,7 +251,7 @@ func TestResolveManifestSingleWriterFormat_FiltersMixedAddColumnGroups(t *testin pt.Reset(pt.LocalStorageCfg.Path.Key) }) - basePath := "files/packed_writer_mixed_format/1" + basePath := path.Join(CreateStorageConfig().GetRootPath(), "packed_writer_mixed_format/1") cfg := CreateStorageConfig() writeColumn := func(name string, fieldID int64, format string, asNewColumnGroup bool) WriterOutput { schema := arrow.NewSchema([]arrow.Field{ @@ -347,7 +348,7 @@ func TestPackedFFIWriter(t *testing.T) { }, }, nil) - basePath := "files/packed_writer_test/1" + basePath := path.Join(CreateStorageConfig().GetRootPath(), "packed_writer_test/1") version := int64(0) for i := 0; i < batch; i++ { @@ -450,7 +451,7 @@ func TestFFIPackedWriter_CloseThenCommitUpdates(t *testing.T) { {Columns: []int{0}, GroupID: storagecommon.DefaultShortColumnGroupID}, } - basePath := "files/close_commit_test/1" + basePath := path.Join(CreateStorageConfig().GetRootPath(), "close_commit_test/1") cfg := CreateStorageConfig() w, err := NewFFIPackedWriter(basePath, schema, columnGroups, cfg, nil) require.NoError(t, err) diff --git a/internal/storagev2/packed/segment_reader_ffi.go b/internal/storagev2/packed/segment_reader_ffi.go new file mode 100644 index 0000000000..bda3917444 --- /dev/null +++ b/internal/storagev2/packed/segment_reader_ffi.go @@ -0,0 +1,183 @@ +// Licensed to the LF AI & Data foundation under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package packed + +/* +#cgo pkg-config: milvus-storage + +#include +#include "milvus-storage/ffi_c.h" +#include "arrow/c/abi.h" +*/ +import "C" + +import ( + "io" + "unsafe" + + "github.com/apache/arrow/go/v17/arrow" + "github.com/apache/arrow/go/v17/arrow/cdata" + + "github.com/milvus-io/milvus/pkg/v3/proto/indexpb" + "github.com/milvus-io/milvus/pkg/v3/util/merr" +) + +// FFISegmentReader resolves TEXT LOB references through the partition paths in +// textColumns and returns logical UTF8 columns. Compaction uses it when source +// and output manifests belong to different partition namespaces. +type FFISegmentReader struct { + handle C.LoonSegmentReaderHandle + recordReader arrayRecordReader + schema *arrow.Schema +} + +type arrayRecordReader interface { + Read() (arrow.Record, error) +} + +func NewFFISegmentReader( + manifestPath string, + schema *arrow.Schema, + neededColumns []string, + bufferSize int64, + storageConfig *indexpb.StorageConfig, + textColumns []TextColumnConfig, +) (*FFISegmentReader, error) { + if storageConfig == nil { + return nil, merr.WrapErrStorageMsg("storageConfig must not be nil") + } + basePath, version, err := UnmarshalManifestPath(manifestPath) + if err != nil { + return nil, err + } + + var cas cdata.CArrowSchema + cdata.ExportArrowSchema(schema, &cas) + cSchema := (*C.struct_ArrowSchema)(unsafe.Pointer(&cas)) + defer cdata.ReleaseCArrowSchema(&cas) + + cProperties, err := MakePropertiesFromStorageConfig(storageConfig, nil) + if err != nil { + return nil, err + } + defer C.loon_properties_free(cProperties) + + cConfig, cleanupConfig := buildCSegmentReaderConfig(textColumns, bufferSize) + defer cleanupConfig() + cNeededColumns, numColumns, cleanupColumns := buildCSegmentReaderColumns(neededColumns) + defer cleanupColumns() + cBasePath := C.CString(basePath) + defer C.free(unsafe.Pointer(cBasePath)) + + var handle C.LoonSegmentReaderHandle + result := C.loon_segment_reader_open( + cBasePath, + C.int64_t(version), + cSchema, + cNeededColumns, + numColumns, + cConfig, + cProperties, + &handle, + ) + if err := HandleLoonFFIResult(result); err != nil { + return nil, err + } + + var stream cdata.CArrowArrayStream + result = C.loon_segment_reader_get_stream(handle, (*C.struct_ArrowArrayStream)(unsafe.Pointer(&stream))) + if err := HandleLoonFFIResult(result); err != nil { + C.loon_segment_reader_destroy(handle) + return nil, err + } + recordReader, err := cdata.ImportCRecordReader(&stream, schema) + if err != nil { + C.loon_segment_reader_destroy(handle) + return nil, merr.WrapErrStorage(err, "failed to import TEXT-decoding segment reader") + } + + return &FFISegmentReader{handle: handle, recordReader: recordReader, schema: schema}, nil +} + +func buildCSegmentReaderConfig(textColumns []TextColumnConfig, bufferSize int64) (*C.LoonSegmentReaderConfig, func()) { + cConfig := (*C.LoonSegmentReaderConfig)(C.calloc(1, C.sizeof_LoonSegmentReaderConfig)) + cConfig.read_buffer_size = C.int64_t(bufferSize) + cConfig.num_lob_columns = C.size_t(len(textColumns)) + if len(textColumns) == 0 { + cConfig.lob_columns = nil + return cConfig, func() { C.free(unsafe.Pointer(cConfig)) } + } + + cColumns := (*C.LoonLobColumnConfig)(C.calloc(C.size_t(len(textColumns)), C.sizeof_LoonLobColumnConfig)) + columnSlice := unsafe.Slice(cColumns, len(textColumns)) + for i, column := range textColumns { + columnSlice[i].field_id = C.int64_t(column.FieldID) + columnSlice[i].lob_base_path = C.CString(column.LobBasePath) + } + cConfig.lob_columns = cColumns + return cConfig, func() { + for i := range columnSlice { + C.free(unsafe.Pointer(columnSlice[i].lob_base_path)) + } + C.free(unsafe.Pointer(cColumns)) + C.free(unsafe.Pointer(cConfig)) + } +} + +func buildCSegmentReaderColumns(columns []string) (**C.char, C.int64_t, func()) { + cColumns := make([]*C.char, len(columns)) + for i, column := range columns { + cColumns[i] = C.CString(column) + } + cleanup := func() { + for _, column := range cColumns { + C.free(unsafe.Pointer(column)) + } + } + if len(cColumns) == 0 { + return nil, 0, cleanup + } + return (**C.char)(unsafe.Pointer(&cColumns[0])), C.int64_t(len(cColumns)), cleanup +} + +func (r *FFISegmentReader) ReadNext() (arrow.Record, error) { + if r.recordReader == nil { + return nil, io.EOF + } + record, err := r.recordReader.Read() + if err != nil { + if err == io.EOF { + return nil, io.EOF + } + return nil, merr.WrapErrStorage(err, "failed to read TEXT-decoded segment batch") + } + return record, nil +} + +func (r *FFISegmentReader) Close() error { + if r.handle == 0 { + return nil + } + r.recordReader = nil + C.loon_segment_reader_destroy(r.handle) + r.handle = 0 + return nil +} + +func (r *FFISegmentReader) Schema() *arrow.Schema { + return r.schema +} diff --git a/internal/storagev2/packed/transaction_test.go b/internal/storagev2/packed/transaction_test.go index d10b9dc152..fe5a4fa3ff 100644 --- a/internal/storagev2/packed/transaction_test.go +++ b/internal/storagev2/packed/transaction_test.go @@ -537,7 +537,7 @@ func TestCreateMilvusTableManifestFromSegmentManifests_ImportsSourceDeltalogs(t func TestGetDeltaLogsFromManifestWithExtfsResolvesRelativeSourceDeltalogs(t *testing.T) { cfg := manifestTestStorageConfig(t) - sourceBasePath := "files/source/segment/10" + sourceBasePath := path.Join(cfg.GetRootPath(), "source/segment/10") sourceManifest := createBaseManifest(t, sourceBasePath, cfg) sourceDeltaPath := path.Join(sourceBasePath, "_delta/9001") diff --git a/internal/streamingnode/server/wal/vchannel/segment/pack_writer.go b/internal/streamingnode/server/wal/vchannel/segment/pack_writer.go index 76d042f94b..0ea695b585 100644 --- a/internal/streamingnode/server/wal/vchannel/segment/pack_writer.go +++ b/internal/streamingnode/server/wal/vchannel/segment/pack_writer.go @@ -2,7 +2,6 @@ package segment import ( "context" - "path" "github.com/samber/lo" "google.golang.org/protobuf/proto" @@ -18,13 +17,11 @@ import ( "github.com/milvus-io/milvus/internal/storage" "github.com/milvus-io/milvus/internal/storagecommon" "github.com/milvus-io/milvus/internal/storagev2/packed" - "github.com/milvus-io/milvus/pkg/v3/common" "github.com/milvus-io/milvus/pkg/v3/mq/msgstream" "github.com/milvus-io/milvus/pkg/v3/proto/datapb" "github.com/milvus-io/milvus/pkg/v3/proto/indexpb" "github.com/milvus-io/milvus/pkg/v3/proto/streamingpb" "github.com/milvus-io/milvus/pkg/v3/util/merr" - "github.com/milvus-io/milvus/pkg/v3/util/metautil" "github.com/milvus-io/milvus/pkg/v3/util/paramtable" "github.com/milvus-io/milvus/pkg/v3/util/retry" "github.com/milvus-io/milvus/pkg/v3/util/typeutil" @@ -396,7 +393,9 @@ func manifestPathForGrowingPack(meta *streamingpb.SegmentAssignmentMeta) string if meta.GetStorageVersion() != storage.StorageV3 { return "" } - k := metautil.JoinIDPath(meta.GetCollectionId(), meta.GetPartitionId(), meta.GetSegmentId()) - basePath := path.Join(paramtable.Get().MinioCfg.RootPath.GetValue(), common.SegmentInsertLogPath, k) + // Use the primary storage root: localStorage.path for local storage, + // minio.rootPath for remote storage, consistent with other V3 writers. + basePath := storage.SegmentManifestBasePath(packed.CreateStorageConfig().GetRootPath(), + meta.GetCollectionId(), meta.GetPartitionId(), meta.GetSegmentId()) return packed.MarshalManifestPath(basePath, packed.ManifestEarliest) } diff --git a/internal/streamingnode/server/wal/vchannel/segment/pack_writer_test.go b/internal/streamingnode/server/wal/vchannel/segment/pack_writer_test.go index 53ea862d64..2d611c9ca3 100644 --- a/internal/streamingnode/server/wal/vchannel/segment/pack_writer_test.go +++ b/internal/streamingnode/server/wal/vchannel/segment/pack_writer_test.go @@ -2,6 +2,7 @@ package segment import ( "context" + "path" "testing" "github.com/stretchr/testify/assert" @@ -10,6 +11,7 @@ import ( "github.com/milvus-io/milvus-proto/go-api/v3/msgpb" "github.com/milvus-io/milvus-proto/go-api/v3/schemapb" "github.com/milvus-io/milvus/internal/storage" + "github.com/milvus-io/milvus/internal/storagev2/packed" "github.com/milvus-io/milvus/pkg/v3/proto/datapb" "github.com/milvus-io/milvus/pkg/v3/proto/messagespb" "github.com/milvus-io/milvus/pkg/v3/proto/streamingpb" @@ -39,6 +41,75 @@ func TestCurrentSplitForGrowingPackFillsNewSplitFormats(t *testing.T) { } } +func TestManifestPathForGrowingPackUsesPrimaryStorageRoot(t *testing.T) { + params := paramtable.Get() + meta := &streamingpb.SegmentAssignmentMeta{ + CollectionId: 1, + PartitionId: 2, + SegmentId: 3, + StorageVersion: storage.StorageV3, + } + localRoot := t.TempDir() + + for _, testCase := range []struct { + name string + storageType string + minioRoot string + wantRoot string + }{ + {name: "local", storageType: "local", minioRoot: "files", wantRoot: localRoot}, + {name: "remote", storageType: "remote", minioRoot: "files", wantRoot: "files"}, + {name: "remote_bucket_root", storageType: "remote", minioRoot: "/", wantRoot: "."}, + } { + t.Run(testCase.name, func(t *testing.T) { + require.NoError(t, params.Save(params.CommonCfg.StorageType.Key, testCase.storageType)) + // minio.rootPath stays set in both cases: under local storage it + // must not leak into the key. + require.NoError(t, params.Save(params.MinioCfg.RootPath.Key, testCase.minioRoot)) + require.NoError(t, params.Save(params.LocalStorageCfg.Path.Key, localRoot)) + t.Cleanup(func() { + _ = params.Reset(params.CommonCfg.StorageType.Key) + _ = params.Reset(params.MinioCfg.RootPath.Key) + _ = params.Reset(params.LocalStorageCfg.Path.Key) + }) + + base, version, err := packed.UnmarshalManifestPath(manifestPathForGrowingPack(meta)) + require.NoError(t, err) + assert.Equal(t, path.Join(testCase.wantRoot, "insert_log", "1", "2", "3"), base) + assert.Equal(t, packed.ManifestEarliest, version) + }) + } +} + +func TestManifestPathForGrowingPackPreservesPersistedPath(t *testing.T) { + for _, manifestPath := range []string{ + packed.MarshalManifestPath("files/insert_log/1/2/3", 7), + packed.MarshalManifestPath(path.Join(t.TempDir(), "insert_log/1/2/3"), 9), + } { + meta := &streamingpb.SegmentAssignmentMeta{ + CollectionId: 1, + PartitionId: 2, + SegmentId: 3, + StorageVersion: storage.StorageV3, + PersistedStorage: &streamingpb.L1SegmentPersistedStorage{ + ManifestPath: manifestPath, + }, + } + assert.Equal(t, manifestPath, manifestPathForGrowingPack(meta)) + } +} + +func TestManifestPathForGrowingPackSkipsNonV3(t *testing.T) { + for _, storageVersion := range []int64{storage.StorageV1, storage.StorageV2} { + assert.Empty(t, manifestPathForGrowingPack(&streamingpb.SegmentAssignmentMeta{ + CollectionId: 1, + PartitionId: 2, + SegmentId: 3, + StorageVersion: storageVersion, + })) + } +} + func TestCurrentSplitFromPersistedStorageRestoresFormat(t *testing.T) { schema := &schemapb.CollectionSchema{Fields: []*schemapb.FieldSchema{ {FieldID: 100, DataType: schemapb.DataType_Int64, IsPrimaryKey: true}, diff --git a/internal/util/segcore/segment.go b/internal/util/segcore/segment.go index fd3b741f59..c6e6a2e368 100644 --- a/internal/util/segcore/segment.go +++ b/internal/util/segcore/segment.go @@ -21,6 +21,7 @@ import ( "google.golang.org/protobuf/proto" "github.com/milvus-io/milvus-proto/go-api/v3/commonpb" + "github.com/milvus-io/milvus/internal/metastore/kv/binlog" "github.com/milvus-io/milvus/internal/storage" "github.com/milvus-io/milvus/internal/storagev2/packed" "github.com/milvus-io/milvus/internal/util/cgo" @@ -30,7 +31,6 @@ import ( "github.com/milvus-io/milvus/pkg/v3/proto/segcorepb" "github.com/milvus-io/milvus/pkg/v3/util/merr" "github.com/milvus-io/milvus/pkg/v3/util/metautil" - "github.com/milvus-io/milvus/pkg/v3/util/paramtable" "github.com/milvus-io/milvus/pkg/v3/util/tsoutil" ) @@ -485,8 +485,9 @@ func resolveStatsWithBasePaths(src *querypb.SegmentLoadInfo) ( } } - // V2: compute basePaths from rootPath + stats metadata. - rootPath := paramtable.Get().MinioCfg.RootPath.GetValue() + // Legacy non-manifest stats use the primary storage root: an absolute + // localStorage.path for local storage, minio.rootPath for object storage. + rootPath := binlog.GetRootPath() textBasePaths := make(map[int64]string, len(textStats)) for fieldID, stats := range textStats { diff --git a/internal/util/segcore/segment_test.go b/internal/util/segcore/segment_test.go index 9f11723e35..210eab1910 100644 --- a/internal/util/segcore/segment_test.go +++ b/internal/util/segcore/segment_test.go @@ -2,10 +2,12 @@ package segcore_test import ( "context" + "path" "path/filepath" "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "google.golang.org/protobuf/proto" "github.com/milvus-io/milvus-proto/go-api/v3/commonpb" @@ -471,6 +473,58 @@ func TestConvertToSegcoreSegmentLoadInfo_IndexStorePathVersion(t *testing.T) { assert.Equal(t, indexpb.IndexStorePathVersion_INDEX_STORE_PATH_VERSION_COLLECTION_ROOTED, result.GetIndexInfos()[0].GetIndexStorePathVersion()) } +func TestConvertToSegcoreSegmentLoadInfo_LegacyStatsRoot(t *testing.T) { + paramtable.Init() + params := paramtable.Get() + for _, item := range []*paramtable.ParamItem{¶ms.CommonCfg.StorageType, ¶ms.LocalStorageCfg.Path, ¶ms.MinioCfg.RootPath} { + key, value := item.Key, item.GetValue() + t.Cleanup(func() { require.NoError(t, params.Save(key, value)) }) + } + const localRoot = "/var/lib/milvus/data" + require.NoError(t, params.Save(params.LocalStorageCfg.Path.Key, localRoot)) + require.NoError(t, params.Save(params.MinioCfg.RootPath.Key, "files")) + for _, backend := range []struct{ name, root string }{{"local", localRoot}, {"remote", "files"}} { + t.Run(backend.name, func(t *testing.T) { + require.NoError(t, params.Save(params.CommonCfg.StorageType.Key, backend.name)) + for _, version := range []int64{storage.StorageV1, storage.StorageV2} { + for _, fullPaths := range []bool{false, true} { + textBase := backend.root + "/text_log/10/1/2/3/4/106" + jsonBase := backend.root + "/json_stats/3/11/1/2/3/4/107" + textFiles := []string{"index.v3"} + jsonFiles := []string{"meta.json", "shared_key_index/0", "shredding_data/0/0"} + inputText := append([]string(nil), textFiles...) + inputJSON := append([]string(nil), jsonFiles...) + if fullPaths { + for i := range inputText { + inputText[i] = path.Join(textBase, inputText[i]) + } + for i := range inputJSON { + inputJSON[i] = path.Join(jsonBase, inputJSON[i]) + } + } + src := &querypb.SegmentLoadInfo{ + StorageVersion: version, CollectionID: 2, PartitionID: 3, SegmentID: 4, + TextStatsLogs: map[int64]*datapb.TextIndexStats{ + 106: {FieldID: 106, BuildID: 10, Version: 1, Files: inputText}, + }, + JsonKeyStatsLogs: map[int64]*datapb.JsonKeyStats{ + 107: {FieldID: 107, BuildID: 11, Version: 1, JsonKeyStatsDataFormat: 3, Files: inputJSON}, + }, + } + original := proto.Clone(src) + result, err := segcore.ConvertToSegcoreSegmentLoadInfo(src) + require.NoError(t, err, "version=%d fullPaths=%t", version, fullPaths) + assert.Equal(t, textBase, result.TextStatsLogs[106].BasePath) + assert.Equal(t, jsonBase, result.JsonKeyStatsLogs[107].BasePath) + assert.Equal(t, textFiles, result.TextStatsLogs[106].Files) + assert.Equal(t, jsonFiles, result.JsonKeyStatsLogs[107].Files) + assert.True(t, proto.Equal(original, src), "conversion must not rewrite source metadata") + } + } + }) + } +} + func TestConvertToSegcoreSegmentLoadInfo_V3ManifestErrorReturnsError(t *testing.T) { src := &querypb.SegmentLoadInfo{ StorageVersion: storage.StorageV3, diff --git a/pkg/util/metautil/binlog.go b/pkg/util/metautil/binlog.go index 8ae885a2ba..46a709b97a 100644 --- a/pkg/util/metautil/binlog.go +++ b/pkg/util/metautil/binlog.go @@ -196,7 +196,12 @@ func ExtractJSONKeyStatsRelativePath(file string) string { // BuildJSONKeyStatsV3Prefix returns the V3 remote base path for JSON key stats. // Format: {segmentBasePath}/_stats/json_stats.{fieldID} func BuildJSONKeyStatsV3Prefix(segmentBasePath string, fieldID int64) string { - return path.Join(segmentBasePath, "_stats", "json_stats."+strconv.FormatInt(fieldID, 10)) + suffix := "_stats/json_stats." + strconv.FormatInt(fieldID, 10) + if segmentBasePath == "" { + return suffix + } + // The manifest base is a complete key; preserve literal remote components. + return strings.TrimRight(segmentBasePath, "/") + "/" + suffix } // JSONStatsSegmentInfo is the segment metadata needed to derive a JSON stats base path. diff --git a/pkg/util/metautil/binlog_test.go b/pkg/util/metautil/binlog_test.go index f48fcd7cbd..073fd5e4c4 100644 --- a/pkg/util/metautil/binlog_test.go +++ b/pkg/util/metautil/binlog_test.go @@ -26,6 +26,22 @@ import ( "github.com/milvus-io/milvus/pkg/v3/util/typeutil" ) +func TestBuildJSONKeyStatsV3PrefixPreservesBase(t *testing.T) { + for _, base := range []string{"files/insert_log/1/2/3", "/data/insert_log/1/2/3", "files/tmp/../insert_log/1/2/3", "files//insert_log/1/2/3"} { + for _, trailing := range []string{"", "/"} { + got := BuildJSONKeyStatsV3Prefix(base+trailing, 100) + if want := base + "/_stats/json_stats.100"; got != want { + t.Fatalf("base %q: got %q, want %q", base+trailing, got, want) + } + } + } + for base, want := range map[string]string{"": "_stats/json_stats.100", "/": "/_stats/json_stats.100"} { + if got := BuildJSONKeyStatsV3Prefix(base, 100); got != want { + t.Fatalf("base %q: got %q, want %q", base, got, want) + } + } +} + func TestParseInsertLogPath(t *testing.T) { type args struct { path string diff --git a/pkg/util/paramtable/component_param.go b/pkg/util/paramtable/component_param.go index 65a66489cc..0ab063c4c1 100644 --- a/pkg/util/paramtable/component_param.go +++ b/pkg/util/paramtable/component_param.go @@ -166,7 +166,7 @@ func (p *ComponentParam) init(bt *BaseTable) { p.MixCoordCfg.init(bt) p.ProxyCfg.init(bt) p.QueryCoordCfg.init(bt) - p.QueryNodeCfg.init(bt) + p.QueryNodeCfg.init(bt, p.LocalStorageCfg.Path.GetValue()) p.DataCoordCfg.init(bt) p.DataNodeCfg.init(bt) p.StreamingCfg.init(bt) @@ -4488,7 +4488,7 @@ func formatDurationWithMillisecondFallback(v string) string { return v } -func (p *queryNodeConfig) init(base *BaseTable) { +func (p *queryNodeConfig) init(base *BaseTable, localStoragePath string) { p.StrictGroupAcceptanceThreshold = ParamItem{ Key: "queryNode.groupBy.strictGroupAcceptanceThreshold", Version: "2.6.23", DefaultValue: "0.1", Export: true, @@ -5178,7 +5178,6 @@ This defaults to true, indicating that Milvus creates temporary index for growin Doc: "Deprecated: The folder that storing data files for mmap, setting to a path will enable Milvus to load data with mmap", Formatter: func(v string) string { if len(v) == 0 { - localStoragePath := getLocalStoragePath(base) return path.Join(localStoragePath, "mmap") } return v @@ -5451,7 +5450,6 @@ Max read concurrency must greater than or equal to 1, and less than or equal to Formatter: func(v string) string { if len(v) == 0 { // use local storage path to check correct device - localStoragePath := getLocalStoragePath(base) if _, err := os.Stat(localStoragePath); os.IsNotExist(err) { if err := os.MkdirAll(localStoragePath, os.ModePerm); err != nil { mlog.Fatal(context.TODO(), "failed to mkdir", mlog.String("localStoragePath", localStoragePath), mlog.Err(err)) @@ -9687,12 +9685,3 @@ func (params *ComponentParam) Reset(key string) error { func (params *ComponentParam) GetWithDefault(key string, dft string) string { return params.baseTable.GetWithDefault(key, dft) } - -func getLocalStoragePath(base *BaseTable) string { - localStoragePath := base.Get("localStorage.path") - if len(localStoragePath) == 0 { - localStoragePath = defaultLocalStoragePath - mlog.Warn(context.TODO(), "localStorage.path is not set, using default value", mlog.String("localStorage.path", localStoragePath)) - } - return localStoragePath -} diff --git a/pkg/util/paramtable/component_param_test.go b/pkg/util/paramtable/component_param_test.go index af9063aa39..ac1aee5a94 100644 --- a/pkg/util/paramtable/component_param_test.go +++ b/pkg/util/paramtable/component_param_test.go @@ -47,6 +47,40 @@ func TestQueryNodeStrictGroupSettings(t *testing.T) { assert.Equal(t, 100, cfg.StrictGroupProbeCandidates.GetAsInt()) } +func TestComponentParamDerivedLocalStoragePathsUseCanonicalFrozenValue(t *testing.T) { + workingDir := t.TempDir() + t.Chdir(workingDir) + configDir := t.TempDir() + t.Setenv("MILVUSCONF", configDir) + configFile := filepath.Join(configDir, "local-storage.yaml") + root := filepath.Join(t.TempDir(), "canonical") + require.NoError(t, os.WriteFile(configFile, []byte("localStorage:\n path: \" "+root+"/./ \"\n"), 0o600)) + + bt := NewBaseTable(SkipRemote(true), SkipEnv(true), Files([]string{filepath.Base(configFile)}), Interval(10*time.Millisecond)) + t.Cleanup(bt.mgr.Close) + params := &ComponentParam{} + params.Init(bt) + + wantRoot := filepath.Clean(root) + require.Equal(t, wantRoot, params.LocalStorageCfg.Path.GetValue()) + require.Equal(t, filepath.Join(wantRoot, "pprof"), params.ProfileCfg.PprofPath.GetValue()) + require.Equal(t, filepath.Join(wantRoot, "mmap"), params.QueryNodeCfg.MmapDirPath.GetValue()) + initialDiskCapacity := params.QueryNodeCfg.DiskCapacityLimit.GetValue() + + // File/config-center updates replace the raw source value before the + // Forbidden handler rejects them. Derived paths must keep using the same + // canonical startup root as LocalStorageCfg.Path, not that rejected raw value. + require.NoError(t, os.WriteFile(configFile, []byte("localStorage:\n path: relative/data\n"), 0o600)) + require.Eventually(t, func() bool { + return bt.Get("localStorage.path") == "relative/data" + }, time.Second, 10*time.Millisecond) + require.Equal(t, wantRoot, params.LocalStorageCfg.Path.GetValue()) + require.Equal(t, filepath.Join(wantRoot, "pprof"), params.ProfileCfg.PprofPath.GetValue()) + require.Equal(t, filepath.Join(wantRoot, "mmap"), params.QueryNodeCfg.MmapDirPath.GetValue()) + require.Equal(t, initialDiskCapacity, params.QueryNodeCfg.DiskCapacityLimit.GetValue()) + require.NoDirExists(t, filepath.Join(workingDir, "relative")) +} + func shouldPanic(t *testing.T, name string, f func()) { defer func() { recover() }() f() diff --git a/pkg/util/paramtable/param_item.go b/pkg/util/paramtable/param_item.go index dd25892133..8c1639d92a 100644 --- a/pkg/util/paramtable/param_item.go +++ b/pkg/util/paramtable/param_item.go @@ -260,22 +260,23 @@ func (pi *ParamItem) getWithRaw() (result, raw string, err error) { if s := pi.tempValue.Load(); s != nil { return pi.gateValue(*s), *s, nil } - if pi.manager == nil { panic(fmt.Sprintf("manager is nil %s", pi.Key)) } // raw is always the primary key's value, used for CAS comparison. // effectiveRaw is the value actually used for computing result (may come from fallback). - _, raw, err = pi.manager.GetConfig(pi.Key) + source, raw, err := pi.manager.GetConfig(pi.Key) effectiveRaw := raw + effectiveSource := source if err != nil || raw == pi.DefaultValue { // try fallback if the entry is not exist or default value, // because default value may already defined in milvus.yaml // and we don't want the fallback keys be overridden. for _, key := range pi.FallbackKeys { - var fallbackRaw string - _, fallbackRaw, err = pi.manager.GetConfig(key) + fallbackSource, fallbackRaw, fallbackErr := pi.manager.GetConfig(key) + err = fallbackErr if err == nil { + effectiveSource = fallbackSource effectiveRaw = fallbackRaw break } @@ -286,6 +287,15 @@ func (pi *ParamItem) getWithRaw() (result, raw string, err error) { effectiveRaw = pi.DefaultValue raw = pi.DefaultValue } + // Config sources replace their value before Manager.OnEvent can reject the + // change. Keep a forbidden item's formatted startup value when that happens. + // RuntimeSource remains an explicit process-local override used by tests and + // operational tooling. + if pi.Forbidden && effectiveSource != config.RuntimeSource { + if initial := pi.lastValue.Load(); initial != nil { + return *initial, raw, nil + } + } result = pi.gateValue(effectiveRaw) if pi.Formatter != nil { result = pi.Formatter(result) diff --git a/pkg/util/paramtable/param_item_test.go b/pkg/util/paramtable/param_item_test.go index cd1d9e3cf7..abfde5a6a7 100644 --- a/pkg/util/paramtable/param_item_test.go +++ b/pkg/util/paramtable/param_item_test.go @@ -85,6 +85,22 @@ func TestParamGroupDeleteDoesNotSurfaceTombstone(t *testing.T) { assert.Empty(t, group.GetValue()) } +func TestForbiddenParamItemAllowsRuntimeOverride(t *testing.T) { + manager := config.NewManager() + manager.SetConfig("test.static.path", "initial") + + param := &ParamItem{ + Key: "test.static.path", + Forbidden: true, + Formatter: strings.ToUpper, + } + param.Init(manager) + require.Equal(t, "INITIAL", param.GetValue()) + + manager.SetConfig("test.static.path", "runtime") + require.Equal(t, "RUNTIME", param.GetValue()) +} + func TestGetWithRaw_FallbackKeyCacheSuccess(t *testing.T) { // When primary key equals DefaultValue and a fallback key has a different value, // getWithRaw should return the fallback value as result but the primary key's diff --git a/pkg/util/paramtable/service_param.go b/pkg/util/paramtable/service_param.go index f2821aace1..5db92b5c8b 100644 --- a/pkg/util/paramtable/service_param.go +++ b/pkg/util/paramtable/service_param.go @@ -23,6 +23,7 @@ import ( "net/url" "os" "path" + "path/filepath" "strconv" "strings" "time" @@ -33,6 +34,7 @@ import ( "github.com/milvus-io/milvus/pkg/v3/mlog" "github.com/milvus-io/milvus/pkg/v3/util" "github.com/milvus-io/milvus/pkg/v3/util/etcd" + "github.com/milvus-io/milvus/pkg/v3/util/merr" "github.com/milvus-io/milvus/pkg/v3/util/metricsinfo" ) @@ -116,7 +118,7 @@ func (p *ServiceParam) init(bt *BaseTable) { p.KafkaCfg.Init(bt) p.RocksmqCfg.Init(bt) p.MinioCfg.Init(bt) - p.ProfileCfg.Init(bt) + p.ProfileCfg.init(bt, p.LocalStorageCfg.Path.GetValue()) } func (p *ServiceParam) RocksmqEnable() bool { @@ -611,13 +613,29 @@ func (p *LocalStorageConfig) Init(base *BaseTable) { Version: "2.0.0", DefaultValue: defaultLocalStoragePath, Doc: `Local path to where vector data are stored during a search or a query to avoid repetitve access to MinIO or S3 service. +Must be an absolute filesystem path. Milvus refuses to start if this value is relative or empty. +Migration of data written using a relative path in older versions is not supported. Caution: Changing this parameter after using Milvus for a period of time will affect your access to old data. It is recommended to change this parameter before starting Milvus for the first time.`, - Export: true, + // Every local storage key is a complete filesystem path that starts + // with this value, and the loon local filesystem is rooted at "/", so + // a relative path here would silently depend on the process working + // directory. Reject it during configuration initialization. + Formatter: formatLocalStoragePath, + Forbidden: true, + Export: true, } p.Path.Init(base.mgr) } +func formatLocalStoragePath(value string) string { + value = strings.TrimSpace(value) + if !filepath.IsAbs(value) { + panic(merr.WrapErrParameterInvalidMsg("localStorage.path must be an absolute filesystem path, got %q", value)) + } + return filepath.Clean(value) +} + type MetaStoreConfig struct { MetaStoreType ParamItem `refreshable:"false"` PaginationSize ParamItem `refreshable:"true"` @@ -2088,6 +2106,10 @@ type ProfileConfig struct { } func (p *ProfileConfig) Init(base *BaseTable) { + p.init(base, formatLocalStoragePath(base.GetWithDefault("localStorage.path", defaultLocalStoragePath))) +} + +func (p *ProfileConfig) init(base *BaseTable, localStoragePath string) { p.PprofPath = ParamItem{ Key: "profile.pprof.path", Version: "2.5.5", @@ -2095,7 +2117,6 @@ func (p *ProfileConfig) Init(base *BaseTable) { Doc: "The folder that storing pprof files, by default will use localStoragePath/pprof", Formatter: func(v string) string { if len(v) == 0 { - localStoragePath := getLocalStoragePath(base) return path.Join(localStoragePath, "pprof") } return v diff --git a/pkg/util/paramtable/service_param_test.go b/pkg/util/paramtable/service_param_test.go index 9b7ec84c00..653344bef2 100644 --- a/pkg/util/paramtable/service_param_test.go +++ b/pkg/util/paramtable/service_param_test.go @@ -17,18 +17,119 @@ package paramtable import ( + "os" + "path/filepath" "strconv" "testing" "time" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/milvus-io/milvus/pkg/v3/config" "github.com/milvus-io/milvus/pkg/v3/util" + "github.com/milvus-io/milvus/pkg/v3/util/merr" "github.com/milvus-io/milvus/pkg/v3/util/metricsinfo" "github.com/milvus-io/milvus/pkg/v3/util/typeutil" ) +// Every local storage key is a complete filesystem path that starts with +// localStorage.path, and the loon local filesystem is rooted at "/", so the +// configured value must never depend on the process working directory. +func TestLocalStorageConfig_PathIsAbsolute(t *testing.T) { + for _, tc := range []struct { + name string + configured string + want string + }{ + {name: "default", want: filepath.Clean(defaultLocalStoragePath)}, + {name: "absolute", configured: "/var/lib/milvus/data", want: "/var/lib/milvus/data"}, + {name: "clean_absolute", configured: " /var/lib/milvus/./old/../data/ ", want: "/var/lib/milvus/data"}, + {name: "filesystem_root", configured: "/", want: "/"}, + } { + t.Run(tc.name, func(t *testing.T) { + bt := NewBaseTable(SkipRemote(true), SkipEnv(true), Files([]string{})) + t.Cleanup(bt.mgr.Close) + if tc.configured != "" { + require.NoError(t, bt.Save("localStorage.path", tc.configured)) + } + var params ServiceParam + require.NotPanics(t, func() { params.init(bt) }) + require.Equal(t, tc.want, params.LocalStorageCfg.Path.GetValue()) + require.True(t, filepath.IsAbs(params.LocalStorageCfg.Path.GetValue())) + }) + } +} + +func TestLocalStorageConfig_RejectsInvalidPathDuringInit(t *testing.T) { + assertInvalidInit := func(t *testing.T, bt *BaseTable) { + t.Helper() + t.Cleanup(bt.mgr.Close) + var panicValue any + func() { + defer func() { panicValue = recover() }() + var params ServiceParam + params.init(bt) + }() + require.NotNil(t, panicValue, "service initialization must reject the configured path before starting components") + panicErr, ok := panicValue.(error) + require.True(t, ok, "initialization must panic with a typed configuration error, got %T", panicValue) + require.ErrorIs(t, panicErr, merr.ErrParameterInvalid) + require.Contains(t, panicErr.Error(), "localStorage.path must be an absolute filesystem path") + } + for _, tc := range []struct { + name string + value string + }{ + {name: "relative", value: "relative/data"}, + {name: "current_directory_relative", value: "./data"}, + {name: "parent_directory_relative", value: "../data"}, + {name: "current_directory", value: "."}, + {name: "empty", value: ""}, + {name: "blank", value: " \t\n "}, + } { + t.Run("saved/"+tc.name, func(t *testing.T) { + bt := NewBaseTable(SkipRemote(true), SkipEnv(true), Files([]string{})) + // Save before initialization exercises the startup path, rather + // than calling the formatter through a later GetValue. + require.NoError(t, bt.Save("localStorage.path", tc.value)) + assertInvalidInit(t, bt) + }) + t.Run("environment/"+tc.name, func(t *testing.T) { + t.Setenv("LOCALSTORAGE_PATH", tc.value) + bt := NewBaseTable(SkipRemote(true), Files([]string{})) + assertInvalidInit(t, bt) + }) + } +} + +func TestLocalStorageConfig_PathDoesNotChangeAtRuntime(t *testing.T) { + dir := t.TempDir() + t.Setenv("MILVUSCONF", dir) + configFile := filepath.Join(dir, "local-storage.yaml") + require.NoError(t, os.WriteFile(configFile, []byte("localStorage:\n path: /var/lib/milvus/data\n"), 0o600)) + bt := NewBaseTable(SkipRemote(true), SkipEnv(true), Files([]string{filepath.Base(configFile)}), Interval(10*time.Millisecond)) + t.Cleanup(bt.mgr.Close) + + var params ServiceParam + require.NotPanics(t, func() { params.init(bt) }) + require.Equal(t, "/var/lib/milvus/data", params.LocalStorageCfg.Path.GetValue()) + + // File and config-center sources replace their stored value before emitting + // an update event. A forbidden path must retain its startup value even then. + require.NoError(t, os.WriteFile(configFile, []byte("localStorage:\n path: relative/data\n"), 0o600)) + require.Eventually(t, func() bool { + return bt.Get("localStorage.path") == "relative/data" + }, time.Second, 10*time.Millisecond) + require.NotPanics(t, func() { + require.Equal(t, "/var/lib/milvus/data", params.LocalStorageCfg.Path.GetValue()) + }) + value, raw, err := params.LocalStorageCfg.Path.getWithRaw() + require.NoError(t, err) + require.Equal(t, "/var/lib/milvus/data", value) + require.Equal(t, "relative/data", raw) +} + func TestServiceParam(t *testing.T) { var SParams ServiceParam bt := NewBaseTable(SkipRemote(true)) diff --git a/scripts/segcore_error_boundary_baseline.txt b/scripts/segcore_error_boundary_baseline.txt index 67a5239643..99af3023d4 100644 --- a/scripts/segcore_error_boundary_baseline.txt +++ b/scripts/segcore_error_boundary_baseline.txt @@ -167,6 +167,7 @@ milvus_storage internal/core/src/storage/ArrowFileSystemChunkManager.h milvus_storage internal/core/src/storage/FileManager.h milvus_storage internal/core/src/storage/IndexEntryEncryptedLocalWriter.cpp milvus_storage internal/core/src/storage/IndexEntryEncryptedLocalWriter.h +milvus_storage internal/core/src/storage/minio/MinioChunkManager.cpp milvus_storage internal/core/src/storage/loon_ffi/ffi_reader_c.cpp milvus_storage internal/core/src/storage/loon_ffi/loon_error_code.h milvus_storage internal/core/src/storage/loon_ffi/milvus_table_c.cpp