mirror of
https://github.com/milvus-io/milvus.git
synced 2026-10-12 02:19:10 +00:00
zhagnluandluzhang
48faae3b3a
issue: #53765 JSON stats build benchmark — 500K rows per segment Concurrent segments || Non-parallel || 8 workers || Speedup 1 || 6.810 s || 1.641 s || 4.15× faster 4 || 7.140 s || 4.806 s || 1.49× faster 8 || 10.878 s || 9.658 s || 1.13× faster Setup: Release build, 12 CPUs, local filesystem, one warm-up followed by five measured runs; median values are reported. Data generation is excluded from the timing. Note: “Non-parallel” means each segment is built serially internally. Multiple segment builds are still launched concurrently in 8-segment cases. Signed-off-by: luzhang <[email protected]> Co-authored-by: luzhang <[email protected]>
1690 lines
89 KiB
Go
1690 lines
89 KiB
Go
// Licensed to the LF AI & Data foundation under one
|
|
// or more contributor license agreements. See the NOTICE file
|
|
// distributed with this work for additional information
|
|
// regarding copyright ownership. The ASF licenses this file
|
|
// to you under the Apache License, Version 2.0 (the
|
|
// "License"); you may not use this file except in compliance
|
|
// with the License. You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package paramtable
|
|
|
|
import (
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
"reflect"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
|
|
"github.com/milvus-io/milvus/pkg/v3/config"
|
|
"github.com/milvus-io/milvus/pkg/v3/util/hardware"
|
|
)
|
|
|
|
func TestQueryNodeStrictGroupSettings(t *testing.T) {
|
|
params := &ComponentParam{}
|
|
params.Init(NewBaseTable(SkipRemote(true)))
|
|
cfg := ¶ms.QueryNodeCfg
|
|
assert.Equal(t, 0.1, cfg.StrictGroupAcceptanceThreshold.GetAsFloat())
|
|
assert.Equal(t, 100, cfg.StrictGroupProbeCandidates.GetAsInt())
|
|
params.Save(cfg.StrictGroupAcceptanceThreshold.Key, "0")
|
|
params.Save(cfg.StrictGroupProbeCandidates.Key, "17")
|
|
assert.Equal(t, 0.0, cfg.StrictGroupAcceptanceThreshold.GetAsFloat())
|
|
assert.Equal(t, 17, cfg.StrictGroupProbeCandidates.GetAsInt())
|
|
params.Reset(cfg.StrictGroupAcceptanceThreshold.Key)
|
|
params.Reset(cfg.StrictGroupProbeCandidates.Key)
|
|
assert.Equal(t, 0.1, cfg.StrictGroupAcceptanceThreshold.GetAsFloat())
|
|
assert.Equal(t, 100, cfg.StrictGroupProbeCandidates.GetAsInt())
|
|
}
|
|
|
|
func TestComponentParamDerivedLocalStoragePathsUseCanonicalFrozenValue(t *testing.T) {
|
|
workingDir := t.TempDir()
|
|
t.Chdir(workingDir)
|
|
configDir := t.TempDir()
|
|
t.Setenv("MILVUSCONF", configDir)
|
|
configFile := filepath.Join(configDir, "local-storage.yaml")
|
|
root := filepath.Join(t.TempDir(), "canonical")
|
|
require.NoError(t, os.WriteFile(configFile, []byte("localStorage:\n path: \" "+root+"/./ \"\n"), 0o600))
|
|
|
|
bt := NewBaseTable(SkipRemote(true), SkipEnv(true), Files([]string{filepath.Base(configFile)}), Interval(10*time.Millisecond))
|
|
t.Cleanup(bt.mgr.Close)
|
|
params := &ComponentParam{}
|
|
params.Init(bt)
|
|
|
|
wantRoot := filepath.Clean(root)
|
|
require.Equal(t, wantRoot, params.LocalStorageCfg.Path.GetValue())
|
|
require.Equal(t, filepath.Join(wantRoot, "pprof"), params.ProfileCfg.PprofPath.GetValue())
|
|
require.Equal(t, filepath.Join(wantRoot, "mmap"), params.QueryNodeCfg.MmapDirPath.GetValue())
|
|
initialDiskCapacity := params.QueryNodeCfg.DiskCapacityLimit.GetValue()
|
|
|
|
// File/config-center updates replace the raw source value before the
|
|
// Forbidden handler rejects them. Derived paths must keep using the same
|
|
// canonical startup root as LocalStorageCfg.Path, not that rejected raw value.
|
|
require.NoError(t, os.WriteFile(configFile, []byte("localStorage:\n path: relative/data\n"), 0o600))
|
|
require.Eventually(t, func() bool {
|
|
return bt.Get("localStorage.path") == "relative/data"
|
|
}, time.Second, 10*time.Millisecond)
|
|
require.Equal(t, wantRoot, params.LocalStorageCfg.Path.GetValue())
|
|
require.Equal(t, filepath.Join(wantRoot, "pprof"), params.ProfileCfg.PprofPath.GetValue())
|
|
require.Equal(t, filepath.Join(wantRoot, "mmap"), params.QueryNodeCfg.MmapDirPath.GetValue())
|
|
require.Equal(t, initialDiskCapacity, params.QueryNodeCfg.DiskCapacityLimit.GetValue())
|
|
require.NoDirExists(t, filepath.Join(workingDir, "relative"))
|
|
}
|
|
|
|
func shouldPanic(t *testing.T, name string, f func()) {
|
|
defer func() { recover() }()
|
|
f()
|
|
t.Errorf("%s should have panicked", name)
|
|
}
|
|
|
|
func TestComponentParam_DataCoordBumpSchemaVersionCompactionParams(t *testing.T) {
|
|
Init()
|
|
params := Get()
|
|
params.Reset(params.DataCoordCfg.BumpSchemaVersionCompactionEnabled.Key)
|
|
params.Reset(params.DataCoordCfg.BumpSchemaVersionCompactionTriggerInterval.Key)
|
|
params.Reset(params.DataCoordCfg.BumpSchemaVersionCompactionSlotUsage.Key)
|
|
t.Cleanup(func() {
|
|
params.Reset(params.DataCoordCfg.BumpSchemaVersionCompactionEnabled.Key)
|
|
params.Reset(params.DataCoordCfg.BumpSchemaVersionCompactionTriggerInterval.Key)
|
|
params.Reset(params.DataCoordCfg.BumpSchemaVersionCompactionSlotUsage.Key)
|
|
})
|
|
|
|
assert.False(t, params.DataCoordCfg.BumpSchemaVersionCompactionEnabled.GetAsBool())
|
|
assert.Equal(t, time.Second*20, params.DataCoordCfg.BumpSchemaVersionCompactionTriggerInterval.GetAsDuration(time.Second))
|
|
assert.EqualValues(t, 1, params.DataCoordCfg.BumpSchemaVersionCompactionSlotUsage.GetAsInt64())
|
|
|
|
params.Save(params.DataCoordCfg.BumpSchemaVersionCompactionSlotUsage.Key, "5")
|
|
assert.EqualValues(t, 5, params.DataCoordCfg.BumpSchemaVersionCompactionSlotUsage.GetAsInt64())
|
|
params.Save(params.DataCoordCfg.BumpSchemaVersionCompactionSlotUsage.Key, "0")
|
|
assert.EqualValues(t, 1, params.DataCoordCfg.BumpSchemaVersionCompactionSlotUsage.GetAsInt64())
|
|
params.Save(params.DataCoordCfg.BumpSchemaVersionCompactionSlotUsage.Key, "-1")
|
|
assert.EqualValues(t, 1, params.DataCoordCfg.BumpSchemaVersionCompactionSlotUsage.GetAsInt64())
|
|
}
|
|
|
|
func TestComponentParam_DataCoordSnapshotExportCopyConcurrency(t *testing.T) {
|
|
Init()
|
|
params := Get()
|
|
key := params.DataCoordCfg.SnapshotExportCopyConcurrency.Key
|
|
params.Reset(key)
|
|
t.Cleanup(func() { params.Reset(key) })
|
|
|
|
assert.Equal(t, 16, params.DataCoordCfg.SnapshotExportCopyConcurrency.GetAsInt())
|
|
|
|
params.Save(key, "3")
|
|
assert.Equal(t, 3, params.DataCoordCfg.SnapshotExportCopyConcurrency.GetAsInt())
|
|
|
|
for _, invalid := range []string{"0", "-1", "invalid"} {
|
|
params.Save(key, invalid)
|
|
assert.Equal(t, 16, params.DataCoordCfg.SnapshotExportCopyConcurrency.GetAsInt())
|
|
}
|
|
}
|
|
|
|
func TestComponentParam_DataCoordJSONStatsWriteBatchSize(t *testing.T) {
|
|
Init()
|
|
params := Get()
|
|
item := ¶ms.DataCoordCfg.JSONStatsWriteBatchSize
|
|
legacyKey := item.FallbackKeys[0]
|
|
params.Reset(item.Key)
|
|
params.Reset(legacyKey)
|
|
t.Cleanup(func() {
|
|
params.Reset(item.Key)
|
|
params.Reset(legacyKey)
|
|
})
|
|
|
|
assert.EqualValues(t, 81920, item.GetAsInt64())
|
|
|
|
params.Save(item.Key, "4096")
|
|
assert.EqualValues(t, 4096, item.GetAsInt64())
|
|
|
|
for _, invalid := range []string{"0", "-1", "invalid", "9223372036854775808"} {
|
|
params.Save(item.Key, invalid)
|
|
assert.EqualValues(t, 81920, item.GetAsInt64(), invalid)
|
|
}
|
|
|
|
params.Reset(item.Key)
|
|
params.Save(legacyKey, "-1")
|
|
assert.EqualValues(t, 81920, item.GetAsInt64())
|
|
}
|
|
|
|
func TestMembershipFilterConfig(t *testing.T) {
|
|
base := NewBaseTable(SkipRemote(true))
|
|
params := proxyConfig{}
|
|
params.init(base)
|
|
|
|
assert.Equal(t, 64*1024*1024, params.MaxMembershipFilterSize.GetAsInt())
|
|
base.Save(params.MaxMembershipFilterSize.Key, "1048576")
|
|
assert.Equal(t, 1048576, params.MaxMembershipFilterSize.GetAsInt())
|
|
base.Reset(params.MaxMembershipFilterSize.Key)
|
|
|
|
assert.Equal(t, int64(DefaultMaxMembershipFilterPlanSize), params.MaxMembershipFilterPlanSize.GetAsInt64())
|
|
base.Save(params.MaxMembershipFilterPlanSize.Key, "1048576")
|
|
assert.Equal(t, int64(1048576), params.MaxMembershipFilterPlanSize.GetAsInt64())
|
|
for _, invalid := range []string{"0", "-1", "invalid", "9223372036854775808"} {
|
|
base.Save(params.MaxMembershipFilterPlanSize.Key, invalid)
|
|
assert.Equal(t, int64(DefaultMaxMembershipFilterPlanSize), params.MaxMembershipFilterPlanSize.GetAsInt64(), invalid)
|
|
}
|
|
base.Reset(params.MaxMembershipFilterPlanSize.Key)
|
|
|
|
t.Run("legacy bloom filter keys remain fallbacks", func(t *testing.T) {
|
|
legacySizeKey := params.MaxMembershipFilterSize.FallbackKeys[0]
|
|
base.Save(legacySizeKey, "2097152")
|
|
base.Reset(params.MaxMembershipFilterSize.Key)
|
|
assert.Equal(t, 2*1024*1024, params.MaxMembershipFilterSize.GetAsInt())
|
|
base.Reset(legacySizeKey)
|
|
base.Reset(params.MaxMembershipFilterSize.Key)
|
|
|
|
legacyPlanKey := params.MaxMembershipFilterPlanSize.FallbackKeys[0]
|
|
base.Save(legacyPlanKey, "4194304")
|
|
base.Reset(params.MaxMembershipFilterPlanSize.Key)
|
|
assert.Equal(t, int64(4*1024*1024), params.MaxMembershipFilterPlanSize.GetAsInt64())
|
|
base.Reset(legacyPlanKey)
|
|
base.Reset(params.MaxMembershipFilterPlanSize.Key)
|
|
})
|
|
}
|
|
|
|
// TestMembershipFilterSizeFallbackKeys pins the upgrade path for deployments
|
|
// tuned under the pre-unification per-kind keys: with the new
|
|
// proxy.maxMembershipFilterSize unset, the first fallback key that is
|
|
// explicitly set (bloom-first order) supplies the value; an explicitly set new
|
|
// key always wins over any fallback.
|
|
func TestMembershipFilterSizeFallbackKeys(t *testing.T) {
|
|
Init()
|
|
params := Get()
|
|
|
|
t.Run("old bloom key feeds the unified param", func(t *testing.T) {
|
|
params.Save(params.ProxyCfg.MaxMembershipFilterSize.Key, "67108864")
|
|
defer params.Reset(params.ProxyCfg.MaxMembershipFilterSize.Key)
|
|
params.Save("proxy.maxBloomFilterSize", "1048576")
|
|
defer params.Reset("proxy.maxBloomFilterSize")
|
|
assert.Equal(t, int64(1048576), params.ProxyCfg.MaxMembershipFilterSize.GetAsInt64())
|
|
})
|
|
|
|
t.Run("explicit new key wins over fallbacks", func(t *testing.T) {
|
|
params.Save(params.ProxyCfg.MaxMembershipFilterSize.Key, "2097152")
|
|
defer params.Reset(params.ProxyCfg.MaxMembershipFilterSize.Key)
|
|
params.Save("proxy.maxBloomFilterSize", "1048576")
|
|
defer params.Reset("proxy.maxBloomFilterSize")
|
|
params.Save("proxy.maxRoaringFilterSize", "3145728")
|
|
defer params.Reset("proxy.maxRoaringFilterSize")
|
|
assert.Equal(t, int64(2097152), params.ProxyCfg.MaxMembershipFilterSize.GetAsInt64())
|
|
})
|
|
|
|
t.Run("roaring key is used when bloom key is absent", func(t *testing.T) {
|
|
params.Save(params.ProxyCfg.MaxMembershipFilterSize.Key, "67108864")
|
|
defer params.Reset(params.ProxyCfg.MaxMembershipFilterSize.Key)
|
|
params.Save("proxy.maxRoaringFilterSize", "4194304")
|
|
defer params.Reset("proxy.maxRoaringFilterSize")
|
|
assert.Equal(t, int64(4194304), params.ProxyCfg.MaxMembershipFilterSize.GetAsInt64())
|
|
})
|
|
|
|
t.Run("legacy plan key does not widen the per-blob limit", func(t *testing.T) {
|
|
params.Save(params.ProxyCfg.MaxMembershipFilterSize.Key, "67108864")
|
|
defer params.Reset(params.ProxyCfg.MaxMembershipFilterSize.Key)
|
|
params.Save("proxy.maxBloomFilterPlanSize", "134217728")
|
|
defer params.Reset("proxy.maxBloomFilterPlanSize")
|
|
assert.Equal(t, int64(64*1024*1024), params.ProxyCfg.MaxMembershipFilterSize.GetAsInt64())
|
|
assert.Equal(t, int64(128*1024*1024), params.ProxyCfg.MaxMembershipFilterPlanSize.GetAsInt64())
|
|
})
|
|
}
|
|
|
|
func TestComponentParam_StorageIopsParams(t *testing.T) {
|
|
params := &ComponentParam{}
|
|
params.Init(NewBaseTable(SkipRemote(true), SkipEnv(true)))
|
|
|
|
initialRate := ¶ms.CommonCfg.StorageIopsInitialRate
|
|
maxRate := ¶ms.CommonCfg.StorageIopsMaxRate
|
|
assert.Equal(t, "3.0.1", initialRate.Version)
|
|
assert.Equal(t, "3.0.1", maxRate.Version)
|
|
assert.Equal(t, DefaultStorageIopsInitialRate, initialRate.GetAsUint32())
|
|
assert.Equal(t, DefaultStorageIopsMaxRate, maxRate.GetAsUint32())
|
|
|
|
assert.NoError(t, params.Save(initialRate.Key, "3000"))
|
|
assert.NoError(t, params.Save(maxRate.Key, "0"))
|
|
assert.Equal(t, uint32(3000), initialRate.GetAsUint32())
|
|
assert.Equal(t, uint32(0), maxRate.GetAsUint32())
|
|
|
|
for _, invalid := range []string{"", "-1", "invalid", "4294967296"} {
|
|
assert.NoError(t, params.Save(initialRate.Key, invalid))
|
|
assert.Equal(t, DefaultStorageIopsInitialRate, initialRate.GetAsUint32())
|
|
assert.NoError(t, params.Save(maxRate.Key, invalid))
|
|
assert.Equal(t, DefaultStorageIopsMaxRate, maxRate.GetAsUint32())
|
|
}
|
|
}
|
|
|
|
func TestLoadAdmissionAsyncMemoryDefault(t *testing.T) {
|
|
pt := &ComponentParam{}
|
|
pt.Init(NewBaseTable(SkipRemote(true), SkipEnv(true), Files(nil)))
|
|
assert.EqualValues(t, 2*1024*1024*1024, pt.CommonCfg.LoadTransientBudgetBytes.GetAsInt64())
|
|
}
|
|
|
|
func TestResolveLoadAdmissionLimits(t *testing.T) {
|
|
pt := &ComponentParam{}
|
|
pt.Init(NewBaseTable(SkipRemote(true), SkipEnv(true), Files(nil)))
|
|
cpuSlots := int64(DefaultLoadAdmissionSlotsPerCPU * hardware.GetCPUNum())
|
|
for _, enabled := range []bool{false, true} {
|
|
for _, memory := range []string{"", "0", "67108864", "2147483648"} {
|
|
for _, slots := range []string{"", "0", "7", fmt.Sprint(cpuSlots)} {
|
|
t.Run(fmt.Sprintf("enabled=%v/memory=%s/slots=%s", enabled, memory, slots), func(t *testing.T) {
|
|
expected := func(item *ParamItem, value string, asyncDefault int64) int64 {
|
|
if value != "" {
|
|
require.NoError(t, pt.Save(item.Key, value))
|
|
return getAsInt64(value)
|
|
}
|
|
require.NoError(t, pt.Reset(item.Key))
|
|
if enabled {
|
|
return asyncDefault
|
|
}
|
|
return 0
|
|
}
|
|
wantMemory := expected(&pt.CommonCfg.LoadTransientBudgetBytes, memory, 2*1024*1024*1024)
|
|
wantSlots := expected(&pt.CommonCfg.LoadAdmissionSlots, slots, cpuSlots)
|
|
gotMemory, gotSlots := pt.CommonCfg.ResolveLoadAdmissionLimits(enabled)
|
|
assert.Equal(t, wantMemory, gotMemory)
|
|
assert.Equal(t, wantSlots, gotSlots)
|
|
})
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestResolveLoadAdmissionLimitsInvalidConfiguration(t *testing.T) {
|
|
pt := &ComponentParam{}
|
|
pt.Init(NewBaseTable(SkipRemote(true), SkipEnv(true), Files(nil)))
|
|
for _, enabled := range []bool{false, true} {
|
|
for _, invalid := range []string{"typo", "", "1.5", "9223372036854775808", "-1"} {
|
|
t.Run(fmt.Sprintf("enabled=%v/value=%q", enabled, invalid), func(t *testing.T) {
|
|
require.NoError(t, pt.Save(pt.CommonCfg.LoadTransientBudgetBytes.Key, invalid))
|
|
require.NoError(t, pt.Save(pt.CommonCfg.LoadAdmissionSlots.Key, invalid))
|
|
memory, slots := pt.CommonCfg.ResolveLoadAdmissionLimits(enabled)
|
|
assert.EqualValues(t, DefaultLoadTransientBudgetBytes, memory)
|
|
assert.EqualValues(t, DefaultLoadAdmissionSlotsPerCPU*hardware.GetCPUNum(), slots)
|
|
})
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestResolveLoadAdmissionLimitsPreservesConfiguredSources(t *testing.T) {
|
|
dir := t.TempDir()
|
|
t.Setenv("MILVUSCONF", dir)
|
|
file := filepath.Join(dir, "load-admission.yaml")
|
|
require.NoError(t, os.WriteFile(file, []byte("common:\n loadTransientBudgetBytes: 0\n loadAdmissionSlots: 9\n"), 0o600))
|
|
pt := &ComponentParam{}
|
|
pt.Init(NewBaseTable(SkipRemote(true), SkipEnv(true), Files([]string{filepath.Base(file)})))
|
|
for _, enabled := range []bool{false, true} {
|
|
memory, slots := pt.CommonCfg.ResolveLoadAdmissionLimits(enabled)
|
|
assert.Zero(t, memory, "file-sourced explicit zero must not select the async default")
|
|
assert.EqualValues(t, 9, slots)
|
|
}
|
|
require.NoError(t, pt.Save(pt.CommonCfg.LoadTransientBudgetBytes.Key, "1234"))
|
|
memory, _ := pt.CommonCfg.ResolveLoadAdmissionLimits(false)
|
|
assert.EqualValues(t, 1234, memory)
|
|
require.NoError(t, pt.Reset(pt.CommonCfg.LoadTransientBudgetBytes.Key))
|
|
memory, _ = pt.CommonCfg.ResolveLoadAdmissionLimits(true)
|
|
assert.Zero(t, memory, "reset must reveal the file's explicit zero")
|
|
|
|
t.Setenv("MILVUS_CONF_COMMON_LOADTRANSIENTBUDGETBYTES", "4096")
|
|
t.Setenv("MILVUS_CONF_COMMON_LOADADMISSIONSLOTS", "0")
|
|
envPt := &ComponentParam{}
|
|
envPt.Init(NewBaseTable(SkipRemote(true), Files(nil)))
|
|
for _, enabled := range []bool{false, true} {
|
|
memory, slots := envPt.CommonCfg.ResolveLoadAdmissionLimits(enabled)
|
|
assert.EqualValues(t, 4096, memory)
|
|
assert.Zero(t, slots, "environment-sourced explicit zero must remain unlimited")
|
|
}
|
|
}
|
|
|
|
func TestStorageV2AsyncLoadThreadPoolSize(t *testing.T) {
|
|
pt := &ComponentParam{}
|
|
pt.Init(NewBaseTable(SkipRemote(true), SkipEnv(true), Files(nil)))
|
|
item := &pt.QueryNodeCfg.StorageV2AsyncLoadThreadPoolSize
|
|
wantDefault := max(1, min(hardware.GetCPUNum(), 16))
|
|
assert.False(t, item.Export)
|
|
assert.Equal(t, wantDefault, item.GetAsInt())
|
|
for _, invalid := range []string{"0", "-1", "typo", "", "1.5", "2147483648"} {
|
|
require.NoError(t, pt.Save(item.Key, invalid))
|
|
assert.Equal(t, wantDefault, item.GetAsInt(), invalid)
|
|
}
|
|
require.NoError(t, pt.Save(item.Key, "3"))
|
|
assert.Equal(t, 3, item.GetAsInt())
|
|
require.NoError(t, pt.Remove(item.Key))
|
|
assert.Equal(t, wantDefault, item.GetAsInt())
|
|
}
|
|
|
|
func TestComponentParam_LazyColumnGroupEnabled(t *testing.T) {
|
|
Init()
|
|
params := Get()
|
|
item := ¶ms.QueryNodeCfg.TieredLazyColumnGroupEnabled
|
|
t.Cleanup(func() { params.Reset(item.Key) })
|
|
|
|
assert.Equal(t, "queryNode.segcore.tieredStorage.lazyColumnGroupEnabled", item.Key)
|
|
assert.Equal(t, "false", item.DefaultValue)
|
|
assert.True(t, item.Export)
|
|
assert.False(t, item.GetAsBool())
|
|
|
|
assert.NoError(t, params.Save(item.Key, "true"))
|
|
assert.True(t, item.GetAsBool())
|
|
assert.NoError(t, params.Save(item.Key, "false"))
|
|
assert.False(t, item.GetAsBool())
|
|
}
|
|
|
|
func TestComponentParam(t *testing.T) {
|
|
Init()
|
|
params := Get()
|
|
|
|
t.Run("query node zero copy config key", func(t *testing.T) {
|
|
assert.Equal(t, "queryNode.search.enableResultZeroCopy", params.QueryNodeCfg.EnableResultZeroCopy.Key)
|
|
})
|
|
|
|
t.Run("query node mmap writeback config", func(t *testing.T) {
|
|
item := ¶ms.QueryNodeCfg.MmapWriteback
|
|
t.Cleanup(func() {
|
|
params.Reset(item.Key)
|
|
})
|
|
|
|
assert.Equal(t, "queryNode.mmap.writeback", item.Key)
|
|
assert.False(t, item.Export)
|
|
assert.False(t, item.GetAsBool())
|
|
|
|
params.Save(item.Key, "true")
|
|
assert.True(t, item.GetAsBool())
|
|
})
|
|
|
|
t.Run("query node storage v2 async load config", func(t *testing.T) {
|
|
item := ¶ms.QueryNodeCfg.StorageV2EnableAsyncLoad
|
|
t.Cleanup(func() {
|
|
params.Reset(item.Key)
|
|
})
|
|
|
|
assert.Equal(t, "queryNode.segcore.storageV2.enableAsyncLoad", item.Key)
|
|
assert.False(t, item.Export)
|
|
assert.False(t, item.GetAsBool())
|
|
|
|
params.Save(item.Key, "true")
|
|
assert.True(t, item.GetAsBool())
|
|
})
|
|
|
|
t.Run("query node storage v2 async read window config", func(t *testing.T) {
|
|
item := ¶ms.QueryNodeCfg.StorageV2AsyncLoadReadWindowSizeBytes
|
|
t.Cleanup(func() {
|
|
params.Reset(item.Key)
|
|
})
|
|
|
|
assert.Equal(t, "queryNode.segcore.storageV2.asyncLoadReadWindowSizeBytes", item.Key)
|
|
assert.True(t, item.Export)
|
|
assert.EqualValues(t, 16*1024*1024, item.GetAsInt64())
|
|
|
|
params.Save(item.Key, "0")
|
|
assert.EqualValues(t, 16*1024*1024, item.GetAsInt64())
|
|
params.Save(item.Key, "-1")
|
|
assert.EqualValues(t, 16*1024*1024, item.GetAsInt64())
|
|
params.Save(item.Key, "16MiB")
|
|
assert.EqualValues(t, 16*1024*1024, item.GetAsInt64())
|
|
params.Save(item.Key, "9223372036854775808")
|
|
assert.EqualValues(t, 16*1024*1024, item.GetAsInt64())
|
|
params.Save(item.Key, "33554432")
|
|
assert.EqualValues(t, 32*1024*1024, item.GetAsInt64())
|
|
})
|
|
|
|
t.Run("test commonConfig", func(t *testing.T) {
|
|
Params := ¶ms.CommonCfg
|
|
|
|
assert.NotEqual(t, Params.DefaultPartitionName.GetValue(), "")
|
|
t.Logf("default partition name = %s", Params.DefaultPartitionName.GetValue())
|
|
|
|
assert.NotEqual(t, Params.DefaultIndexName.GetValue(), "")
|
|
t.Logf("default index name = %s", Params.DefaultIndexName.GetValue())
|
|
|
|
assert.NotEqual(t, Params.SimdType.GetValue(), "")
|
|
t.Logf("knowhere simd type = %s", Params.SimdType.GetValue())
|
|
|
|
assert.Equal(t, Params.IndexSliceSize.GetAsInt64(), int64(DefaultIndexSliceSize))
|
|
t.Logf("knowhere index slice size = %d", Params.IndexSliceSize.GetAsInt64())
|
|
|
|
assert.False(t, Params.LoadTransientBudgetBytes.Export)
|
|
defer params.Reset(Params.LoadTransientBudgetBytes.Key)
|
|
assert.Equal(t, int64(DefaultLoadTransientBudgetBytes), Params.LoadTransientBudgetBytes.GetAsInt64())
|
|
params.Save(Params.LoadTransientBudgetBytes.Key, "-1")
|
|
assert.Equal(t, int64(DefaultLoadTransientBudgetBytes), Params.LoadTransientBudgetBytes.GetAsInt64())
|
|
params.Save(Params.LoadTransientBudgetBytes.Key, "67108864")
|
|
assert.Equal(t, int64(67108864), Params.LoadTransientBudgetBytes.GetAsInt64())
|
|
|
|
assert.False(t, Params.LoadAdmissionSlots.Export)
|
|
defer params.Reset(Params.LoadAdmissionSlots.Key)
|
|
assert.Equal(t, int64(2*hardware.GetCPUNum()), Params.LoadAdmissionSlots.GetAsInt64())
|
|
params.Save(Params.LoadAdmissionSlots.Key, "-1")
|
|
assert.Equal(t, int64(2*hardware.GetCPUNum()), Params.LoadAdmissionSlots.GetAsInt64())
|
|
params.Save(Params.LoadAdmissionSlots.Key, "16")
|
|
assert.Equal(t, int64(16), Params.LoadAdmissionSlots.GetAsInt64())
|
|
params.Save(Params.LoadAdmissionSlots.Key, "0")
|
|
assert.Equal(t, int64(0), Params.LoadAdmissionSlots.GetAsInt64())
|
|
|
|
assert.Equal(t, int64(0), Params.ArrowReaderHoleSizeLimitBytes.GetAsInt64())
|
|
assert.Equal(t, int64(0), Params.ArrowReaderRangeSizeLimitBytes.GetAsInt64())
|
|
params.Save(Params.ArrowReaderHoleSizeLimitBytes.Key, "1048576")
|
|
params.Save(Params.ArrowReaderRangeSizeLimitBytes.Key, "67108864")
|
|
assert.Equal(t, int64(1048576), Params.ArrowReaderHoleSizeLimitBytes.GetAsInt64())
|
|
assert.Equal(t, int64(67108864), Params.ArrowReaderRangeSizeLimitBytes.GetAsInt64())
|
|
|
|
assert.Equal(t, int32(0), Params.StorageReaderThreadPoolSize.GetAsInt32())
|
|
assert.Equal(t, int64(0), Params.IndexBuildReadWindowBytes.GetAsInt64())
|
|
params.Save(Params.StorageReaderThreadPoolSize.Key, "16")
|
|
params.Save(Params.IndexBuildReadWindowBytes.Key, "536870912")
|
|
assert.Equal(t, int32(16), Params.StorageReaderThreadPoolSize.GetAsInt32())
|
|
assert.Equal(t, int64(536870912), Params.IndexBuildReadWindowBytes.GetAsInt64())
|
|
|
|
assert.False(t, Params.UseArrowFSChunkManager.GetAsBool())
|
|
params.Save(Params.UseArrowFSChunkManager.Key, "true")
|
|
assert.True(t, Params.UseArrowFSChunkManager.GetAsBool())
|
|
params.Reset(Params.UseArrowFSChunkManager.Key)
|
|
|
|
defer params.Reset(Params.ExternalVectorPartialNullPolicy.Key)
|
|
assert.Equal(t, "error", Params.ExternalVectorPartialNullPolicy.GetValue())
|
|
params.Save(Params.ExternalVectorPartialNullPolicy.Key, "null")
|
|
assert.Equal(t, "null", Params.ExternalVectorPartialNullPolicy.GetValue())
|
|
|
|
assert.Equal(t, Params.GracefulTime.GetAsInt64(), int64(DefaultGracefulTime))
|
|
t.Logf("default grafeful time = %d", Params.GracefulTime.GetAsInt64())
|
|
|
|
assert.Equal(t, Params.GracefulStopTimeout.GetAsInt64(), int64(DefaultGracefulStopTimeout))
|
|
assert.Equal(t, params.QueryNodeCfg.GracefulStopTimeout.GetAsInt64(), Params.GracefulStopTimeout.GetAsInt64())
|
|
assert.Equal(t, params.DataNodeCfg.GracefulStopTimeout.GetAsInt64(), Params.GracefulStopTimeout.GetAsInt64())
|
|
t.Logf("default grafeful stop timeout = %d", Params.GracefulStopTimeout.GetAsInt())
|
|
params.Save(Params.GracefulStopTimeout.Key, "50")
|
|
assert.Equal(t, Params.GracefulStopTimeout.GetAsInt64(), int64(50))
|
|
|
|
// -- rootcoord --
|
|
assert.Equal(t, Params.RootCoordTimeTick.GetValue(), "by-dev-rootcoord-timetick")
|
|
t.Logf("rootcoord timetick channel = %s", Params.RootCoordTimeTick.GetValue())
|
|
|
|
assert.Equal(t, Params.RootCoordStatistics.GetValue(), "by-dev-rootcoord-statistics")
|
|
t.Logf("rootcoord statistics channel = %s", Params.RootCoordStatistics.GetValue())
|
|
|
|
assert.Equal(t, Params.RootCoordDml.GetValue(), "by-dev-rootcoord-dml")
|
|
t.Logf("rootcoord dml channel = %s", Params.RootCoordDml.GetValue())
|
|
|
|
// -- querycoord --
|
|
assert.Equal(t, Params.QueryCoordTimeTick.GetValue(), "by-dev-queryTimeTick")
|
|
t.Logf("querycoord timetick channel = %s", Params.QueryCoordTimeTick.GetValue())
|
|
|
|
// -- datacoord --
|
|
assert.Equal(t, Params.DataCoordTimeTick.GetValue(), "by-dev-datacoord-timetick-channel")
|
|
t.Logf("datacoord timetick channel = %s", Params.DataCoordTimeTick.GetValue())
|
|
|
|
assert.Equal(t, Params.DataCoordSegmentInfo.GetValue(), "by-dev-segment-info-channel")
|
|
t.Logf("datacoord segment info channel = %s", Params.DataCoordSegmentInfo.GetValue())
|
|
|
|
assert.Equal(t, Params.DataCoordSubName.GetValue(), "by-dev-dataCoord")
|
|
t.Logf("datacoord subname = %s", Params.DataCoordSubName.GetValue())
|
|
|
|
assert.Equal(t, Params.DataNodeSubName.GetValue(), "by-dev-dataNode")
|
|
t.Logf("datanode subname = %s", Params.DataNodeSubName.GetValue())
|
|
|
|
assert.Equal(t, Params.SessionTTL.GetAsInt64(), int64(DefaultSessionTTL))
|
|
t.Logf("default session TTL time = %d", Params.SessionTTL.GetAsInt64())
|
|
assert.Equal(t, Params.SessionRetryTimes.GetAsInt64(), int64(DefaultSessionRetryTimes))
|
|
t.Logf("default session retry times = %d", Params.SessionRetryTimes.GetAsInt64())
|
|
|
|
params.Save("common.security.superUsers", "super1,super2,super3")
|
|
assert.Equal(t, []string{"super1", "super2", "super3"}, Params.SuperUsers.GetAsStrings())
|
|
|
|
assert.Equal(t, "Milvus", Params.DefaultRootPassword.GetValue())
|
|
params.Save("common.security.defaultRootPassword", "defaultMilvus")
|
|
assert.Equal(t, "defaultMilvus", Params.DefaultRootPassword.GetValue())
|
|
|
|
params.Save("common.security.superUsers", "")
|
|
assert.Equal(t, []string{}, Params.SuperUsers.GetAsStrings())
|
|
|
|
assert.Equal(t, false, Params.PreCreatedTopicEnabled.GetAsBool())
|
|
|
|
params.Save("common.preCreatedTopic.names", "topic1,topic2,topic3")
|
|
assert.Equal(t, []string{"topic1", "topic2", "topic3"}, Params.TopicNames.GetAsStrings())
|
|
|
|
params.Save("common.preCreatedTopic.timeticker", "timeticker")
|
|
assert.Equal(t, []string{"timeticker"}, Params.TimeTicker.GetAsStrings())
|
|
|
|
assert.True(t, params.CommonCfg.BloomFilterEnabled.GetAsBool())
|
|
params.Save("common.bloomFilterEnabled", "false")
|
|
assert.False(t, params.CommonCfg.BloomFilterEnabled.GetAsBool())
|
|
params.Reset("common.bloomFilterEnabled")
|
|
assert.True(t, params.CommonCfg.BloomFilterEnabled.GetAsBool())
|
|
|
|
assert.Equal(t, 1000, params.CommonCfg.BloomFilterApplyBatchSize.GetAsInt())
|
|
|
|
params.Save("common.gcenabled", "false")
|
|
assert.False(t, Params.GCEnabled.GetAsBool())
|
|
params.Save("common.gchelper.enabled", "false")
|
|
assert.False(t, Params.GCHelperEnabled.GetAsBool())
|
|
params.Save("common.overloadedMemoryThresholdPercentage", "40")
|
|
assert.Equal(t, 0.4, Params.OverloadedMemoryThresholdPercentage.GetAsFloat())
|
|
params.Save("common.gchelper.maximumGoGC", "100")
|
|
assert.Equal(t, 100, Params.MaximumGOGCConfig.GetAsInt())
|
|
params.Save("common.gchelper.minimumGoGC", "80")
|
|
assert.Equal(t, 80, Params.MinimumGOGCConfig.GetAsInt())
|
|
|
|
assert.Equal(t, 0, len(Params.ReadOnlyPrivileges.GetAsStrings()))
|
|
assert.Equal(t, 0, len(Params.ReadWritePrivileges.GetAsStrings()))
|
|
assert.Equal(t, 0, len(Params.AdminPrivileges.GetAsStrings()))
|
|
|
|
assert.False(t, params.CommonCfg.LocalRPCEnabled.GetAsBool())
|
|
params.Save("common.localRPCEnabled", "true")
|
|
assert.True(t, params.CommonCfg.LocalRPCEnabled.GetAsBool())
|
|
|
|
assert.Equal(t, 60*time.Second, params.CommonCfg.SyncTaskPoolReleaseTimeoutSeconds.GetAsDuration(time.Second))
|
|
params.Save("common.sync.taskPoolReleaseTimeoutSeconds", "100")
|
|
assert.Equal(t, 100*time.Second, params.CommonCfg.SyncTaskPoolReleaseTimeoutSeconds.GetAsDuration(time.Second))
|
|
|
|
assert.Equal(t, 2.0, Params.NodeSchedulerMaxConcurrencyRatio.GetAsFloat())
|
|
params.Save(Params.NodeSchedulerMaxConcurrencyRatio.Key, "0.5")
|
|
assert.Equal(t, 0.5, Params.NodeSchedulerMaxConcurrencyRatio.GetAsFloat())
|
|
params.Reset(Params.NodeSchedulerMaxConcurrencyRatio.Key)
|
|
assert.Equal(t, 2.0, Params.NodeSchedulerMaxConcurrencyRatio.GetAsFloat())
|
|
|
|
assert.Equal(t, 1, params.CommonCfg.StorageZstdConcurrency.GetAsInt())
|
|
params.Save("common.storage.zstd.concurrency", "2")
|
|
assert.Equal(t, 2, params.CommonCfg.StorageZstdConcurrency.GetAsInt())
|
|
|
|
assert.Equal(t, "close", params.CommonCfg.ProxyFileResourceMode.GetValue())
|
|
params.Save("common.fileResource.mode.proxy", "sync")
|
|
assert.Equal(t, "sync", params.CommonCfg.ProxyFileResourceMode.GetValue())
|
|
assert.Equal(t, int64(0), params.CommonCfg.FileResourceMaxFileSize.GetAsSize())
|
|
params.Save("common.fileResource.maxFileSize", "2g")
|
|
assert.Equal(t, int64(2*1024*1024*1024), params.CommonCfg.FileResourceMaxFileSize.GetAsSize())
|
|
params.Save("common.fileResource.maxFileSize", "-1")
|
|
assert.Equal(t, int64(0), params.CommonCfg.FileResourceMaxFileSize.GetAsSize())
|
|
assert.Equal(t, 5*time.Minute, params.CommonCfg.FileResourceDownloadTimeout.GetAsDurationByParse())
|
|
params.Save("common.fileResource.downloadTimeout", "30s")
|
|
assert.Equal(t, 30*time.Second, params.CommonCfg.FileResourceDownloadTimeout.GetAsDurationByParse())
|
|
params.Save("common.fileResource.downloadTimeout", "invalid")
|
|
assert.Equal(t, 5*time.Minute, params.CommonCfg.FileResourceDownloadTimeout.GetAsDurationByParse())
|
|
|
|
assert.Equal(t, 0, params.CommonCfg.ClusterID.GetAsInt())
|
|
params.Save("common.clusterID", "32")
|
|
assert.Panics(t, func() {
|
|
params.CommonCfg.ClusterID.GetAsInt()
|
|
})
|
|
params.Save("common.clusterID", "0")
|
|
})
|
|
|
|
t.Run("test logConfig", func(t *testing.T) {
|
|
Params := ¶ms.LogCfg
|
|
assert.True(t, Params.AsyncWriteEnable.GetAsBool())
|
|
assert.Equal(t, 10*time.Second, Params.AsyncWriteFlushInterval.GetAsDurationByParse())
|
|
assert.Equal(t, 100*time.Millisecond, Params.AsyncWriteDroppedTimeout.GetAsDurationByParse())
|
|
assert.Equal(t, "error", Params.AsyncWriteNonDroppableLevel.GetValue())
|
|
assert.Equal(t, 1*time.Second, Params.AsyncWriteStopTimeout.GetAsDurationByParse())
|
|
assert.Equal(t, 1024, Params.AsyncWritePendingLength.GetAsInt())
|
|
assert.Equal(t, int64(4*1024), Params.AsyncWriteBufferSize.GetAsSize())
|
|
assert.Equal(t, int64(1024*1024), Params.AsyncWriteMaxBytesPerLog.GetAsSize())
|
|
})
|
|
|
|
t.Run("test rootCoordConfig", func(t *testing.T) {
|
|
Params := ¶ms.RootCoordCfg
|
|
|
|
assert.NotEqual(t, Params.MaxPartitionNum.GetAsInt64(), 0)
|
|
t.Logf("master MaxPartitionNum = %d", Params.MaxPartitionNum.GetAsInt64())
|
|
assert.NotEqual(t, Params.MinSegmentSizeToEnableIndex.GetAsInt64(), 0)
|
|
t.Logf("master MinSegmentSizeToEnableIndex = %d", Params.MinSegmentSizeToEnableIndex.GetAsInt64())
|
|
assert.Equal(t, Params.EnableActiveStandby.GetAsBool(), false)
|
|
t.Logf("rootCoord EnableActiveStandby = %t", Params.EnableActiveStandby.GetAsBool())
|
|
|
|
params.Save("rootCoord.gracefulStopTimeout", "100")
|
|
assert.Equal(t, 100*time.Second, Params.GracefulStopTimeout.GetAsDuration(time.Second))
|
|
|
|
assert.Equal(t, "{}", Params.DefaultDBProperties.GetValue())
|
|
params.Save("rootCoord.defaultDBProperties", "{\"key\":\"value\"}")
|
|
assert.Equal(t, "{\"key\":\"value\"}", Params.DefaultDBProperties.GetValue())
|
|
|
|
// Client telemetry. The defaults are the contract the telemetry manager was written
|
|
// against, so they are asserted exactly rather than for mere presence.
|
|
assert.Equal(t, 2, Params.ClientTelemetryRetainedWindows.GetAsInt())
|
|
assert.Equal(t, time.Minute, Params.ClientTelemetryCleanupInterval.GetAsDuration(time.Second))
|
|
assert.Equal(t, 10*time.Minute, Params.ClientTelemetryInactiveClientThreshold.GetAsDuration(time.Second))
|
|
assert.Equal(t, time.Minute, Params.ClientTelemetryClientStatusThreshold.GetAsDuration(time.Second))
|
|
assert.Equal(t, 10*time.Second, Params.ClientTelemetryCommandCleanupTimeout.GetAsDuration(time.Second))
|
|
assert.Equal(t, 1024*1024, Params.ClientTelemetryMaxMetricsPerClient.GetAsInt())
|
|
assert.Equal(t, 100, Params.ClientTelemetryMaxOperationTypesPerClient.GetAsInt())
|
|
assert.Equal(t, 100000, Params.ClientTelemetryMaxClientsInMemory.GetAsInt())
|
|
|
|
params.Save("rootCoord.clientTelemetry.retainedWindows", "3")
|
|
assert.Equal(t, 3, Params.ClientTelemetryRetainedWindows.GetAsInt())
|
|
|
|
SetCreateTime(time.Now())
|
|
SetUpdateTime(time.Now())
|
|
})
|
|
|
|
t.Run("test proxyConfig", func(t *testing.T) {
|
|
Params := ¶ms.ProxyCfg
|
|
|
|
assert.Equal(t, "proxy.splitChunk", Params.SplitChunkProxy.Key)
|
|
assert.True(t, Params.SplitChunkProxy.GetAsBool())
|
|
params.Save(Params.SplitChunkProxy.Key, "false")
|
|
assert.False(t, Params.SplitChunkProxy.GetAsBool())
|
|
params.Reset(Params.SplitChunkProxy.Key)
|
|
assert.True(t, Params.SplitChunkProxy.GetAsBool())
|
|
|
|
t.Logf("TimeTickInterval: %v", &Params.TimeTickInterval)
|
|
|
|
t.Logf("healthCheckTimeout: %v", &Params.HealthCheckTimeout)
|
|
|
|
t.Logf("MsgStreamTimeTickBufSize: %d", Params.MsgStreamTimeTickBufSize.GetAsInt64())
|
|
|
|
t.Logf("MaxNameLength: %d", Params.MaxNameLength.GetAsInt64())
|
|
assert.Equal(t, 1024, Params.MaxUserDescriptionLength.GetAsInt())
|
|
|
|
t.Logf("MaxFieldNum: %d", Params.MaxFieldNum.GetAsInt64())
|
|
assert.Equal(t, int64(256), Params.MaxFieldNum.GetAsInt64())
|
|
|
|
t.Logf("MaxVectorFieldNum: %d", Params.MaxVectorFieldNum.GetAsInt64())
|
|
|
|
t.Logf("MaxShardNum: %d", Params.MaxShardNum.GetAsInt64())
|
|
|
|
assert.Equal(t, int64(DefaultMaxMembershipFilterPlanSize), Params.MaxMembershipFilterPlanSize.GetAsInt64())
|
|
params.Save(Params.MaxMembershipFilterPlanSize.Key, "1048576")
|
|
assert.Equal(t, int64(1048576), Params.MaxMembershipFilterPlanSize.GetAsInt64())
|
|
for _, invalid := range []string{"0", "-1", "invalid", "9223372036854775808"} {
|
|
params.Save(Params.MaxMembershipFilterPlanSize.Key, invalid)
|
|
assert.Equal(t, int64(DefaultMaxMembershipFilterPlanSize), Params.MaxMembershipFilterPlanSize.GetAsInt64(), invalid)
|
|
}
|
|
params.Reset(Params.MaxMembershipFilterPlanSize.Key)
|
|
|
|
t.Logf("MaxDimension: %d", Params.MaxDimension.GetAsInt64())
|
|
|
|
t.Logf("MaxTaskNum: %d", Params.MaxTaskNum.GetAsInt64())
|
|
assert.Equal(t, int64(1024), Params.MaxTaskNum.GetAsInt64())
|
|
|
|
t.Logf("AccessLog.Enable: %t", Params.AccessLog.Enable.GetAsBool())
|
|
|
|
t.Logf("AccessLog.MaxSize: %d", Params.AccessLog.MaxSize.GetAsInt64())
|
|
|
|
t.Logf("AccessLog.MaxBackups: %d", Params.AccessLog.MaxBackups.GetAsInt64())
|
|
|
|
t.Logf("AccessLog.MaxDays: %d", Params.AccessLog.RotatedTime.GetAsInt64())
|
|
|
|
t.Logf("ShardLeaderCacheInterval: %d", Params.ShardLeaderCacheInterval.GetAsInt64())
|
|
|
|
assert.Equal(t, Params.ReplicaSelectionPolicy.GetValue(), "look_aside")
|
|
params.Save(Params.ReplicaSelectionPolicy.Key, "round_robin")
|
|
assert.Equal(t, Params.ReplicaSelectionPolicy.GetValue(), "round_robin")
|
|
params.Save(Params.ReplicaSelectionPolicy.Key, "look_aside")
|
|
assert.Equal(t, Params.ReplicaSelectionPolicy.GetValue(), "look_aside")
|
|
assert.Equal(t, Params.CheckQueryNodeHealthInterval.GetAsInt(), 1000)
|
|
assert.Equal(t, Params.CostMetricsExpireTime.GetAsInt(), 1000)
|
|
assert.Equal(t, Params.RetryTimesOnReplica.GetAsInt(), 5)
|
|
assert.EqualValues(t, Params.HealthCheckTimeout.GetAsInt64(), 3000)
|
|
|
|
// Test ReplicaBlacklistDuration default value
|
|
assert.Equal(t, 30*time.Second, Params.ReplicaBlacklistDuration.GetAsDurationByParse())
|
|
params.Save("proxy.replicaBlacklistDuration", "60s")
|
|
assert.Equal(t, 60*time.Second, Params.ReplicaBlacklistDuration.GetAsDurationByParse())
|
|
|
|
// Test ReplicaBlacklistCleanupInterval default value
|
|
assert.Equal(t, 10*time.Second, Params.ReplicaBlacklistCleanupInterval.GetAsDurationByParse())
|
|
params.Save("proxy.replicaBlacklistCleanupInterval", "30s")
|
|
assert.Equal(t, 30*time.Second, Params.ReplicaBlacklistCleanupInterval.GetAsDurationByParse())
|
|
|
|
params.Save("proxy.gracefulStopTimeout", "100")
|
|
assert.Equal(t, 100*time.Second, Params.GracefulStopTimeout.GetAsDuration(time.Second))
|
|
|
|
assert.False(t, Params.MustUsePartitionKey.GetAsBool())
|
|
params.Save("proxy.mustUsePartitionKey", "true")
|
|
assert.True(t, Params.MustUsePartitionKey.GetAsBool())
|
|
|
|
assert.False(t, Params.SkipAutoIDCheck.GetAsBool())
|
|
params.Save("proxy.skipAutoIDCheck", "true")
|
|
assert.True(t, Params.SkipAutoIDCheck.GetAsBool())
|
|
|
|
assert.False(t, Params.SkipPartitionKeyCheck.GetAsBool())
|
|
params.Save("proxy.skipPartitionKeyCheck", "true")
|
|
assert.True(t, Params.SkipPartitionKeyCheck.GetAsBool())
|
|
|
|
assert.Equal(t, int64(10), Params.CheckWorkloadRequestNum.GetAsInt64())
|
|
assert.Equal(t, float64(0.1), Params.WorkloadToleranceFactor.GetAsFloat())
|
|
assert.Equal(t, int64(10000), Params.MaxSearchAggregationResultEntries.GetAsInt64())
|
|
params.Save(Params.MaxSearchAggregationResultEntries.Key, "1024")
|
|
assert.Equal(t, int64(1024), Params.MaxSearchAggregationResultEntries.GetAsInt64())
|
|
params.Reset(Params.MaxSearchAggregationResultEntries.Key)
|
|
assert.Equal(t, int64(10000), Params.MaxSearchAggregationResultEntries.GetAsInt64())
|
|
|
|
assert.Equal(t, 100, Params.RLSMaxPoliciesPerCollection.GetAsInt())
|
|
assert.Equal(t, 1000, Params.RLSMaxPrincipalsPerCollection.GetAsInt())
|
|
assert.Equal(t, 50, Params.RLSMaxTagsPerPrincipal.GetAsInt())
|
|
assert.Equal(t, 4096, Params.RLSMaxExpressionLength.GetAsInt())
|
|
assert.Equal(t, 16384, Params.RLSMaxCombinedExpressionLength.GetAsInt())
|
|
assert.Equal(t, 255, Params.RLSMaxPolicyNameLength.GetAsInt())
|
|
assert.Equal(t, 1024, Params.RLSMaxPolicyDescriptionLength.GetAsInt())
|
|
assert.Equal(t, 255, Params.RLSMaxPrincipalNameLength.GetAsInt())
|
|
assert.Equal(t, 128, Params.RLSMaxTagKeyLength.GetAsInt())
|
|
assert.Equal(t, 1024, Params.RLSMaxTagValueLength.GetAsInt())
|
|
assert.Equal(t, 1024, Params.RLSMaxArrayLiteralElements.GetAsInt())
|
|
assert.Equal(t, time.Hour, Params.RLSMetaRefreshInterval.GetAsDuration(time.Second))
|
|
params.Save(Params.RLSMaxPoliciesPerCollection.Key, "2")
|
|
assert.Equal(t, 2, Params.RLSMaxPoliciesPerCollection.GetAsInt())
|
|
params.Save(Params.RLSMaxPoliciesPerCollection.Key, "0")
|
|
assert.Equal(t, 100, Params.RLSMaxPoliciesPerCollection.GetAsInt())
|
|
params.Reset(Params.RLSMaxPoliciesPerCollection.Key)
|
|
params.Save(Params.RLSMaxPrincipalsPerCollection.Key, "2")
|
|
assert.Equal(t, 2, Params.RLSMaxPrincipalsPerCollection.GetAsInt())
|
|
params.Save(Params.RLSMaxPrincipalsPerCollection.Key, "0")
|
|
assert.Equal(t, 1000, Params.RLSMaxPrincipalsPerCollection.GetAsInt())
|
|
params.Reset(Params.RLSMaxPrincipalsPerCollection.Key)
|
|
params.Save(Params.RLSMetaRefreshInterval.Key, "60")
|
|
assert.Equal(t, time.Minute, Params.RLSMetaRefreshInterval.GetAsDuration(time.Second))
|
|
params.Reset(Params.RLSMetaRefreshInterval.Key)
|
|
|
|
assert.Equal(t, int64(16), Params.DDLConcurrency.GetAsInt64())
|
|
assert.Equal(t, int64(16), Params.DCLConcurrency.GetAsInt64())
|
|
|
|
assert.Equal(t, 72, Params.MaxPasswordLength.GetAsInt())
|
|
params.Save("proxy.maxPasswordLength", "100")
|
|
assert.Equal(t, 72, Params.MaxPasswordLength.GetAsInt())
|
|
params.Save("proxy.maxPasswordLength", "-10")
|
|
assert.Equal(t, 72, Params.MaxPasswordLength.GetAsInt())
|
|
|
|
assert.Equal(t, int64(4096), Params.MaxArrayCapacity.GetAsInt64())
|
|
params.Save("proxy.maxArrayCapacity", "5000")
|
|
assert.Equal(t, int64(5000), Params.MaxArrayCapacity.GetAsInt64())
|
|
params.Save("proxy.maxArrayCapacity", "0")
|
|
assert.Equal(t, int64(4096), Params.MaxArrayCapacity.GetAsInt64())
|
|
params.Save("proxy.maxArrayCapacity", "-1")
|
|
assert.Equal(t, int64(4096), Params.MaxArrayCapacity.GetAsInt64())
|
|
})
|
|
|
|
// t.Run("test proxyConfig panic", func(t *testing.T) {
|
|
// Params := params.ProxyCfg
|
|
//
|
|
// shouldPanic(t, "proxy.timeTickInterval", func() {
|
|
// params.Save("proxy.timeTickInterval", "")
|
|
// Params.TimeTickInterval.GetValue()
|
|
// })
|
|
//
|
|
// shouldPanic(t, "proxy.msgStream.timeTick.bufSize", func() {
|
|
// params.Save("proxy.msgStream.timeTick.bufSize", "abc")
|
|
// Params.MsgStreamTimeTickBufSize.GetAsInt()
|
|
// })
|
|
//
|
|
// shouldPanic(t, "proxy.maxNameLength", func() {
|
|
// params.Save("proxy.maxNameLength", "abc")
|
|
// Params.MaxNameLength.GetAsInt()
|
|
// })
|
|
//
|
|
// shouldPanic(t, "proxy.maxUsernameLength", func() {
|
|
// params.Save("proxy.maxUsernameLength", "abc")
|
|
// Params.MaxUsernameLength.GetAsInt()
|
|
// })
|
|
//
|
|
// shouldPanic(t, "proxy.minPasswordLength", func() {
|
|
// params.Save("proxy.minPasswordLength", "abc")
|
|
// Params.MinPasswordLength.GetAsInt()
|
|
// })
|
|
//
|
|
// shouldPanic(t, "proxy.maxPasswordLength", func() {
|
|
// params.Save("proxy.maxPasswordLength", "abc")
|
|
// Params.MaxPasswordLength.GetAsInt()
|
|
// })
|
|
//
|
|
// shouldPanic(t, "proxy.maxFieldNum", func() {
|
|
// params.Save("proxy.maxFieldNum", "abc")
|
|
// Params.MaxFieldNum.GetAsInt()
|
|
// })
|
|
//
|
|
// shouldPanic(t, "proxy.maxShardNum", func() {
|
|
// params.Save("proxy.maxShardNum", "abc")
|
|
// Params.MaxShardNum.GetAsInt()
|
|
// })
|
|
//
|
|
// shouldPanic(t, "proxy.maxDimension", func() {
|
|
// params.Save("proxy.maxDimension", "-asdf")
|
|
// Params.MaxDimension.GetAsInt()
|
|
// })
|
|
//
|
|
// shouldPanic(t, "proxy.maxTaskNum", func() {
|
|
// params.Save("proxy.maxTaskNum", "-asdf")
|
|
// Params.MaxTaskNum.GetAsInt()
|
|
// })
|
|
//
|
|
// shouldPanic(t, "proxy.maxUserNum", func() {
|
|
// params.Save("proxy.maxUserNum", "abc")
|
|
// Params.MaxUserNum.GetAsInt()
|
|
// })
|
|
//
|
|
// shouldPanic(t, "proxy.maxRoleNum", func() {
|
|
// params.Save("proxy.maxRoleNum", "abc")
|
|
// Params.MaxRoleNum.GetAsInt()
|
|
// })
|
|
// })
|
|
|
|
t.Run("test queryCoordConfig", func(t *testing.T) {
|
|
Params := ¶ms.QueryCoordCfg
|
|
assert.Equal(t, Params.EnableActiveStandby.GetAsBool(), false)
|
|
t.Logf("queryCoord EnableActiveStandby = %t", Params.EnableActiveStandby.GetAsBool())
|
|
|
|
params.Save("queryCoord.NextTargetSurviveTime", "100")
|
|
NextTargetSurviveTime := &Params.NextTargetSurviveTime
|
|
assert.Equal(t, int64(100), NextTargetSurviveTime.GetAsInt64())
|
|
|
|
params.Save("queryCoord.UpdateNextTargetInterval", "100")
|
|
UpdateNextTargetInterval := &Params.UpdateNextTargetInterval
|
|
assert.Equal(t, int64(100), UpdateNextTargetInterval.GetAsInt64())
|
|
|
|
params.Save("queryCoord.checkNodeInReplicaInterval", "100")
|
|
checkNodeInReplicaInterval := &Params.CheckNodeInReplicaInterval
|
|
assert.Equal(t, 100, checkNodeInReplicaInterval.GetAsInt())
|
|
|
|
params.Save("queryCoord.checkResourceGroupInterval", "10")
|
|
checkResourceGroupInterval := &Params.CheckResourceGroupInterval
|
|
assert.Equal(t, 10, checkResourceGroupInterval.GetAsInt())
|
|
|
|
enableResourceGroupAutoRecover := &Params.EnableRGAutoRecover
|
|
assert.Equal(t, true, enableResourceGroupAutoRecover.GetAsBool())
|
|
params.Save("queryCoord.enableRGAutoRecover", "false")
|
|
enableResourceGroupAutoRecover = &Params.EnableRGAutoRecover
|
|
assert.Equal(t, false, enableResourceGroupAutoRecover.GetAsBool())
|
|
|
|
checkHealthInterval := Params.CheckHealthInterval.GetAsInt()
|
|
assert.Equal(t, 3000, checkHealthInterval)
|
|
checkHealthRPCTimeout := Params.CheckHealthRPCTimeout.GetAsInt()
|
|
assert.Equal(t, 2000, checkHealthRPCTimeout)
|
|
|
|
updateInterval := Params.UpdateCollectionLoadStatusInterval.GetAsDuration(time.Minute)
|
|
assert.Equal(t, updateInterval, time.Minute*5)
|
|
|
|
assert.Equal(t, 0.1, Params.GlobalRowCountFactor.GetAsFloat())
|
|
params.Save("queryCoord.globalRowCountFactor", "0.4")
|
|
assert.Equal(t, 0.4, Params.GlobalRowCountFactor.GetAsFloat())
|
|
|
|
assert.Equal(t, 0.05, Params.ScoreUnbalanceTolerationFactor.GetAsFloat())
|
|
params.Save("queryCoord.scoreUnbalanceTolerationFactor", "0.4")
|
|
assert.Equal(t, 0.4, Params.ScoreUnbalanceTolerationFactor.GetAsFloat())
|
|
|
|
assert.Equal(t, 1.3, Params.ReverseUnbalanceTolerationFactor.GetAsFloat())
|
|
params.Save("queryCoord.reverseUnBalanceTolerationFactor", "1.5")
|
|
assert.Equal(t, 1.5, Params.ReverseUnbalanceTolerationFactor.GetAsFloat())
|
|
|
|
assert.Equal(t, 1000, Params.SegmentCheckInterval.GetAsInt())
|
|
assert.Equal(t, 1000, Params.ChannelCheckInterval.GetAsInt())
|
|
assert.Equal(t, 300, Params.BalanceCheckInterval.GetAsInt())
|
|
params.Save(Params.BalanceCheckInterval.Key, "3000")
|
|
assert.Equal(t, 3000, Params.BalanceCheckInterval.GetAsInt())
|
|
assert.Equal(t, 10000, Params.IndexCheckInterval.GetAsInt())
|
|
assert.Equal(t, 3, Params.CollectionRecoverTimesLimit.GetAsInt())
|
|
assert.Equal(t, true, Params.AutoBalance.GetAsBool())
|
|
assert.Equal(t, true, Params.AutoBalanceChannel.GetAsBool())
|
|
assert.Equal(t, 10, Params.CheckAutoBalanceConfigInterval.GetAsInt())
|
|
|
|
params.Save("queryCoord.gracefulStopTimeout", "100")
|
|
assert.Equal(t, 100*time.Second, Params.GracefulStopTimeout.GetAsDuration(time.Second))
|
|
assert.Equal(t, true, Params.EnableStoppingBalance.GetAsBool())
|
|
|
|
assert.Equal(t, "ChannelLevelScoreBalancer", Params.Balancer.GetValue())
|
|
assert.Equal(t, 3, Params.ChannelExclusiveNodeFactor.GetAsInt())
|
|
|
|
assert.Equal(t, 200, Params.CollectionObserverInterval.GetAsInt())
|
|
params.Save("queryCoord.collectionObserverInterval", "100")
|
|
assert.Equal(t, 100, Params.CollectionObserverInterval.GetAsInt())
|
|
params.Reset("queryCoord.collectionObserverInterval")
|
|
|
|
assert.Equal(t, 0.1, Params.DelegatorMemoryOverloadFactor.GetAsFloat())
|
|
assert.Equal(t, 5, Params.CollectionBalanceSegmentBatchSize.GetAsInt())
|
|
assert.Equal(t, 1, Params.CollectionBalanceChannelBatchSize.GetAsInt())
|
|
|
|
assert.Equal(t, 0, Params.ClusterLevelLoadReplicaNumber.GetAsInt())
|
|
assert.Len(t, Params.ClusterLevelLoadResourceGroups.GetAsStrings(), 0)
|
|
assert.False(t, Params.ClusterLevelLoadForceOverrideUserReplicaMode.GetAsBool())
|
|
|
|
assert.Equal(t, 10, Params.CollectionChannelCountFactor.GetAsInt())
|
|
assert.Equal(t, 3000, Params.AutoBalanceInterval.GetAsInt())
|
|
|
|
assert.Equal(t, 5, Params.BalanceSegmentBatchSize.GetAsInt())
|
|
assert.Equal(t, 1, Params.BalanceChannelBatchSize.GetAsInt())
|
|
assert.Equal(t, true, Params.EnableBalanceOnMultipleCollections.GetAsBool())
|
|
|
|
assert.Equal(t, 20, Params.QueryNodeTaskParallelismFactor.GetAsInt())
|
|
params.Save("queryCoord.queryNodeTaskParallelismFactor", "2")
|
|
assert.Equal(t, 2, Params.QueryNodeTaskParallelismFactor.GetAsInt())
|
|
|
|
assert.Equal(t, 100, Params.BalanceCheckCollectionMaxCount.GetAsInt())
|
|
assert.Equal(t, 30, Params.ResourceExhaustionPenaltyDuration.GetAsInt())
|
|
assert.Equal(t, 10, Params.ResourceExhaustionCleanupInterval.GetAsInt())
|
|
})
|
|
|
|
t.Run("test queryNodeConfig", func(t *testing.T) {
|
|
Params := ¶ms.QueryNodeCfg
|
|
|
|
interval := Params.StatsPublishInterval.GetAsInt()
|
|
assert.Equal(t, 1000, interval)
|
|
|
|
length := Params.FlowGraphMaxQueueLength.GetAsInt32()
|
|
assert.Equal(t, int32(16), length)
|
|
|
|
assert.Equal(t, 8, Params.DMLMicroBatchMaxMsgNum.GetAsInt())
|
|
params.Save(Params.DMLMicroBatchMaxMsgNum.Key, "4")
|
|
assert.Equal(t, 4, Params.DMLMicroBatchMaxMsgNum.GetAsInt())
|
|
params.Reset(Params.DMLMicroBatchMaxMsgNum.Key)
|
|
|
|
maxParallelism := Params.FlowGraphMaxParallelism.GetAsInt32()
|
|
assert.Equal(t, int32(1024), maxParallelism)
|
|
|
|
// test query side config
|
|
chunkRows := Params.ChunkRows.GetAsInt64()
|
|
assert.Equal(t, int64(128), chunkRows)
|
|
nlist := Params.InterimIndexNlist.GetAsInt64()
|
|
assert.Equal(t, int64(128), nlist)
|
|
|
|
nprobe := Params.InterimIndexNProbe.GetAsInt64()
|
|
assert.Equal(t, int64(16), nprobe)
|
|
|
|
assert.Equal(t, 0.5, Params.InterimIndexBuildParallelRate.GetAsFloat())
|
|
// growingBuildThreadRate defaults to 0, which keeps growing index build single threaded.
|
|
assert.Equal(t, 0.0, Params.InterimIndexGrowingBuildThreadRate.GetAsFloat())
|
|
params.Save(Params.InterimIndexGrowingBuildThreadRate.Key, "0.25")
|
|
assert.Equal(t, 0.25, Params.InterimIndexGrowingBuildThreadRate.GetAsFloat())
|
|
params.Reset(Params.InterimIndexGrowingBuildThreadRate.Key)
|
|
assert.Equal(t, 0.0, Params.InterimIndexGrowingBuildThreadRate.GetAsFloat())
|
|
|
|
// enableGISSplitFusion defaults to true: the GIS coarse/refine split and
|
|
// same-column fusion rewrite is on unless explicitly disabled.
|
|
assert.True(t, Params.EnableGISSplitFusion.GetAsBool())
|
|
assert.Equal(t, "true", Params.EnableGISSplitFusion.DefaultValue)
|
|
params.Save(Params.EnableGISSplitFusion.Key, "false")
|
|
assert.False(t, Params.EnableGISSplitFusion.GetAsBool())
|
|
params.Reset(Params.EnableGISSplitFusion.Key)
|
|
assert.True(t, Params.EnableGISSplitFusion.GetAsBool())
|
|
|
|
assert.Equal(t, int32(1024), Params.MaxUnsolvedQueueSize.GetAsInt32())
|
|
assert.Equal(t, "1024", Params.MaxUnsolvedQueueSize.DefaultValue)
|
|
assert.Equal(t, int64(64), Params.MaxGroupNQ.GetAsInt64())
|
|
assert.Equal(t, 16.0, Params.NQMergeRatio.GetAsFloat())
|
|
assert.Equal(t, 20.0, Params.TopKMergeRatio.GetAsFloat())
|
|
assert.Equal(t, 50*time.Millisecond, Params.MaxDeadlineMergeGap.GetAsDurationByParse())
|
|
defer params.Reset(Params.MaxDeadlineMergeGap.Key)
|
|
assert.NoError(t, params.Save(Params.MaxDeadlineMergeGap.Key, "100ms"))
|
|
assert.Equal(t, 100*time.Millisecond, Params.MaxDeadlineMergeGap.GetAsDurationByParse())
|
|
assert.NoError(t, params.Save(Params.MaxDeadlineMergeGap.Key, "100"))
|
|
assert.Equal(t, 100*time.Millisecond, Params.MaxDeadlineMergeGap.GetAsDurationByParse())
|
|
assert.Equal(t, "fifo", Params.SchedulePolicyName.GetValue())
|
|
assert.Equal(t, 50*time.Millisecond, Params.SchedulePolicyTaskDeadlineAdvance.GetAsDurationByParse())
|
|
defer params.Reset(Params.SchedulePolicyTaskDeadlineAdvance.Key)
|
|
assert.NoError(t, params.Save(Params.SchedulePolicyTaskDeadlineAdvance.Key, "100ms"))
|
|
assert.Equal(t, 100*time.Millisecond, Params.SchedulePolicyTaskDeadlineAdvance.GetAsDurationByParse())
|
|
assert.NoError(t, params.Save(Params.SchedulePolicyTaskDeadlineAdvance.Key, "100"))
|
|
assert.Equal(t, 100*time.Millisecond, Params.SchedulePolicyTaskDeadlineAdvance.GetAsDurationByParse())
|
|
assert.Equal(t, 10.0, Params.CPURatio.GetAsFloat())
|
|
assert.Equal(t, uint32(hardware.GetCPUNum()), Params.KnowhereThreadPoolSize.GetAsUint32())
|
|
assert.Equal(t, "10s", Params.StandaloneMigrateDataTimeout.DefaultValue)
|
|
|
|
// chunk cache
|
|
assert.Equal(t, "willneed", Params.ReadAheadPolicy.GetValue())
|
|
|
|
// test small indexNlist/NProbe default
|
|
params.Remove("queryNode.segcore.smallIndex.nlist")
|
|
params.Remove("queryNode.segcore.smallIndex.nprobe")
|
|
params.Save("queryNode.segcore.chunkRows", "8192")
|
|
chunkRows = Params.ChunkRows.GetAsInt64()
|
|
assert.Equal(t, int64(8192), chunkRows)
|
|
|
|
enableInterimIndex := Params.EnableInterminSegmentIndex.GetAsBool()
|
|
assert.Equal(t, true, enableInterimIndex)
|
|
|
|
params.Save("queryNode.segcore.interimIndex.enableIndex", "true")
|
|
enableInterimIndex = Params.EnableInterminSegmentIndex.GetAsBool()
|
|
assert.Equal(t, true, enableInterimIndex)
|
|
|
|
assert.Equal(t, false, Params.KnowhereScoreConsistency.GetAsBool())
|
|
params.Save("queryNode.segcore.knowhereScoreConsistency", "true")
|
|
assert.Equal(t, true, Params.KnowhereScoreConsistency.GetAsBool())
|
|
params.Save("queryNode.segcore.knowhereScoreConsistency", "false")
|
|
|
|
nlist = Params.InterimIndexNlist.GetAsInt64()
|
|
assert.Equal(t, int64(128), nlist)
|
|
|
|
nprobe = Params.InterimIndexNProbe.GetAsInt64()
|
|
assert.Equal(t, int64(16), nprobe)
|
|
|
|
params.Remove("queryNode.segcore.growing.nlist")
|
|
params.Remove("queryNode.segcore.growing.nprobe")
|
|
params.Save("queryNode.segcore.chunkRows", "64")
|
|
chunkRows = Params.ChunkRows.GetAsInt64()
|
|
assert.Equal(t, int64(128), chunkRows)
|
|
|
|
params.Save("queryNode.gracefulStopTimeout", "100")
|
|
gracefulStopTimeout := &Params.GracefulStopTimeout
|
|
assert.Equal(t, int64(100), gracefulStopTimeout.GetAsInt64())
|
|
|
|
assert.Equal(t, false, Params.EnableWorkerSQCostMetrics.GetAsBool())
|
|
|
|
params.Save("querynode.gracefulStopTimeout", "100")
|
|
assert.Equal(t, 100*time.Second, Params.GracefulStopTimeout.GetAsDuration(time.Second))
|
|
|
|
assert.Equal(t, 2.5, Params.MemoryIndexLoadPredictMemoryUsageFactor.GetAsFloat())
|
|
params.Save("queryNode.memoryIndexLoadPredictMemoryUsageFactor", "2.0")
|
|
assert.Equal(t, 2.0, Params.MemoryIndexLoadPredictMemoryUsageFactor.GetAsFloat())
|
|
|
|
assert.NotZero(t, Params.DiskCacheCapacityLimit.GetAsSize())
|
|
params.Save("queryNode.diskCacheCapacityLimit", "70")
|
|
assert.Equal(t, int64(70), Params.DiskCacheCapacityLimit.GetAsSize())
|
|
params.Save("queryNode.diskCacheCapacityLimit", "70m")
|
|
assert.Equal(t, int64(70*1024*1024), Params.DiskCacheCapacityLimit.GetAsSize())
|
|
|
|
assert.Equal(t, 2, Params.BloomFilterApplyParallelFactor.GetAsInt())
|
|
assert.Equal(t, hardware.GetCPUNum(), Params.DelegatorPostLoadConcurrencyFactor.GetAsInt())
|
|
params.Save(Params.DelegatorPostLoadConcurrencyFactor.Key, "2")
|
|
assert.Equal(t, hardware.GetCPUNum()*2, Params.DelegatorPostLoadConcurrencyFactor.GetAsInt())
|
|
params.Save(Params.DelegatorPostLoadConcurrencyFactor.Key, "0")
|
|
assert.Equal(t, hardware.GetCPUNum(), Params.DelegatorPostLoadConcurrencyFactor.GetAsInt())
|
|
params.Reset(Params.DelegatorPostLoadConcurrencyFactor.Key)
|
|
|
|
assert.Equal(t, true, Params.SkipGrowingSegmentBF.GetAsBool())
|
|
assert.Equal(t, true, Params.EnableSegmentFilter.GetAsBool())
|
|
|
|
assert.Equal(t, "/var/lib/milvus/data/mmap", Params.MmapDirPath.GetValue())
|
|
|
|
assert.Equal(t, 60*time.Second, Params.DiskSizeFetchInterval.GetAsDuration(time.Second))
|
|
|
|
assert.Equal(t, 1.0, Params.PartialResultRequiredDataRatio.GetAsFloat())
|
|
params.Save(Params.PartialResultRequiredDataRatio.Key, "0.8")
|
|
assert.Equal(t, 0.8, Params.PartialResultRequiredDataRatio.GetAsFloat())
|
|
|
|
assert.False(t, Params.InternalCollectionUseTakeForOutput.GetAsBool())
|
|
params.Save(Params.InternalCollectionUseTakeForOutput.Key, "true")
|
|
assert.True(t, Params.InternalCollectionUseTakeForOutput.GetAsBool())
|
|
assert.True(t, Params.ExternalCollectionUseTakeForOutput.GetAsBool())
|
|
params.Save(Params.ExternalCollectionUseTakeForOutput.Key, "false")
|
|
assert.False(t, Params.ExternalCollectionUseTakeForOutput.GetAsBool())
|
|
defer params.Reset(Params.TakeForOutputResultCountLimit.Key)
|
|
assert.Equal(t, int64(10000), Params.TakeForOutputResultCountLimit.GetAsInt64())
|
|
params.Save(Params.TakeForOutputResultCountLimit.Key, "0")
|
|
assert.Equal(t, int64(0), Params.TakeForOutputResultCountLimit.GetAsInt64())
|
|
params.Save(Params.TakeForOutputResultCountLimit.Key, "2048")
|
|
assert.Equal(t, int64(2048), Params.TakeForOutputResultCountLimit.GetAsInt64())
|
|
params.Save(Params.TakeForOutputResultCountLimit.Key, "-1")
|
|
assert.Equal(t, int64(10000), Params.TakeForOutputResultCountLimit.GetAsInt64())
|
|
params.Save(Params.TakeForOutputResultCountLimit.Key, "invalid")
|
|
assert.Equal(t, int64(10000), Params.TakeForOutputResultCountLimit.GetAsInt64())
|
|
|
|
// test CatchUpStreamingDataTsLag parameter
|
|
assert.Equal(t, 1*time.Second, Params.CatchUpStreamingDataTsLag.GetAsDurationByParse())
|
|
params.Save(Params.CatchUpStreamingDataTsLag.Key, "5s")
|
|
assert.Equal(t, 5*time.Second, Params.CatchUpStreamingDataTsLag.GetAsDurationByParse())
|
|
params.Save(Params.CatchUpStreamingDataTsLag.Key, "0s")
|
|
assert.Equal(t, time.Duration(0), Params.CatchUpStreamingDataTsLag.GetAsDurationByParse())
|
|
})
|
|
|
|
t.Run("test dataCoordConfig", func(t *testing.T) {
|
|
Params := ¶ms.DataCoordCfg
|
|
assert.Equal(t, 24*60*60*time.Second, Params.SegmentMaxLifetime.GetAsDuration(time.Second))
|
|
assert.True(t, Params.EnableGarbageCollection.GetAsBool())
|
|
assert.Equal(t, Params.EnableActiveStandby.GetAsBool(), false)
|
|
t.Logf("dataCoord EnableActiveStandby = %t", Params.EnableActiveStandby.GetAsBool())
|
|
assert.Equal(t, int64(4096), Params.GrowingSegmentsMemSizeInMB.GetAsInt64())
|
|
|
|
assert.Equal(t, true, Params.AutoBalance.GetAsBool())
|
|
assert.Equal(t, 10, Params.CheckAutoBalanceConfigInterval.GetAsInt())
|
|
assert.Equal(t, false, Params.AutoUpgradeSegmentIndex.GetAsBool())
|
|
assert.Equal(t, 2, Params.FilesPerPreImportTask.GetAsInt())
|
|
assert.Equal(t, 172800*time.Second, Params.ImportTaskRetention.GetAsDuration(time.Second))
|
|
assert.Equal(t, 16384, Params.MaxSizeInMBPerImportTask.GetAsInt())
|
|
assert.Equal(t, 2*time.Second, Params.ImportScheduleInterval.GetAsDuration(time.Second))
|
|
assert.Equal(t, 2*time.Second, Params.ImportCheckIntervalHigh.GetAsDuration(time.Second))
|
|
assert.Equal(t, 120*time.Second, Params.ImportCheckIntervalLow.GetAsDuration(time.Second))
|
|
assert.Equal(t, 1024, Params.MaxFilesPerImportReq.GetAsInt())
|
|
assert.Equal(t, 1024, Params.MaxImportJobNum.GetAsInt())
|
|
assert.Equal(t, true, Params.WaitForIndex.GetAsBool())
|
|
assert.Equal(t, false, Params.ImportInReplicatingCluster.GetAsBool())
|
|
assert.Equal(t, false, Params.EnableL0Import.GetAsBool())
|
|
assert.Equal(t, 4, Params.ImportFileNumPerSlot.GetAsInt())
|
|
assert.Equal(t, 160*1024*1024, Params.ImportMemoryLimitPerSlot.GetAsInt())
|
|
|
|
params.Save("datacoord.gracefulStopTimeout", "100")
|
|
assert.Equal(t, 100*time.Second, Params.GracefulStopTimeout.GetAsDuration(time.Second))
|
|
|
|
assert.Equal(t, hardware.GetCPUNum(), Params.GCRemoveConcurrent.GetAsInt())
|
|
params.Save("dataCoord.gc.removeConcurrent", "32")
|
|
assert.Equal(t, 32, Params.GCRemoveConcurrent.GetAsInt())
|
|
assert.Equal(t, 0.6, Params.GCSlowDownCPUUsageThreshold.GetAsFloat())
|
|
params.Save("dataCoord.gc.slowDownCPUUsageThreshold", "0.5")
|
|
assert.Equal(t, 0.5, Params.GCSlowDownCPUUsageThreshold.GetAsFloat())
|
|
params.Save("dataCoord.compaction.gcInterval", "100")
|
|
assert.Equal(t, float64(100), Params.CompactionGCIntervalInSeconds.GetAsDuration(time.Second).Seconds())
|
|
params.Save("dataCoord.compaction.dropTolerance", "100")
|
|
assert.Equal(t, float64(100), Params.CompactionDropToleranceInSeconds.GetAsDuration(time.Second).Seconds())
|
|
assert.Equal(t, int64(10000), Params.CompactionPreAllocateIDExpansionFactor.GetAsInt64())
|
|
assert.False(t, Params.StorageFormatCompactionEnabled.GetAsBool())
|
|
|
|
params.Save("dataCoord.compaction.clustering.enable", "true")
|
|
assert.Equal(t, true, Params.ClusteringCompactionEnable.GetAsBool())
|
|
params.Save("dataCoord.compaction.clustering.newDataSizeThreshold", "10")
|
|
assert.Equal(t, int64(10), Params.ClusteringCompactionNewDataSizeThreshold.GetAsSize())
|
|
params.Save("dataCoord.compaction.clustering.newDataSizeThreshold", "10k")
|
|
assert.Equal(t, int64(10*1024), Params.ClusteringCompactionNewDataSizeThreshold.GetAsSize())
|
|
params.Save("dataCoord.compaction.clustering.newDataSizeThreshold", "10m")
|
|
assert.Equal(t, int64(10*1024*1024), Params.ClusteringCompactionNewDataSizeThreshold.GetAsSize())
|
|
params.Save("dataCoord.compaction.clustering.newDataSizeThreshold", "10g")
|
|
assert.Equal(t, int64(10*1024*1024*1024), Params.ClusteringCompactionNewDataSizeThreshold.GetAsSize())
|
|
params.Save("dataCoord.compaction.clustering.maxSegmentSizeRatio", "1.2")
|
|
assert.Equal(t, 1.2, Params.ClusteringCompactionMaxSegmentSizeRatio.GetAsFloat())
|
|
params.Save("dataCoord.compaction.clustering.preferSegmentSizeRatio", "0.5")
|
|
assert.Equal(t, 0.5, Params.ClusteringCompactionPreferSegmentSizeRatio.GetAsFloat())
|
|
params.Save("dataCoord.slot.clusteringCompactionUsage", "10")
|
|
assert.Equal(t, 10, Params.ClusteringCompactionSlotUsage.GetAsInt())
|
|
params.Save("dataCoord.slot.mixCompactionUsage", "5")
|
|
assert.Equal(t, 5, Params.MixCompactionSlotUsage.GetAsInt())
|
|
params.Save("dataCoord.slot.l0DeleteCompactionUsage", "4")
|
|
assert.Equal(t, 4, Params.L0DeleteCompactionSlotUsage.GetAsInt())
|
|
assert.Equal(t, 16, Params.L0ManifestUpdatePoolSize.GetAsInt())
|
|
params.Save("dataCoord.compaction.levelzero.manifestUpdatePoolSize", "4")
|
|
assert.Equal(t, 4, Params.L0ManifestUpdatePoolSize.GetAsInt())
|
|
params.Save("dataCoord.compaction.levelzero.manifestUpdatePoolSize", "0")
|
|
assert.Equal(t, 1, Params.L0ManifestUpdatePoolSize.GetAsInt())
|
|
params.Save("datacoord.scheduler.taskSlowThreshold", "1000")
|
|
assert.Equal(t, 1000*time.Second, Params.TaskSlowThreshold.GetAsDuration(time.Second))
|
|
|
|
params.Save("datacoord.statsTask.enable", "true")
|
|
assert.True(t, Params.EnableSortCompaction.GetAsBool())
|
|
params.Save("datacoord.taskCheckInterval", "500")
|
|
assert.Equal(t, 500*time.Second, Params.TaskCheckInterval.GetAsDuration(time.Second))
|
|
params.Save("datacoord.statsTaskTriggerCount", "3")
|
|
assert.Equal(t, 3, Params.SortCompactionTriggerCount.GetAsInt())
|
|
// The stats admission limit is independent from the sort compaction trigger count.
|
|
assert.Equal(t, 100, Params.StatsTaskPendingLimit.GetAsInt())
|
|
params.Save("datacoord.statsTaskPendingLimit", "7")
|
|
assert.Equal(t, 7, Params.StatsTaskPendingLimit.GetAsInt())
|
|
|
|
assert.Equal(t, 100, Params.MaxSegmentsPerCopyTask.GetAsInt())
|
|
params.Save("dataCoord.import.maxSegmentsPerCopyTask", "200")
|
|
assert.Equal(t, 200, Params.MaxSegmentsPerCopyTask.GetAsInt())
|
|
})
|
|
|
|
t.Run("test dataNodeConfig", func(t *testing.T) {
|
|
Params := ¶ms.DataNodeCfg
|
|
|
|
SetNodeID(2)
|
|
|
|
id := GetNodeID()
|
|
t.Logf("NodeID: %d", id)
|
|
|
|
length := Params.FlowGraphMaxQueueLength.GetAsInt()
|
|
t.Logf("flowGraphMaxQueueLength: %d", length)
|
|
|
|
maxParallelism := Params.FlowGraphMaxParallelism.GetAsInt()
|
|
t.Logf("flowGraphMaxParallelism: %d", maxParallelism)
|
|
|
|
flowGraphSkipModeEnable := Params.FlowGraphSkipModeEnable.GetAsBool()
|
|
t.Logf("flowGraphSkipModeEnable: %t", flowGraphSkipModeEnable)
|
|
|
|
flowGraphSkipModeSkipNum := Params.FlowGraphSkipModeSkipNum.GetAsInt()
|
|
t.Logf("flowGraphSkipModeSkipNum: %d", flowGraphSkipModeSkipNum)
|
|
|
|
flowGraphSkipModeColdTime := Params.FlowGraphSkipModeColdTime.GetAsInt()
|
|
t.Logf("flowGraphSkipModeColdTime: %d", flowGraphSkipModeColdTime)
|
|
|
|
maxParallelSyncTaskNum := Params.MaxParallelSyncTaskNum.GetAsInt()
|
|
t.Logf("maxParallelSyncTaskNum: %d", maxParallelSyncTaskNum)
|
|
|
|
maxParallelSyncMgrTasksPerCPUCore := Params.MaxParallelSyncMgrTasksPerCPUCore.GetAsInt()
|
|
t.Logf("maxParallelSyncMgrTasksPerCPUCore: %d", maxParallelSyncMgrTasksPerCPUCore)
|
|
assert.Equal(t, 16, maxParallelSyncMgrTasksPerCPUCore)
|
|
|
|
size := Params.FlushInsertBufferSize.GetAsInt()
|
|
t.Logf("FlushInsertBufferSize: %d", size)
|
|
|
|
period := &Params.SyncPeriod
|
|
t.Logf("SyncPeriod: %v", period)
|
|
assert.Equal(t, 10*time.Minute, Params.SyncPeriod.GetAsDuration(time.Second))
|
|
|
|
channelWorkPoolSize := Params.ChannelWorkPoolSize.GetAsInt()
|
|
t.Logf("channelWorkPoolSize: %d", channelWorkPoolSize)
|
|
assert.Equal(t, -1, Params.ChannelWorkPoolSize.GetAsInt())
|
|
|
|
updateChannelCheckpointMaxParallel := Params.UpdateChannelCheckpointMaxParallel.GetAsInt()
|
|
t.Logf("updateChannelCheckpointMaxParallel: %d", updateChannelCheckpointMaxParallel)
|
|
assert.Equal(t, 10, Params.UpdateChannelCheckpointMaxParallel.GetAsInt())
|
|
assert.Equal(t, 128, Params.MaxChannelCheckpointsPerRPC.GetAsInt())
|
|
assert.Equal(t, 10*time.Second, Params.ChannelCheckpointUpdateTickInSeconds.GetAsDuration(time.Second))
|
|
|
|
assert.Equal(t, 4, Params.ImportConcurrencyPerCPUCore.GetAsInt())
|
|
assert.Equal(t, int64(16), Params.MaxImportFileSizeInGB.GetAsInt64())
|
|
assert.Equal(t, 16*1024*1024, Params.ImportBaseBufferSize.GetAsInt())
|
|
assert.Equal(t, 16*1024*1024, Params.ImportDeleteBufferSize.GetAsInt())
|
|
assert.Equal(t, 10.0, Params.ImportMemoryLimitPercentage.GetAsFloat())
|
|
assert.Equal(t, 0, Params.ImportMaxWriteRetryAttempts.GetAsInt())
|
|
assert.Equal(t, 1, Params.ImportWriteRetryInitialInterval.GetAsInt())
|
|
assert.Equal(t, 60, Params.ImportWriteRetryMaxInterval.GetAsInt())
|
|
// a non-positive or unparseable interval must fall back to the default,
|
|
// otherwise retry.Sleep(0) yields a zero-delay unbounded retry loop.
|
|
for _, invalid := range []string{"0", "-1", "1s"} {
|
|
params.Save(Params.ImportWriteRetryInitialInterval.Key, invalid)
|
|
assert.Equal(t, 1, Params.ImportWriteRetryInitialInterval.GetAsInt())
|
|
params.Save(Params.ImportWriteRetryMaxInterval.Key, invalid)
|
|
assert.Equal(t, 60, Params.ImportWriteRetryMaxInterval.GetAsInt())
|
|
}
|
|
params.Save(Params.ImportWriteRetryInitialInterval.Key, "2")
|
|
assert.Equal(t, 2, Params.ImportWriteRetryInitialInterval.GetAsInt())
|
|
params.Save(Params.ImportWriteRetryMaxInterval.Key, "120")
|
|
assert.Equal(t, 120, Params.ImportWriteRetryMaxInterval.GetAsInt())
|
|
params.Reset(Params.ImportWriteRetryInitialInterval.Key)
|
|
params.Reset(Params.ImportWriteRetryMaxInterval.Key)
|
|
assert.Equal(t, time.Hour, Params.ImportCopyObjectTimeout.GetAsDuration(time.Second))
|
|
params.Save(Params.ImportCopyObjectTimeout.Key, "120")
|
|
assert.Equal(t, 2*time.Minute, Params.ImportCopyObjectTimeout.GetAsDuration(time.Second))
|
|
params.Save(Params.ImportCopyObjectTimeout.Key, "0")
|
|
assert.Equal(t, time.Hour, Params.ImportCopyObjectTimeout.GetAsDuration(time.Second))
|
|
params.Reset(Params.ImportCopyObjectTimeout.Key)
|
|
params.Save("datanode.gracefulStopTimeout", "100")
|
|
assert.Equal(t, 100*time.Second, Params.GracefulStopTimeout.GetAsDuration(time.Second))
|
|
assert.Equal(t, 16, Params.SlotCap.GetAsInt())
|
|
|
|
// compaction
|
|
assert.Equal(t, 10, Params.MaxCompactionConcurrency.GetAsInt())
|
|
|
|
assert.Equal(t, 4, Params.MaxVecIndexBuildConcurrency.GetAsInt())
|
|
assert.Equal(t, 8, Params.JSONStatsBuildMaxWorkers.GetAsInt())
|
|
assert.Equal(t, int64(DefaultJSONStatsBuildMaxInflightBytes),
|
|
Params.JSONStatsBuildMaxInflightBytes.GetAsInt64())
|
|
params.Save(Params.JSONStatsBuildMaxInflightBytes.Key, "-1")
|
|
assert.Equal(t, int64(DefaultJSONStatsBuildMaxInflightBytes),
|
|
Params.JSONStatsBuildMaxInflightBytes.GetAsInt64())
|
|
params.Save(Params.JSONStatsBuildMaxInflightBytes.Key, "invalid")
|
|
assert.Equal(t, int64(DefaultJSONStatsBuildMaxInflightBytes),
|
|
Params.JSONStatsBuildMaxInflightBytes.GetAsInt64())
|
|
params.Save(Params.JSONStatsBuildMaxInflightBytes.Key, "0")
|
|
assert.Equal(t, int64(0),
|
|
Params.JSONStatsBuildMaxInflightBytes.GetAsInt64())
|
|
params.Save(Params.JSONStatsBuildMaxInflightBytes.Key, "67108864")
|
|
assert.Equal(t, int64(67108864),
|
|
Params.JSONStatsBuildMaxInflightBytes.GetAsInt64())
|
|
params.Reset(Params.JSONStatsBuildMaxInflightBytes.Key)
|
|
|
|
// clustering compaction
|
|
params.Save("datanode.clusteringCompaction.memoryBufferRatio", "0.1")
|
|
assert.Equal(t, 0.1, Params.ClusteringCompactionMemoryBufferRatio.GetAsFloat())
|
|
params.Save("datanode.clusteringCompaction.workPoolSize", "2")
|
|
assert.Equal(t, int64(2), Params.ClusteringCompactionWorkerPoolSize.GetAsInt64())
|
|
|
|
assert.Equal(t, 2, Params.BloomFilterApplyParallelFactor.GetAsInt())
|
|
assert.Equal(t, "dataNode.storage.format", Params.StorageFormat.Key)
|
|
assert.Equal(t, "parquet", Params.StorageFormat.GetValue())
|
|
params.Save(Params.StorageFormat.Key, "vortex")
|
|
assert.Equal(t, "vortex", Params.StorageFormat.GetValue())
|
|
params.Reset(Params.StorageFormat.Key)
|
|
assert.Equal(t, 16, Params.WorkerSlotUnit.GetAsInt())
|
|
assert.Equal(t, 0.25, Params.StandaloneSlotRatio.GetAsFloat())
|
|
})
|
|
|
|
t.Run("test streamingConfig", func(t *testing.T) {
|
|
assert.Equal(t, "streaming.splitChunkSN", params.StreamingCfg.SplitChunkSN.Key)
|
|
assert.False(t, params.StreamingCfg.SplitChunkSN.GetAsBool())
|
|
params.Save(params.StreamingCfg.SplitChunkSN.Key, "true")
|
|
assert.True(t, params.StreamingCfg.SplitChunkSN.GetAsBool())
|
|
params.Reset(params.StreamingCfg.SplitChunkSN.Key)
|
|
assert.False(t, params.StreamingCfg.SplitChunkSN.GetAsBool())
|
|
assert.Equal(t, false, params.StreamingCfg.WALScannerPauseConsumption.GetAsBool())
|
|
assert.Equal(t, 1*time.Minute, params.StreamingCfg.WALBalancerTriggerInterval.GetAsDurationByParse())
|
|
assert.Equal(t, 10*time.Millisecond, params.StreamingCfg.WALBalancerBackoffInitialInterval.GetAsDurationByParse())
|
|
assert.Equal(t, 5*time.Second, params.StreamingCfg.WALBalancerBackoffMaxInterval.GetAsDurationByParse())
|
|
assert.Equal(t, 2.0, params.StreamingCfg.WALBalancerBackoffMultiplier.GetAsFloat())
|
|
assert.Equal(t, "vchannelFair", params.StreamingCfg.WALBalancerPolicyName.GetValue())
|
|
assert.Equal(t, true, params.StreamingCfg.WALBalancerPolicyAllowRebalance.GetAsBool())
|
|
assert.Equal(t, 5*time.Minute, params.StreamingCfg.WALBalancerPolicyMinRebalanceIntervalThreshold.GetAsDurationByParse())
|
|
assert.Equal(t, 1*time.Second, params.StreamingCfg.WALBalancerPolicyAllowRebalanceRecoveryLagThreshold.GetAsDurationByParse())
|
|
assert.Equal(t, 0.4, params.StreamingCfg.WALBalancerPolicyVChannelFairPChannelWeight.GetAsFloat())
|
|
assert.Equal(t, 0.3, params.StreamingCfg.WALBalancerPolicyVChannelFairVChannelWeight.GetAsFloat())
|
|
assert.Equal(t, 0.01, params.StreamingCfg.WALBalancerPolicyVChannelFairAntiAffinityWeight.GetAsFloat())
|
|
assert.Equal(t, 0.01, params.StreamingCfg.WALBalancerPolicyVChannelFairRebalanceTolerance.GetAsFloat())
|
|
assert.Equal(t, 3, params.StreamingCfg.WALBalancerPolicyVChannelFairRebalanceMaxStep.GetAsInt())
|
|
assert.Equal(t, 30*time.Minute, params.StreamingCfg.WALBalancerOperationTimeout.GetAsDurationByParse())
|
|
assert.Equal(t, 5*time.Second, params.StreamingCfg.WALBalancerNodeLostGracePeriod.GetAsDurationByParse())
|
|
assert.Equal(t, 4.0, params.StreamingCfg.WALBroadcasterConcurrencyRatio.GetAsFloat())
|
|
assert.Equal(t, 5*time.Minute, params.StreamingCfg.WALBroadcasterTombstoneCheckInternal.GetAsDurationByParse())
|
|
assert.Equal(t, 8192, params.StreamingCfg.WALBroadcasterTombstoneMaxCount.GetAsInt())
|
|
assert.Equal(t, 24*time.Hour, params.StreamingCfg.WALBroadcasterTombstoneMaxLifetime.GetAsDurationByParse())
|
|
assert.Equal(t, 10*time.Second, params.StreamingCfg.TxnDefaultKeepaliveTimeout.GetAsDurationByParse())
|
|
assert.Equal(t, 30*time.Second, params.StreamingCfg.WALWriteAheadBufferKeepalive.GetAsDurationByParse())
|
|
assert.Equal(t, int64(64*1024*1024), params.StreamingCfg.WALWriteAheadBufferCapacity.GetAsSize())
|
|
assert.Equal(t, 128, params.StreamingCfg.WALReadAheadBufferLength.GetAsInt())
|
|
assert.Equal(t, 1*time.Second, params.StreamingCfg.LoggingAppendSlowThreshold.GetAsDurationByParse())
|
|
assert.Equal(t, int64(640000000), params.StreamingCfg.PartialUpdateVersionIndexMaxBytes.GetAsSize())
|
|
assert.Equal(t, 3*time.Second, params.StreamingCfg.WALRecoveryGracefulCloseTimeout.GetAsDurationByParse())
|
|
assert.Equal(t, 24*time.Hour, params.StreamingCfg.WALRecoverySchemaExpirationTolerance.GetAsDurationByParse())
|
|
assert.Equal(t, 100, params.StreamingCfg.WALRecoveryMaxDirtyMessage.GetAsInt())
|
|
assert.Equal(t, 10*time.Second, params.StreamingCfg.WALRecoveryPersistInterval.GetAsDurationByParse())
|
|
assert.Equal(t, false, params.StreamingCfg.IdempotencyEnabled.GetAsBool())
|
|
assert.Equal(t, int64(16*1024*1024), params.StreamingCfg.IdempotencyMaxBytesPerWindow.GetAsSize())
|
|
assert.Equal(t, int64(256*1024*1024), params.StreamingCfg.IdempotencyMaxRetainedBytes.GetAsSize())
|
|
assert.Equal(t, 256, params.StreamingCfg.IdempotencyMaxRetainedChunks.GetAsInt())
|
|
assert.Equal(t, 256, params.StreamingCfg.IdempotencyMaxKeyLength.GetAsInt())
|
|
assert.Equal(t, float64(0.6), params.StreamingCfg.FlushMemoryThreshold.GetAsFloat())
|
|
assert.Equal(t, float64(0.2), params.StreamingCfg.FlushGrowingSegmentBytesHwmThreshold.GetAsFloat())
|
|
assert.Equal(t, float64(0.1), params.StreamingCfg.FlushGrowingSegmentBytesLwmThreshold.GetAsFloat())
|
|
assert.Equal(t, 10*time.Minute, params.StreamingCfg.FlushL0MaxLifetime.GetAsDurationByParse())
|
|
assert.Equal(t, 500000, params.StreamingCfg.FlushL0MaxRowNum.GetAsInt())
|
|
assert.Equal(t, int64(32*1024*1024), params.StreamingCfg.FlushL0MaxSize.GetAsSize())
|
|
assert.Equal(t, 30, params.StreamingCfg.OldVersionLastConfirmedWindowSize.GetAsInt())
|
|
assert.Equal(t, 2*time.Second, params.StreamingCfg.DelegatorEmptyTimeTickMaxFilterInterval.GetAsDurationByParse())
|
|
assert.Equal(t, 1*time.Second, params.StreamingCfg.FlushEmptyTimeTickMaxFilterInterval.GetAsDurationByParse())
|
|
assert.Equal(t, 0, params.StreamingCfg.WALBalancerExpectedInitialStreamingNodeNum.GetAsInt())
|
|
|
|
// wal rate limit
|
|
assert.Equal(t, int64(20*1024*1024), params.StreamingCfg.WALRateLimitDefaultBurst.GetAsSize())
|
|
assert.Equal(t, 0.85, params.StreamingCfg.WALRateLimitNodeMemorySlowdownThreshold.GetAsFloat())
|
|
assert.Equal(t, 0.90, params.StreamingCfg.WALRateLimitNodeMemoryRejectThreshold.GetAsFloat())
|
|
assert.Equal(t, 0.80, params.StreamingCfg.WALRateLimitNodeMemoryRecoverThreshold.GetAsFloat())
|
|
|
|
// append rate limit
|
|
assert.Equal(t, false, params.StreamingCfg.WALRateLimitAppendRateEnabled.GetAsBool())
|
|
assert.Equal(t, int64(32*1024*1024), params.StreamingCfg.WALRateLimitAppendRateSlowdownThreshold.GetAsSize())
|
|
assert.Equal(t, int64(28*1024*1024), params.StreamingCfg.WALRateLimitAppendRateRecoverThreshold.GetAsSize())
|
|
|
|
// append rate adaptive rate limit
|
|
assert.Equal(t, 0*time.Second, params.StreamingCfg.WALRateLimitAppendRateAdaptiveRateLimit.SlowdownStartupDelayInterval.GetAsDurationByParse())
|
|
assert.Equal(t, int64(32*1024*1024), params.StreamingCfg.WALRateLimitAppendRateAdaptiveRateLimit.SlowdownHWM.GetAsSize())
|
|
assert.Equal(t, int64(2*1024*1024), params.StreamingCfg.WALRateLimitAppendRateAdaptiveRateLimit.SlowdownLWM.GetAsSize())
|
|
assert.Equal(t, 10*time.Second, params.StreamingCfg.WALRateLimitAppendRateAdaptiveRateLimit.SlowdownDecreaseInterval.GetAsDurationByParse())
|
|
assert.Equal(t, 0.9, params.StreamingCfg.WALRateLimitAppendRateAdaptiveRateLimit.SlowdownDecreaseRatio.GetAsFloat())
|
|
assert.Equal(t, 0*time.Second, params.StreamingCfg.WALRateLimitAppendRateAdaptiveRateLimit.SlowdownRejectDelayInterval.GetAsDurationByParse())
|
|
assert.Equal(t, int64(32*1024*1024), params.StreamingCfg.WALRateLimitAppendRateAdaptiveRateLimit.RecoveryHWM.GetAsSize())
|
|
assert.Equal(t, int64(4*1024*1024), params.StreamingCfg.WALRateLimitAppendRateAdaptiveRateLimit.RecoveryLWM.GetAsSize())
|
|
assert.Equal(t, 10*time.Second, params.StreamingCfg.WALRateLimitAppendRateAdaptiveRateLimit.RecoveryNormalDelayInterval.GetAsDurationByParse())
|
|
assert.Equal(t, int64(512*1024), params.StreamingCfg.WALRateLimitAppendRateAdaptiveRateLimit.RecoveryIncremental.GetAsSize())
|
|
assert.Equal(t, 1*time.Second, params.StreamingCfg.WALRateLimitAppendRateAdaptiveRateLimit.RecoveryIncreaseInterval.GetAsDurationByParse())
|
|
|
|
// node memory adaptive rate limit
|
|
assert.Equal(t, 0*time.Second, params.StreamingCfg.WALRateLimitNodeMemoryAdaptiveRateLimit.SlowdownStartupDelayInterval.GetAsDurationByParse())
|
|
assert.Equal(t, int64(4*1024*1024), params.StreamingCfg.WALRateLimitNodeMemoryAdaptiveRateLimit.SlowdownHWM.GetAsSize())
|
|
assert.Equal(t, int64(256*1024), params.StreamingCfg.WALRateLimitNodeMemoryAdaptiveRateLimit.SlowdownLWM.GetAsSize())
|
|
assert.Equal(t, 10*time.Second, params.StreamingCfg.WALRateLimitNodeMemoryAdaptiveRateLimit.SlowdownDecreaseInterval.GetAsDurationByParse())
|
|
assert.Equal(t, 0.8, params.StreamingCfg.WALRateLimitNodeMemoryAdaptiveRateLimit.SlowdownDecreaseRatio.GetAsFloat())
|
|
assert.Equal(t, 0*time.Second, params.StreamingCfg.WALRateLimitNodeMemoryAdaptiveRateLimit.SlowdownRejectDelayInterval.GetAsDurationByParse())
|
|
assert.Equal(t, int64(16*1024*1024), params.StreamingCfg.WALRateLimitNodeMemoryAdaptiveRateLimit.RecoveryHWM.GetAsSize())
|
|
assert.Equal(t, int64(1*1024*1024), params.StreamingCfg.WALRateLimitNodeMemoryAdaptiveRateLimit.RecoveryLWM.GetAsSize())
|
|
assert.Equal(t, 10*time.Second, params.StreamingCfg.WALRateLimitNodeMemoryAdaptiveRateLimit.RecoveryNormalDelayInterval.GetAsDurationByParse())
|
|
assert.Equal(t, int64(256*1024), params.StreamingCfg.WALRateLimitNodeMemoryAdaptiveRateLimit.RecoveryIncremental.GetAsSize())
|
|
assert.Equal(t, 1*time.Second, params.StreamingCfg.WALRateLimitNodeMemoryAdaptiveRateLimit.RecoveryIncreaseInterval.GetAsDurationByParse())
|
|
|
|
// recovery storage adaptive rate limit
|
|
assert.Equal(t, 30*time.Second, params.StreamingCfg.WALRateLimitRecoveryStorageAdaptiveRateLimit.SlowdownStartupDelayInterval.GetAsDurationByParse())
|
|
assert.Equal(t, int64(32*1024*1024), params.StreamingCfg.WALRateLimitRecoveryStorageAdaptiveRateLimit.SlowdownHWM.GetAsSize())
|
|
assert.Equal(t, int64(512*1024), params.StreamingCfg.WALRateLimitRecoveryStorageAdaptiveRateLimit.SlowdownLWM.GetAsSize())
|
|
assert.Equal(t, 10*time.Second, params.StreamingCfg.WALRateLimitRecoveryStorageAdaptiveRateLimit.SlowdownDecreaseInterval.GetAsDurationByParse())
|
|
assert.Equal(t, 0.8, params.StreamingCfg.WALRateLimitRecoveryStorageAdaptiveRateLimit.SlowdownDecreaseRatio.GetAsFloat())
|
|
assert.Equal(t, 90*time.Second, params.StreamingCfg.WALRateLimitRecoveryStorageAdaptiveRateLimit.SlowdownRejectDelayInterval.GetAsDurationByParse())
|
|
assert.Equal(t, int64(32*1024*1024), params.StreamingCfg.WALRateLimitRecoveryStorageAdaptiveRateLimit.RecoveryHWM.GetAsSize())
|
|
assert.Equal(t, int64(1*1024*1024), params.StreamingCfg.WALRateLimitRecoveryStorageAdaptiveRateLimit.RecoveryLWM.GetAsSize())
|
|
assert.Equal(t, 30*time.Second, params.StreamingCfg.WALRateLimitRecoveryStorageAdaptiveRateLimit.RecoveryNormalDelayInterval.GetAsDurationByParse())
|
|
assert.Equal(t, int64(1*1024*1024), params.StreamingCfg.WALRateLimitRecoveryStorageAdaptiveRateLimit.RecoveryIncremental.GetAsSize())
|
|
assert.Equal(t, 1*time.Second, params.StreamingCfg.WALRateLimitRecoveryStorageAdaptiveRateLimit.RecoveryIncreaseInterval.GetAsDurationByParse())
|
|
|
|
// flusher adaptive rate limit
|
|
assert.Equal(t, 10*time.Second, params.StreamingCfg.WALRateLimitFlusherAdaptiveRateLimit.SlowdownStartupDelayInterval.GetAsDurationByParse())
|
|
assert.Equal(t, int64(16*1024*1024), params.StreamingCfg.WALRateLimitFlusherAdaptiveRateLimit.SlowdownHWM.GetAsSize())
|
|
assert.Equal(t, int64(2*1024*1024), params.StreamingCfg.WALRateLimitFlusherAdaptiveRateLimit.SlowdownLWM.GetAsSize())
|
|
assert.Equal(t, 30*time.Second, params.StreamingCfg.WALRateLimitFlusherAdaptiveRateLimit.SlowdownDecreaseInterval.GetAsDurationByParse())
|
|
assert.Equal(t, 0.8, params.StreamingCfg.WALRateLimitFlusherAdaptiveRateLimit.SlowdownDecreaseRatio.GetAsFloat())
|
|
assert.Equal(t, 2*time.Minute, params.StreamingCfg.WALRateLimitFlusherAdaptiveRateLimit.SlowdownRejectDelayInterval.GetAsDurationByParse())
|
|
assert.Equal(t, int64(64*1024*1024), params.StreamingCfg.WALRateLimitFlusherAdaptiveRateLimit.RecoveryHWM.GetAsSize())
|
|
assert.Equal(t, int64(16*1024*1024), params.StreamingCfg.WALRateLimitFlusherAdaptiveRateLimit.RecoveryLWM.GetAsSize())
|
|
assert.Equal(t, 5*time.Second, params.StreamingCfg.WALRateLimitFlusherAdaptiveRateLimit.RecoveryNormalDelayInterval.GetAsDurationByParse())
|
|
assert.Equal(t, int64(5*1024*1024), params.StreamingCfg.WALRateLimitFlusherAdaptiveRateLimit.RecoveryIncremental.GetAsSize())
|
|
assert.Equal(t, 1*time.Second, params.StreamingCfg.WALRateLimitFlusherAdaptiveRateLimit.RecoveryIncreaseInterval.GetAsDurationByParse())
|
|
|
|
params.Save(params.StreamingCfg.WALBalancerTriggerInterval.Key, "50s")
|
|
params.Save(params.StreamingCfg.WALBalancerBackoffInitialInterval.Key, "50s")
|
|
params.Save(params.StreamingCfg.WALBalancerBackoffMultiplier.Key, "3.5")
|
|
params.Save(params.StreamingCfg.WALBroadcasterConcurrencyRatio.Key, "1.5")
|
|
params.Save(params.StreamingCfg.TxnDefaultKeepaliveTimeout.Key, "3500ms")
|
|
params.Save(params.StreamingCfg.WALWriteAheadBufferKeepalive.Key, "10s")
|
|
params.Save(params.StreamingCfg.WALWriteAheadBufferCapacity.Key, "128k")
|
|
params.Save(params.StreamingCfg.WALBalancerPolicyName.Key, "pchannelFair")
|
|
params.Save(params.StreamingCfg.WALBalancerPolicyVChannelFairPChannelWeight.Key, "0.5")
|
|
params.Save(params.StreamingCfg.WALBalancerPolicyVChannelFairVChannelWeight.Key, "0.4")
|
|
params.Save(params.StreamingCfg.WALBalancerPolicyVChannelFairAntiAffinityWeight.Key, "0.02")
|
|
params.Save(params.StreamingCfg.WALBalancerPolicyVChannelFairRebalanceTolerance.Key, "0.02")
|
|
params.Save(params.StreamingCfg.WALBalancerPolicyVChannelFairRebalanceMaxStep.Key, "4")
|
|
params.Save(params.StreamingCfg.WALBalancerPolicyAllowRebalance.Key, "false")
|
|
params.Save(params.StreamingCfg.WALBalancerPolicyMinRebalanceIntervalThreshold.Key, "10s")
|
|
params.Save(params.StreamingCfg.WALBalancerPolicyAllowRebalanceRecoveryLagThreshold.Key, "1s")
|
|
params.Save(params.StreamingCfg.LoggingAppendSlowThreshold.Key, "3s")
|
|
params.Save(params.StreamingCfg.WALRecoveryGracefulCloseTimeout.Key, "4s")
|
|
params.Save(params.StreamingCfg.WALRecoveryMaxDirtyMessage.Key, "200")
|
|
params.Save(params.StreamingCfg.WALRecoveryPersistInterval.Key, "20s")
|
|
params.Save(params.StreamingCfg.IdempotencyEnabled.Key, "true")
|
|
params.Save(params.StreamingCfg.IdempotencyMaxBytesPerWindow.Key, "33554432")
|
|
params.Save(params.StreamingCfg.IdempotencyMaxRetainedBytes.Key, "1073741824")
|
|
params.Save(params.StreamingCfg.IdempotencyMaxRetainedChunks.Key, "512")
|
|
params.Save(params.StreamingCfg.IdempotencyMaxKeyLength.Key, "2048")
|
|
params.Save(params.StreamingCfg.FlushMemoryThreshold.Key, "0.7")
|
|
params.Save(params.StreamingCfg.FlushGrowingSegmentBytesHwmThreshold.Key, "0.25")
|
|
params.Save(params.StreamingCfg.FlushGrowingSegmentBytesLwmThreshold.Key, "0.15")
|
|
params.Save(params.StreamingCfg.WALRateLimitDefaultBurst.Key, "10485760") // 10MB
|
|
params.Save(params.StreamingCfg.WALRateLimitAppendRateEnabled.Key, "true")
|
|
params.Save(params.StreamingCfg.WALRateLimitAppendRateSlowdownThreshold.Key, "128m")
|
|
params.Save(params.StreamingCfg.WALRateLimitAppendRateRecoverThreshold.Key, "100m")
|
|
assert.Equal(t, 50*time.Second, params.StreamingCfg.WALBalancerTriggerInterval.GetAsDurationByParse())
|
|
assert.Equal(t, 50*time.Second, params.StreamingCfg.WALBalancerBackoffInitialInterval.GetAsDurationByParse())
|
|
assert.Equal(t, 3.5, params.StreamingCfg.WALBalancerBackoffMultiplier.GetAsFloat())
|
|
assert.Equal(t, 1.5, params.StreamingCfg.WALBroadcasterConcurrencyRatio.GetAsFloat())
|
|
assert.Equal(t, "pchannelFair", params.StreamingCfg.WALBalancerPolicyName.GetValue())
|
|
assert.Equal(t, false, params.StreamingCfg.WALBalancerPolicyAllowRebalance.GetAsBool())
|
|
assert.Equal(t, 10*time.Second, params.StreamingCfg.WALBalancerPolicyMinRebalanceIntervalThreshold.GetAsDurationByParse())
|
|
assert.Equal(t, 1*time.Second, params.StreamingCfg.WALBalancerPolicyAllowRebalanceRecoveryLagThreshold.GetAsDurationByParse())
|
|
assert.Equal(t, 0.5, params.StreamingCfg.WALBalancerPolicyVChannelFairPChannelWeight.GetAsFloat())
|
|
assert.Equal(t, 0.4, params.StreamingCfg.WALBalancerPolicyVChannelFairVChannelWeight.GetAsFloat())
|
|
assert.Equal(t, 0.02, params.StreamingCfg.WALBalancerPolicyVChannelFairAntiAffinityWeight.GetAsFloat())
|
|
assert.Equal(t, 0.02, params.StreamingCfg.WALBalancerPolicyVChannelFairRebalanceTolerance.GetAsFloat())
|
|
assert.Equal(t, 4, params.StreamingCfg.WALBalancerPolicyVChannelFairRebalanceMaxStep.GetAsInt())
|
|
assert.Equal(t, 3500*time.Millisecond, params.StreamingCfg.TxnDefaultKeepaliveTimeout.GetAsDurationByParse())
|
|
assert.Equal(t, 10*time.Second, params.StreamingCfg.WALWriteAheadBufferKeepalive.GetAsDurationByParse())
|
|
assert.Equal(t, int64(128*1024), params.StreamingCfg.WALWriteAheadBufferCapacity.GetAsSize())
|
|
assert.Equal(t, 3*time.Second, params.StreamingCfg.LoggingAppendSlowThreshold.GetAsDurationByParse())
|
|
assert.Equal(t, 4*time.Second, params.StreamingCfg.WALRecoveryGracefulCloseTimeout.GetAsDurationByParse())
|
|
assert.Equal(t, 200, params.StreamingCfg.WALRecoveryMaxDirtyMessage.GetAsInt())
|
|
assert.Equal(t, 20*time.Second, params.StreamingCfg.WALRecoveryPersistInterval.GetAsDurationByParse())
|
|
assert.Equal(t, true, params.StreamingCfg.IdempotencyEnabled.GetAsBool())
|
|
assert.Equal(t, int64(32*1024*1024), params.StreamingCfg.IdempotencyMaxBytesPerWindow.GetAsSize())
|
|
assert.Equal(t, int64(1024*1024*1024), params.StreamingCfg.IdempotencyMaxRetainedBytes.GetAsSize())
|
|
assert.Equal(t, 512, params.StreamingCfg.IdempotencyMaxRetainedChunks.GetAsInt())
|
|
assert.Equal(t, 2048, params.StreamingCfg.IdempotencyMaxKeyLength.GetAsInt())
|
|
assert.Equal(t, float64(0.7), params.StreamingCfg.FlushMemoryThreshold.GetAsFloat())
|
|
assert.Equal(t, float64(0.25), params.StreamingCfg.FlushGrowingSegmentBytesHwmThreshold.GetAsFloat())
|
|
assert.Equal(t, float64(0.15), params.StreamingCfg.FlushGrowingSegmentBytesLwmThreshold.GetAsFloat())
|
|
assert.Equal(t, 10*1024*1024, params.StreamingCfg.WALRateLimitDefaultBurst.GetAsInt())
|
|
assert.Equal(t, true, params.StreamingCfg.WALRateLimitAppendRateEnabled.GetAsBool())
|
|
assert.Equal(t, int64(128*1024*1024), params.StreamingCfg.WALRateLimitAppendRateSlowdownThreshold.GetAsSize())
|
|
assert.Equal(t, int64(100*1024*1024), params.StreamingCfg.WALRateLimitAppendRateRecoverThreshold.GetAsSize())
|
|
})
|
|
|
|
t.Run("channel config priority", func(t *testing.T) {
|
|
Params := ¶ms.CommonCfg
|
|
params.Save(Params.RootCoordDml.Key, "dml1")
|
|
params.Save(Params.RootCoordDml.FallbackKeys[0], "dml2")
|
|
|
|
assert.Equal(t, "by-dev-dml1", Params.RootCoordDml.GetValue())
|
|
})
|
|
|
|
t.Run("clustering compaction config", func(t *testing.T) {
|
|
Params := ¶ms.CommonCfg
|
|
params.Save("common.usePartitionKeyAsClusteringKey", "true")
|
|
assert.Equal(t, true, Params.UsePartitionKeyAsClusteringKey.GetAsBool())
|
|
params.Save("common.useVectorAsClusteringKey", "true")
|
|
assert.Equal(t, true, Params.UseVectorAsClusteringKey.GetAsBool())
|
|
params.Save("common.enableVectorClusteringKey", "true")
|
|
assert.Equal(t, true, Params.EnableVectorClusteringKey.GetAsBool())
|
|
})
|
|
}
|
|
|
|
func TestDataCoordCompactionTargetConfig(t *testing.T) {
|
|
base := NewBaseTable(SkipRemote(true))
|
|
var params ComponentParam
|
|
params.Init(base)
|
|
|
|
cfg := ¶ms.DataCoordCfg
|
|
assert.Equal(t, "dataCoord.compaction.enableTargetBasedCompaction", cfg.EnableTargetBasedCompaction.Key)
|
|
assert.Equal(t, "3.0.0", cfg.EnableTargetBasedCompaction.Version)
|
|
assert.Equal(t, "false", cfg.EnableTargetBasedCompaction.DefaultValue)
|
|
assert.False(t, cfg.EnableTargetBasedCompaction.GetAsBool())
|
|
|
|
base.Save(cfg.EnableTargetBasedCompaction.Key, "true")
|
|
assert.True(t, cfg.EnableTargetBasedCompaction.GetAsBool())
|
|
|
|
field, ok := reflect.TypeOf(dataCoordConfig{}).FieldByName("EnableTargetBasedCompaction")
|
|
assert.True(t, ok)
|
|
assert.Equal(t, "false", field.Tag.Get("refreshable"))
|
|
|
|
assert.Equal(t, "dataCoord.compaction.target.maxEventsPerReconcile", cfg.TargetCompactionMaxEvents.Key)
|
|
assert.Equal(t, "3.0.0", cfg.TargetCompactionMaxEvents.Version)
|
|
assert.Equal(t, "100", cfg.TargetCompactionMaxEvents.DefaultValue)
|
|
assert.Equal(t, 100, cfg.TargetCompactionMaxEvents.GetAsInt())
|
|
|
|
for _, value := range []string{"0", "-1", "invalid"} {
|
|
base.Save(cfg.TargetCompactionMaxEvents.Key, value)
|
|
assert.Equal(t, 100, cfg.TargetCompactionMaxEvents.GetAsInt())
|
|
}
|
|
base.Save(cfg.TargetCompactionMaxEvents.Key, "25")
|
|
assert.Equal(t, 25, cfg.TargetCompactionMaxEvents.GetAsInt())
|
|
|
|
field, ok = reflect.TypeOf(dataCoordConfig{}).FieldByName("TargetCompactionMaxEvents")
|
|
assert.True(t, ok)
|
|
assert.Equal(t, "true", field.Tag.Get("refreshable"))
|
|
}
|
|
|
|
func TestForbiddenItem(t *testing.T) {
|
|
Init()
|
|
params := Get()
|
|
|
|
params.baseTable.mgr.OnEvent(&config.Event{
|
|
Key: params.CommonCfg.ClusterPrefix.Key,
|
|
Value: "new-cluster",
|
|
})
|
|
assert.Equal(t, "by-dev", params.CommonCfg.ClusterPrefix.GetValue())
|
|
}
|
|
|
|
func TestFormatDurationWithMillisecondFallback(t *testing.T) {
|
|
assert.Equal(t, "", formatDurationWithMillisecondFallback(""))
|
|
assert.Equal(t, "", formatDurationWithMillisecondFallback(" "))
|
|
assert.Equal(t, "100ms", formatDurationWithMillisecondFallback("100"))
|
|
assert.Equal(t, "1.5ms", formatDurationWithMillisecondFallback("1.5"))
|
|
assert.Equal(t, "2s", formatDurationWithMillisecondFallback("2s"))
|
|
assert.Equal(t, "invalid", formatDurationWithMillisecondFallback("invalid"))
|
|
}
|
|
|
|
func TestCachedParam(t *testing.T) {
|
|
Init()
|
|
params := Get()
|
|
|
|
assert.Equal(t, 256*1024*1024, params.QueryCoordGrpcServerCfg.ServerMaxRecvSize.GetAsInt())
|
|
assert.Equal(t, 256*1024*1024, params.QueryCoordGrpcServerCfg.ServerMaxRecvSize.GetAsInt())
|
|
|
|
assert.Equal(t, int32(16), params.DataNodeCfg.FlowGraphMaxQueueLength.GetAsInt32())
|
|
assert.Equal(t, int32(16), params.DataNodeCfg.FlowGraphMaxQueueLength.GetAsInt32())
|
|
assert.Equal(t, int64(1000000), params.DataNodeCfg.ExternalCollectionTargetRowsPerSegment.GetAsInt64())
|
|
|
|
assert.Equal(t, uint(100000), params.CommonCfg.BloomFilterSize.GetAsUint())
|
|
assert.Equal(t, uint(100000), params.CommonCfg.BloomFilterSize.GetAsUint())
|
|
assert.True(t, params.CommonCfg.BloomFilterEnabled.GetAsBool())
|
|
assert.Equal(t, "BlockedBloomFilter", params.CommonCfg.BloomFilterType.GetValue())
|
|
|
|
assert.Equal(t, uint64(8388608), params.MQCfg.PursuitBufferSize.GetAsUint64())
|
|
assert.Equal(t, uint64(8388608), params.MQCfg.PursuitBufferSize.GetAsUint64())
|
|
|
|
assert.Equal(t, 60, params.MQCfg.PursuitBufferTime.GetAsInt())
|
|
|
|
assert.Equal(t, int64(1024), params.DataCoordCfg.SegmentMaxSize.GetAsInt64())
|
|
assert.Equal(t, int64(1024), params.DataCoordCfg.SegmentMaxSize.GetAsInt64())
|
|
|
|
assert.Equal(t, 0.85, params.QuotaConfig.DataNodeMemoryLowWaterLevel.GetAsFloat())
|
|
assert.Equal(t, 0.85, params.QuotaConfig.DataNodeMemoryLowWaterLevel.GetAsFloat())
|
|
|
|
assert.Equal(t, 1*time.Hour, params.DataCoordCfg.GCInterval.GetAsDuration(time.Second))
|
|
assert.Equal(t, 1*time.Hour, params.DataCoordCfg.GCInterval.GetAsDuration(time.Second))
|
|
|
|
params.Save(params.QuotaConfig.DiskQuota.Key, "192")
|
|
assert.Equal(t, float64(192*1024*1024), params.QuotaConfig.DiskQuota.GetAsFloat())
|
|
assert.Equal(t, float64(192*1024*1024), params.QuotaConfig.DiskQuotaPerCollection.GetAsFloat())
|
|
params.Save(params.QuotaConfig.DiskQuota.Key, "256")
|
|
assert.Equal(t, float64(256*1024*1024), params.QuotaConfig.DiskQuota.GetAsFloat())
|
|
assert.Equal(t, float64(256*1024*1024), params.QuotaConfig.DiskQuotaPerCollection.GetAsFloat())
|
|
params.Save(params.QuotaConfig.DiskQuota.Key, "192")
|
|
}
|
|
|
|
func TestFallbackParam(t *testing.T) {
|
|
Init()
|
|
params := Get()
|
|
params.Save("common.chanNamePrefix.cluster", "foo")
|
|
|
|
assert.Equal(t, "foo", params.CommonCfg.ClusterPrefix.GetValue())
|
|
}
|
|
|
|
func TestImportIdempotencyParams(t *testing.T) {
|
|
params := ComponentParam{}
|
|
params.Init(NewBaseTable(SkipRemote(true)))
|
|
|
|
assert.Equal(t, 256, params.StreamingCfg.IdempotencyMaxKeyLength.GetAsInt())
|
|
|
|
// The advertised idempotency window is the tombstone retention, so an import
|
|
// job must outlive its tombstone. Otherwise a retry inside the window resolves
|
|
// to a jobID GetImportProgress can no longer find.
|
|
assert.Equal(t, 172800, params.DataCoordCfg.ImportTaskRetention.GetAsInt())
|
|
|
|
// Headroom, not equality. A tombstone's age is measured from the last
|
|
// StreamingCoord start, so a restart extends its remaining life by up to another
|
|
// maxLifetime while this retention keeps counting from the job's own completion.
|
|
// Equal defaults satisfy the >= above yet break on the first restart, which is
|
|
// how adversarial review on milvus#52544 found this; one restart per tombstone
|
|
// lifetime is what the 2x covers.
|
|
assert.GreaterOrEqual(t,
|
|
params.DataCoordCfg.ImportTaskRetention.GetAsDuration(time.Second),
|
|
2*params.StreamingCfg.WALBroadcasterTombstoneMaxLifetime.GetAsDurationByParse())
|
|
}
|
|
|
|
func TestWriteSegmentIndexToManifest(t *testing.T) {
|
|
params := ComponentParam{}
|
|
params.Init(NewBaseTable(SkipRemote(true)))
|
|
|
|
// Off by default: manifest publication is opt-in, and the legacy
|
|
// etcd-only path must be what a cluster gets without configuration.
|
|
assert.False(t, params.DataCoordCfg.WriteSegmentIndexToManifest.GetAsBool())
|
|
|
|
params.Save(params.DataCoordCfg.WriteSegmentIndexToManifest.Key, "true")
|
|
assert.True(t, params.DataCoordCfg.WriteSegmentIndexToManifest.GetAsBool())
|
|
|
|
// A value that does not parse as a boolean reads as false, i.e. the
|
|
// legacy behavior. Under the old etcd-write polarity this silently
|
|
// entered the one-way off state; with publication opt-in the failure is
|
|
// merely "the operator thinks it is on", with no durability consequence.
|
|
params.Save(params.DataCoordCfg.WriteSegmentIndexToManifest.Key, "yes")
|
|
assert.False(t, params.DataCoordCfg.WriteSegmentIndexToManifest.GetAsBool())
|
|
}
|
|
|
|
func TestSegmentIndexManifestLoadConcurrency(t *testing.T) {
|
|
params := ComponentParam{}
|
|
params.Init(NewBaseTable(SkipRemote(true)))
|
|
|
|
// The manifest reload is object-storage IO behind cgo, not metastore IO:
|
|
// borrowing metastore.readConcurrency (32, and shared with the querycoord
|
|
// and rootcoord catalogs) would cap a fail-closed startup scan that runs
|
|
// once per healthy V3 segment.
|
|
assert.Equal(t, 64, params.DataCoordCfg.SegmentIndexManifestLoadConcurrency.GetAsInt())
|
|
assert.NotEqual(t,
|
|
params.MetaStoreCfg.ReadConcurrency.Key,
|
|
params.DataCoordCfg.SegmentIndexManifestLoadConcurrency.Key)
|
|
|
|
// A pool size below 1 would deadlock the scan, so it is clamped, not trusted.
|
|
params.Save(params.DataCoordCfg.SegmentIndexManifestLoadConcurrency.Key, "0")
|
|
assert.Equal(t, 1, params.DataCoordCfg.SegmentIndexManifestLoadConcurrency.GetAsInt())
|
|
params.Save(params.DataCoordCfg.SegmentIndexManifestLoadConcurrency.Key, "-8")
|
|
assert.Equal(t, 1, params.DataCoordCfg.SegmentIndexManifestLoadConcurrency.GetAsInt())
|
|
|
|
// Excessive native concurrency is clamped independently of integer overflow.
|
|
params.Save(params.DataCoordCfg.SegmentIndexManifestLoadConcurrency.Key, "2147483648")
|
|
assert.Equal(t, 256, params.DataCoordCfg.SegmentIndexManifestLoadConcurrency.GetAsInt())
|
|
}
|