mirror of
https://github.com/vxcontrol/cloud.git
synced 2026-07-19 19:53:42 -04:00
Dmitry Ng
276 lines
7.0 KiB
Go
276 lines
7.0 KiB
Go
package anonymizer
|
|
|
|
import (
|
|
"bytes"
|
|
"fmt"
|
|
"io"
|
|
"math/rand"
|
|
"strings"
|
|
"testing"
|
|
|
|
"github.com/vxcontrol/cloud/anonymizer/patterns"
|
|
"github.com/vxcontrol/cloud/anonymizer/testdata"
|
|
)
|
|
|
|
func BenchmarkWrapper_WithRealReplacer(b *testing.B) {
|
|
datasets, err := testdata.LoadAllTestData()
|
|
if err != nil {
|
|
b.Fatalf("failed to load test data: %v", err)
|
|
}
|
|
|
|
insensitiveDataset, err := testdata.LoadInsensitiveData()
|
|
if err != nil {
|
|
b.Fatalf("failed to load insensitive data: %v", err)
|
|
}
|
|
|
|
allLoadedPatterns, err := patterns.LoadPatterns(patterns.PatternListTypeAll)
|
|
if err != nil {
|
|
b.Fatalf("failed to load all patterns: %v", err)
|
|
}
|
|
|
|
replacer, err := NewReplacer(allLoadedPatterns.Regexes(), allLoadedPatterns.Names())
|
|
if err != nil {
|
|
b.Fatalf("failed to create replacer: %v", err)
|
|
}
|
|
|
|
rng := rand.New(rand.NewSource(1234567))
|
|
getWrappedDataset := func() (string, string) {
|
|
prefixIdx := rng.Intn(len(insensitiveDataset.Entries))
|
|
suffixIdx := rng.Intn(len(insensitiveDataset.Entries))
|
|
prefix := insensitiveDataset.Entries[prefixIdx].Examples
|
|
suffix := insensitiveDataset.Entries[suffixIdx].Examples
|
|
prefixLines := strings.Split(prefix, "\n")
|
|
suffixLines := strings.Split(suffix, "\n")
|
|
prefixLineIdx := rng.Intn(len(prefixLines))
|
|
suffixLineIdx := rng.Intn(len(suffixLines))
|
|
prefix = prefixLines[prefixLineIdx]
|
|
suffix = suffixLines[suffixLineIdx]
|
|
return prefix, suffix
|
|
}
|
|
|
|
testDataBuf := bytes.NewBuffer(nil)
|
|
for testDataBuf.Len() < 100*1024 { // at least 100KB
|
|
for _, dataset := range datasets {
|
|
for _, entry := range dataset.Entries {
|
|
prefix, suffix := getWrappedDataset()
|
|
testDataBuf.WriteString(prefix)
|
|
testDataBuf.WriteString(entry.Examples)
|
|
testDataBuf.WriteString(suffix)
|
|
testDataBuf.WriteString(" ")
|
|
}
|
|
testDataBuf.WriteString("\n")
|
|
}
|
|
}
|
|
|
|
testData := testDataBuf.Bytes()
|
|
chunkSizes := []int{1024, 4 * 1024, 8 * 1024, 16 * 1024, 32 * 1024, 64 * 1024}
|
|
|
|
for _, size := range chunkSizes {
|
|
b.Run(fmt.Sprintf("single_thread_reader_%dKB", size/1024), func(b *testing.B) {
|
|
b.ResetTimer()
|
|
b.ReportAllocs()
|
|
|
|
for b.Loop() {
|
|
wrapper := replacer.WrapReader(bytes.NewReader(testData))
|
|
buf := make([]byte, size)
|
|
for {
|
|
_, err := wrapper.Read(buf)
|
|
if err == io.EOF {
|
|
break
|
|
}
|
|
}
|
|
}
|
|
|
|
b.ReportMetric(float64(size), "bytes/op")
|
|
b.ReportMetric(float64(size*b.N)/float64(b.Elapsed().Seconds()), "bytes/sec")
|
|
})
|
|
}
|
|
|
|
b.Run("single_thread_reader", func(b *testing.B) {
|
|
b.ResetTimer()
|
|
b.ReportAllocs()
|
|
|
|
for b.Loop() {
|
|
wrapper := replacer.WrapReader(bytes.NewReader(testData))
|
|
io.Copy(io.Discard, wrapper)
|
|
}
|
|
|
|
b.ReportMetric(float64(len(testData)), "bytes/op")
|
|
b.ReportMetric(float64(len(testData)*b.N)/float64(b.Elapsed().Seconds()), "bytes/sec")
|
|
})
|
|
|
|
b.Run("parallel_reader", func(b *testing.B) {
|
|
b.ResetTimer()
|
|
b.ReportAllocs()
|
|
|
|
b.RunParallel(func(pb *testing.PB) {
|
|
for pb.Next() {
|
|
wrapper := replacer.WrapReader(bytes.NewReader(testData))
|
|
io.Copy(io.Discard, wrapper)
|
|
}
|
|
})
|
|
|
|
b.ReportMetric(float64(len(testData)), "bytes/op")
|
|
b.ReportMetric(float64(len(testData)*b.N)/float64(b.Elapsed().Seconds()), "bytes/sec")
|
|
})
|
|
|
|
b.Run("single_thread_replacer", func(b *testing.B) {
|
|
b.ResetTimer()
|
|
b.ReportAllocs()
|
|
|
|
for b.Loop() {
|
|
_ = replacer.ReplaceBytes(testData)
|
|
}
|
|
|
|
b.ReportMetric(float64(len(testData)), "bytes/op")
|
|
b.ReportMetric(float64(len(testData)*b.N)/float64(b.Elapsed().Seconds()), "bytes/sec")
|
|
})
|
|
|
|
b.Run("parallel_replacer", func(b *testing.B) {
|
|
b.ResetTimer()
|
|
b.ReportAllocs()
|
|
|
|
b.RunParallel(func(pb *testing.PB) {
|
|
for pb.Next() {
|
|
_ = replacer.ReplaceBytes(testData)
|
|
}
|
|
})
|
|
|
|
b.ReportMetric(float64(len(testData)), "bytes/op")
|
|
b.ReportMetric(float64(len(testData)*b.N)/float64(b.Elapsed().Seconds()), "bytes/sec")
|
|
})
|
|
}
|
|
|
|
func BenchmarkWrapper_MemoryEfficiency(b *testing.B) {
|
|
// test memory usage with large data and small reads
|
|
testData := strings.Repeat("memory efficiency test data ", 50000) // ~1.4MB
|
|
replacer := &mockReplacer{}
|
|
|
|
b.ResetTimer()
|
|
b.ReportAllocs()
|
|
|
|
for b.Loop() {
|
|
wrapper := newWrapper(strings.NewReader(testData), replacer)
|
|
buf := make([]byte, 1024) // read in small chunks
|
|
for {
|
|
_, err := wrapper.Read(buf)
|
|
if err == io.EOF {
|
|
break
|
|
}
|
|
}
|
|
}
|
|
|
|
b.ReportMetric(float64(len(testData)), "bytes/op")
|
|
b.ReportMetric(float64(len(testData)*b.N)/float64(b.Elapsed().Seconds()), "bytes/sec")
|
|
}
|
|
|
|
func BenchmarkWrapper_ReplacementIntensive(b *testing.B) {
|
|
// benchmark with data that has many patterns to replace
|
|
testData := strings.Repeat("password=secret123 token=abc456 api_key=def789 ", 1000)
|
|
patterns := []string{`password=([^\s]+)`, `token=([^\s]+)`, `api_key=([^\s]+)`}
|
|
names := []string{"PASSWORD", "TOKEN", "API_KEY"}
|
|
|
|
replacer, err := NewReplacer(patterns, names)
|
|
if err != nil {
|
|
b.Fatal(err)
|
|
}
|
|
|
|
b.ResetTimer()
|
|
b.ReportAllocs()
|
|
|
|
for b.Loop() {
|
|
wrapper := newWrapper(strings.NewReader(testData), replacer)
|
|
io.Copy(io.Discard, wrapper)
|
|
}
|
|
|
|
b.ReportMetric(float64(len(testData)), "bytes/op")
|
|
b.ReportMetric(float64(len(testData)*b.N)/float64(b.Elapsed().Seconds()), "bytes/sec")
|
|
}
|
|
|
|
func BenchmarkWrapper_NoReplacementNeeded(b *testing.B) {
|
|
// benchmark with data that has no patterns to replace
|
|
testData := strings.Repeat("clean log data with no sensitive information ", 1000)
|
|
patterns := []string{`password=([^\s]+)`, `token=([^\s]+)`}
|
|
names := []string{"PASSWORD", "TOKEN"}
|
|
|
|
replacer, err := NewReplacer(patterns, names)
|
|
if err != nil {
|
|
b.Fatal(err)
|
|
}
|
|
|
|
b.ResetTimer()
|
|
b.ReportAllocs()
|
|
|
|
for b.Loop() {
|
|
wrapper := newWrapper(strings.NewReader(testData), replacer)
|
|
io.Copy(io.Discard, wrapper)
|
|
}
|
|
|
|
b.ReportMetric(float64(len(testData)), "bytes/op")
|
|
b.ReportMetric(float64(len(testData)*b.N)/float64(b.Elapsed().Seconds()), "bytes/sec")
|
|
}
|
|
|
|
func BenchmarkWrapper_OverlapHeavy(b *testing.B) {
|
|
// benchmark scenario where patterns frequently span chunk boundaries
|
|
baseChunk := strings.Repeat("x", chunkSize-10)
|
|
pattern := "password=secret123"
|
|
testData := baseChunk + pattern + baseChunk + pattern + baseChunk
|
|
|
|
replacer := &simpleReplacer{
|
|
pattern: pattern,
|
|
replacement: "password=MASKED",
|
|
}
|
|
|
|
b.ResetTimer()
|
|
b.ReportAllocs()
|
|
|
|
for b.Loop() {
|
|
wrapper := newWrapper(strings.NewReader(testData), replacer)
|
|
io.Copy(io.Discard, wrapper)
|
|
}
|
|
|
|
b.ReportMetric(float64(len(testData)), "bytes/op")
|
|
b.ReportMetric(float64(len(testData)*b.N)/float64(b.Elapsed().Seconds()), "bytes/sec")
|
|
}
|
|
|
|
// helper types for testing
|
|
type errorReader struct {
|
|
err error
|
|
}
|
|
|
|
func (e *errorReader) Read(p []byte) (n int, err error) {
|
|
return 0, e.err
|
|
}
|
|
|
|
type intermittentReader struct {
|
|
data []byte
|
|
chunks []int
|
|
index int
|
|
pos int
|
|
}
|
|
|
|
func (r *intermittentReader) Read(p []byte) (n int, err error) {
|
|
if r.index >= len(r.chunks) {
|
|
return 0, io.EOF
|
|
}
|
|
|
|
chunkSize := r.chunks[r.index]
|
|
r.index++
|
|
|
|
if chunkSize == 0 {
|
|
return 0, nil // empty read
|
|
}
|
|
|
|
if r.pos >= len(r.data) {
|
|
return 0, io.EOF
|
|
}
|
|
|
|
available := len(r.data) - r.pos
|
|
toRead := min(min(chunkSize, len(p)), available)
|
|
|
|
copy(p, r.data[r.pos:r.pos+toRead])
|
|
r.pos += toRead
|
|
|
|
return toRead, nil
|
|
}
|