From ee7ba59f133f7d2f20e422f861405db0cf564caa Mon Sep 17 00:00:00 2001 From: Daniil Forshev Date: Fri, 6 Mar 2026 15:17:34 +0500 Subject: [PATCH 01/10] feat: add DocsFilter --- config/config.go | 20 +- docsfilter/docs_filter.go | 413 ++++++++++++++++++++++++++++++++++++ docsfilter/encoding.go | 285 +++++++++++++++++++++++++ docsfilter/encoding_test.go | 42 ++++ docsfilter/filter.go | 64 ++++++ docsfilter/metrics.go | 26 +++ 6 files changed, 843 insertions(+), 7 deletions(-) create mode 100644 docsfilter/docs_filter.go create mode 100644 docsfilter/encoding.go create mode 100644 docsfilter/encoding_test.go create mode 100644 docsfilter/filter.go create mode 100644 docsfilter/metrics.go diff --git a/config/config.go b/config/config.go index 00df0d94..84e7582d 100644 --- a/config/config.go +++ b/config/config.go @@ -277,13 +277,13 @@ type Config struct { } `config:"tracing"` // Additional filtering options - Filtering struct { - // If a search query time range overlaps with the [from; to] range - // the search query will be `AND`-ed with an additional predicate with the provided query expression - Query string `config:"query"` - From time.Time `config:"from"` - To time.Time `config:"to"` - } `config:"filtering"` + Filtering Filter `config:"filtering"` + DocsFilter struct { + DataDir string `config:"data_dir"` + Concurrency int `config:"concurrency"` + Filters []Filter `config:"filters"` + CacheSize Bytes `config:"cache_size" default:"100MiB"` + } `config:"docs_filter"` // Experimental provides flags // For configuring experimental features. @@ -305,3 +305,9 @@ func (b *Bytes) UnmarshalString(s string) error { *b = Bytes(bytes) return nil } + +type Filter struct { + Query string `config:"query"` + From time.Time `config:"from"` + To time.Time `config:"to"` +} diff --git a/docsfilter/docs_filter.go b/docsfilter/docs_filter.go new file mode 100644 index 00000000..2a092f7e --- /dev/null +++ b/docsfilter/docs_filter.go @@ -0,0 +1,413 @@ +package docsfilter + +import ( + "context" + "fmt" + "math" + "os" + "path" + "runtime" + "strings" + "sync" + "time" + + "go.uber.org/zap" + + "github.com/ozontech/seq-db/frac" + "github.com/ozontech/seq-db/frac/processor" + "github.com/ozontech/seq-db/fracmanager" + "github.com/ozontech/seq-db/logger" + "github.com/ozontech/seq-db/parser" + "github.com/ozontech/seq-db/seq" + "github.com/ozontech/seq-db/util" +) + +const ( + fracInQueueExt = ".queue" + fracDoneExt = ".filter" + tmpExt = ".tmp" +) + +const ( + defaultMaintenanceInterval = 30 * time.Second +) + +type MappingProvider interface { + GetMapping() seq.Mapping +} + +type Config struct { + DataDir string + Workers int + CacheSizeLimit uint64 +} + +type DocsFilter struct { + ctx context.Context + + config Config + filters map[string]*Filter + + fracs map[string][]string + fracsMu *sync.RWMutex + + mp MappingProvider + + rateLimit chan struct{} + createDirOnce *sync.Once + + maintenanceInterval time.Duration +} + +func New( + ctx context.Context, + cfg Config, + params []Params, + mp MappingProvider, +) *DocsFilter { + workers := cfg.Workers + if workers <= 0 { + workers = runtime.GOMAXPROCS(0) + } + + filtersMap := make(map[string]*Filter, len(params)) + + for _, p := range params { + f := NewFilter(p) + filtersMap[string(f.Hash())] = f + } + + return &DocsFilter{ + ctx: ctx, + config: cfg, + filters: filtersMap, + fracs: make(map[string][]string), + fracsMu: &sync.RWMutex{}, + mp: mp, + rateLimit: make(chan struct{}, workers), + createDirOnce: &sync.Once{}, + maintenanceInterval: defaultMaintenanceInterval, + } +} + +func (df *DocsFilter) Start(fracs fracmanager.List) { + df.createDataDir() + + err := df.loadFilters() + if err != nil { + logger.Fatal("failed to load previous docs filters", zap.Error(err)) + } + + err = df.buildQueue(fracs) + if err != nil { + logger.Fatal("failed to build docs filters queue", zap.Error(err)) + } + + go df.maintenance() + + mapping := df.mp.GetMapping() + + for _, f := range df.filters { + ast, err := parser.ParseSeqQL(f.params.Query, mapping) + if err != nil { + panic(fmt.Errorf("BUG: search query must be valid: %s", err)) + } + f.ast = ast + + df.processFilter(f, fracs.FilterInRange(seq.MID(f.params.From), seq.MID(f.params.To))) + } +} + +// RefreshFrac replaces frac's tombstone files with newly found results. Used after active frac is sealed. +func (df *DocsFilter) RefreshFrac(fraction frac.Fraction) { + df.fracsMu.RLock() + fracsFiles, has := df.fracs[fraction.Info().Name()] + df.fracsMu.RUnlock() + + if !has { + return + } + + for _, fileName := range fracsFiles { + filter := df.filters[filterNameFromTombstonesPath(fileName)] + + queueFilePath := path.Join(filter.dirPath, makeFileName(fraction.Info().Name(), fracInQueueExt)) + util.MustWriteFileAtomic(queueFilePath, []byte{}, tmpExt) + + filter.processWg.Add(1) + go func() { + if err := df.processFrac(fraction, filter, false); err != nil { + panic(fmt.Errorf("docs filter refresh frac err: %s", err)) + } + }() + } +} + +// RemoveFrac removes fraction's tombstones. Used after frac is deleted +func (df *DocsFilter) RemoveFrac(fracName string) { + df.fracsMu.RLock() + fracsFiles, has := df.fracs[fracName] + df.fracsMu.RUnlock() + + if !has { + return + } + + df.fracsMu.Lock() + delete(df.fracs, fracName) + df.fracsMu.Unlock() + + for _, fileName := range fracsFiles { + util.RemoveFile(fileName) + } +} + +func filterNameFromTombstonesPath(p string) string { + return path.Base(path.Dir(p)) +} + +func (df *DocsFilter) addDoneFrac(fracName, fracPath string) { + df.fracsMu.Lock() + defer df.fracsMu.Unlock() + + df.fracs[fracName] = append(df.fracs[fracName], fracPath) +} + +// loadFilters loads existing filters +func (df *DocsFilter) loadFilters() error { + des, err := os.ReadDir(df.config.DataDir) + if err != nil { + return err + } + + var anyRemove bool + + for _, de := range des { + if !de.IsDir() { + continue + } + + if _, ok := df.filters[de.Name()]; !ok { + logger.Info("there is filter folder on disk, but not in config. need to delete it.") + err := os.RemoveAll(path.Join(df.config.DataDir, de.Name())) + if err != nil && !os.IsNotExist(err) { + return err + } + anyRemove = true + continue + } + + f := df.filters[de.Name()] + f.status = StatusInProgress + f.dirPath = path.Join(df.config.DataDir, de.Name()) + + filterDes, err := os.ReadDir(f.dirPath) + if err != nil { + return fmt.Errorf("reading directory: %s", err) + } + + var hasFracsInQueue bool + + for _, fde := range filterDes { + if fde.IsDir() { + continue + } + name := fde.Name() + + switch path.Ext(name) { + case fracInQueueExt: + hasFracsInQueue = true + case fracDoneExt: + df.addDoneFrac(fracNameFromFilePath(name), path.Join(f.dirPath, name)) + } + } + + if !hasFracsInQueue { + f.status = StatusDone + } + } + + if anyRemove { + util.MustFsyncFile(df.config.DataDir) + } + + return nil +} + +// buildQueue creates a directory for each of unprocessed filters and creates .queue files +func (df *DocsFilter) buildQueue(fracs fracmanager.List) error { + for _, filter := range df.filters { + if filter.status != StatusCreated { + continue + } + filter.dirPath = path.Join(df.config.DataDir, filter.Hash()) + util.MustCreateDir(filter.dirPath) + + filterFracs := fracs.FilterInRange(seq.MID(filter.params.From), seq.MID(filter.params.To)) + for _, f := range filterFracs { + queueFilePath := path.Join(filter.dirPath, makeFileName(f.Info().Name(), fracInQueueExt)) + util.MustWriteFileAtomic(queueFilePath, []byte{}, tmpExt) + } + } + + return nil +} + +// handleFilter finds docs and writes to fs +func (df *DocsFilter) processFilter(filter *Filter, fracs fracmanager.List) { + if len(fracs) == 0 { + return + } + + fracsByName := make(map[string]frac.Fraction) + for _, f := range fracs { + fracsByName[f.Info().Name()] = f + } + + filterDes, err := os.ReadDir(filter.dirPath) + if err != nil { + panic(fmt.Errorf("BUG: reading directory must be successful: %s", err)) + } + + inProgressFilters.Add(1) + + processFracInQueue := func(name string) error { + f, ok := fracsByName[fracNameFromFilePath(name)] + if !ok { // skip missing fracs + return nil + } + filter.processWg.Add(1) + go func() { + if err := df.processFrac(f, filter, false); err != nil { + panic(fmt.Errorf("docs filter process frac err: %s", err)) + } + }() + return nil + } + _ = util.VisitFilesWithExt(filterDes, fracInQueueExt, processFracInQueue) + + go func() { + filter.processWg.Wait() + filter.markAsDone() + inProgressFilters.Add(-1) + }() +} + +func (df *DocsFilter) processFrac(f frac.Fraction, filter *Filter, refresh bool) error { + defer filter.processWg.Done() + + df.rateLimit <- struct{}{} + defer func() { <-df.rateLimit }() + + qpr, err := f.Search(df.ctx, processor.SearchParams{ + AST: filter.ast.Root, + From: seq.MID(filter.params.From), + To: seq.MID(filter.params.To), + Limit: math.MaxInt64, + }) + if err != nil { + return err + } + + queueFilePath := path.Join(filter.dirPath, makeFileName(f.Info().Name(), fracInQueueExt)) + doneFilePath := path.Join(filter.dirPath, makeFileName(f.Info().Name(), fracDoneExt)) + + if len(qpr.IDs) == 0 { + util.RemoveFile(queueFilePath) + return nil + } + + storeDocsFilter := func(rawDocsFilter []byte) error { + util.MustWriteFileAtomic(doneFilePath, rawDocsFilter, tmpExt) + util.RemoveFile(queueFilePath) + return nil + } + + // TODO: here we doing part of the work twice: + // first time we find LIDs inside f.Search() and then find IDs by these LIDs. + // Then we again find LIDs by earlier found IDs in f.FindLIDs(). + // We did it like this because otherwise we had to do serious f.Search() rewrite. + // For now we're ok with some performance penalty. + lids, err := f.FindLIDs(df.ctx, qpr.IDs.IDs()) + if err != nil { + return err + } + + docsFilterBin := DocsFilterBinIn{LIDs: lids} + if err := writeDocsFilter(&docsFilterBin, storeDocsFilter); err != nil { + return err + } + + if !refresh { + df.addDoneFrac(f.Info().Name(), doneFilePath) + } + + return nil +} + +func (df *DocsFilter) maintenance() { + for { + logger.Info("docs filter maintenance iteration") + df.checkDiskUsage() + time.Sleep(df.maintenanceInterval) + } +} + +func (df *DocsFilter) checkDiskUsage() { + du := int64(0) + + for _, f := range df.filters { + des, err := os.ReadDir(f.dirPath) + if err != nil { + logger.Error("docs filter: can't read filter's dir", + zap.String("filter", f.String()), zap.Error(err)) + return + } + + for _, fde := range des { + if fde.IsDir() { + continue + } + info, err := fde.Info() + if err != nil { + logger.Error("docs filter: can't read tombstones file info", + zap.String("filter", f.String()), zap.Error(err)) + return + } + du += info.Size() + } + } + + diskUsage.Set(float64(du)) + storedFilters.Set(float64(len(df.filters))) +} + +func makeFileName(name, ext string) string { + return name + ext +} + +func fracNameFromFilePath(filterFilePath string) string { + return strings.Split(path.Base(filterFilePath), ".")[0] +} + +var marshalBufferPool util.BufferPool + +func writeDocsFilter(df *DocsFilterBinIn, cb func(compressed []byte) error) error { + rawDocsFilter := marshalBufferPool.Get() + defer marshalBufferPool.Put(rawDocsFilter) + + rawDocsFilter.B = marshalDocsFilter(rawDocsFilter.B, df) + if err := cb(rawDocsFilter.B); err != nil { + return err + } + return nil +} + +// createDataDir creates dir data lazily to avoid creating extra folders. +func (df *DocsFilter) createDataDir() { + df.createDirOnce.Do(func() { + if err := os.MkdirAll(df.config.DataDir, 0o777); err != nil { + panic(err) + } + }) +} diff --git a/docsfilter/encoding.go b/docsfilter/encoding.go new file mode 100644 index 00000000..dccfc62f --- /dev/null +++ b/docsfilter/encoding.go @@ -0,0 +1,285 @@ +package docsfilter + +import ( + "encoding/binary" + "errors" + "fmt" + "math" + "unsafe" + + "github.com/ozontech/seq-db/seq" + "github.com/ozontech/seq-db/util" + "github.com/ozontech/seq-db/zstd" +) + +type DocsFilterBinIn struct { + LIDs []seq.LID +} + +type DocsFilterBinOut struct { + LIDs []uint32 +} + +type docsFilterBinVersion uint8 + +const ( + docsFilterBinVersion1 docsFilterBinVersion = iota + 1 +) + +var availableVersions = map[docsFilterBinVersion]struct{}{ + docsFilterBinVersion1: {}, +} + +type lidsCodec byte + +const ( + lidsCodecDelta = 1 + lidsCodecDeltaZstd = 2 +) + +type lidsBlockHeader struct { + Codec lidsCodec + Length uint32 // Number of LIDs in block + MinLID uint32 + MaxLID uint32 + Size uint32 // Size of ids block in bytes. + Offset uint64 // block's offset in file +} + +func (h *lidsBlockHeader) marshal(dst []byte) { + if len(dst) < int(lidsBlockHeaderSizeBytes) { + panic("BUG: marshal lidsBlockHeader: len(dst) is less than header size") + } + + dst[0] = byte(h.Codec) + dst = dst[1:] + binary.BigEndian.PutUint32(dst, h.Length) + dst = dst[sizeOfUint32:] + binary.BigEndian.PutUint32(dst, h.MinLID) + dst = dst[sizeOfUint32:] + binary.BigEndian.PutUint32(dst, h.MaxLID) + dst = dst[sizeOfUint32:] + binary.BigEndian.PutUint32(dst, h.Size) + dst = dst[sizeOfUint32:] + binary.BigEndian.PutUint64(dst, h.Offset) + dst = dst[sizeOfUint64:] +} + +func (h *lidsBlockHeader) unmarshal(src []byte) ([]byte, error) { + if len(src) < int(lidsBlockHeaderSizeBytes) { + return src, errors.New("too few bytes") + } + + h.Codec = lidsCodec(src[0]) + src = src[1:] + h.Length = binary.BigEndian.Uint32(src) + src = src[sizeOfUint32:] + h.MinLID = binary.BigEndian.Uint32(src) + src = src[sizeOfUint32:] + h.MaxLID = binary.BigEndian.Uint32(src) + src = src[sizeOfUint32:] + h.Size = binary.BigEndian.Uint32(src) + src = src[sizeOfUint32:] + h.Offset = binary.BigEndian.Uint64(src) + src = src[sizeOfUint64:] + + return src, nil +} + +func marshalDocsFilter(dst []byte, in *DocsFilterBinIn) []byte { + dst = append(dst, uint8(docsFilterBinVersion1)) + dst = marshalLIDsBlocks(dst, in.LIDs) + return dst +} + +const ( + sizeOfUint32 = unsafe.Sizeof(uint32(0)) + sizeOfUint64 = unsafe.Sizeof(uint64(0)) +) + +const ( + lidsBlockHeaderSizeBytes = 1 + (4 * sizeOfUint32) + sizeOfUint64 + maxLIDsBlockLen = 1024 +) + +var lidsBlockBufPool util.BufferPool + +func marshalLIDsBlocks(dst []byte, in []seq.LID) []byte { + b := lidsBlockBufPool.Get() + defer lidsBlockBufPool.Put(b) + + numberOfBlocks := (len(in) + maxLIDsBlockLen - 1) / maxLIDsBlockLen + dst = binary.BigEndian.AppendUint32(dst, uint32(numberOfBlocks)) + + // reserve space for headers + curHeaderOffset := len(dst) + dst = append(dst, make([]byte, numberOfBlocks*int(lidsBlockHeaderSizeBytes))...) + + var start int + for range numberOfBlocks { + end := min(maxLIDsBlockLen, len(in[start:])) + chunk := in[start : start+end] + + var codec lidsCodec + b.B, codec = marshalLIDsBlock(b.B[:0], chunk) + if len(b.B) > math.MaxUint32 { + panic(fmt.Errorf("unexpected block length %d; want up to %d", len(b.B), math.MaxUint32)) + } + + header := lidsBlockHeader{ + Codec: codec, + Length: uint32(len(chunk)), + MinLID: uint32(chunk[0]), + MaxLID: uint32(chunk[len(chunk)-1]), + Size: uint32(len(b.B)), + Offset: uint64(len(dst)), + } + header.marshal(dst[curHeaderOffset:]) + curHeaderOffset += int(lidsBlockHeaderSizeBytes) + + dst = append(dst, b.B...) + start += end + } + + return dst +} + +func marshalLIDsBlock(dst []byte, in []seq.LID) ([]byte, lidsCodec) { + b := lidsBlockBufPool.Get() + defer lidsBlockBufPool.Put(b) + + prev := seq.LID(0) + for i := range len(in) { + lid := in[i] + deltaLID := lid - prev + prev = lid + b.B = binary.AppendVarint(b.B, int64(deltaLID)) + } + + orig := dst + dst = zstd.CompressLevel(b.B, dst, getCompressLevel(len(b.B))) + + compressRatio := float64(len(dst)-len(orig)) / float64(len(b.B)) + if compressRatio < 1.05 { + orig = append(orig, b.B...) + return orig, lidsCodecDelta + } + + return dst, lidsCodecDeltaZstd +} + +const minLIDsFIlterBytesLen = 10 // 1 byte lidsBinVersion + 8 byte number of LIDs + N (min 1) bytes varint + delta encoded LIDs + +func unmarshalDocsFilter(dst *DocsFilterBinOut, src []byte) (_ []byte, err error) { + if len(src) < minLIDsFIlterBytesLen { + return nil, fmt.Errorf("invalid LIDs filter format; want %d bytes, got %d", minLIDsFIlterBytesLen, len(src)) + } + + version := docsFilterBinVersion(src[0]) + src = src[1:] + if _, ok := availableVersions[version]; !ok { + return nil, fmt.Errorf("invalid LIDs binary version: %d", version) + } + + dst.LIDs, src, err = unmarshalLIDsBlocks(dst.LIDs, src) + if err != nil { + return src, err + } + + return src, nil +} + +func unmarshalLIDsBlocks(dst []uint32, src []byte) ([]uint32, []byte, error) { + numberOfBlocks := binary.BigEndian.Uint32(src) + src = src[sizeOfUint32:] + + var err error + + headers := make([]lidsBlockHeader, 0, numberOfBlocks) + for range numberOfBlocks { + header := lidsBlockHeader{} + src, err = header.unmarshal(src) + if err != nil { + return dst, src, fmt.Errorf("can't unmarshal lids header: %s", err) + } + headers = append(headers, header) + } + + for i := range numberOfBlocks { + dst, src, err = unmarshalLIDsBlock(dst, src, headers[i]) + if err != nil { + return dst, src, err + } + } + + if len(src) > 0 { + return dst, src, fmt.Errorf("unexpected tail when unmarshaling LIDs blocks") + } + + return dst, src, nil +} + +func unmarshalLIDsBlock(dst []uint32, src []byte, header lidsBlockHeader) ([]uint32, []byte, error) { + if len(src) == 0 { + return dst, src, fmt.Errorf("empty LIDs block") + } + + if header.Size == 0 || int(header.Size) > len(src) { + return nil, src, fmt.Errorf("invalid LIDs block length %d; want %d", len(src), header.Size) + } + + block := src[:header.Size] + src = src[header.Size:] + + var err error + + switch header.Codec { + case lidsCodecDeltaZstd: + b := lidsBlockBufPool.Get() + defer lidsBlockBufPool.Put(b) + b.B, err = zstd.Decompress(block, b.B) + if err != nil { + return dst, src, fmt.Errorf("can't decompress ids block: %s", err) + } + dst, err = unmarshalLIDsDelta(dst, b.B, header) + if err != nil { + return dst, src, err + } + return dst, src, nil + case lidsCodecDelta: + dst, err = unmarshalLIDsDelta(dst, block, header) + if err != nil { + return dst, src, err + } + return dst, src, nil + default: + return dst, src, fmt.Errorf("unknown ids codec: %d", header.Codec) + } +} + +func unmarshalLIDsDelta(dst []uint32, block []byte, header lidsBlockHeader) ([]uint32, error) { + prevLID := uint32(0) + for range header.Length { + v, n := binary.Varint(block) + block = block[n:] + lid := prevLID + uint32(v) + prevLID = lid + dst = append(dst, lid) + } + + if len(block) > 0 { + return dst, fmt.Errorf("unexpected tail when unmarshaling LIDs block") + } + + return dst, nil +} + +func getCompressLevel(size int) int { + level := 3 + if size <= 512 { + level = 1 + } else if size <= 4*1024 { + level = 2 + } + return level +} diff --git a/docsfilter/encoding_test.go b/docsfilter/encoding_test.go new file mode 100644 index 00000000..777285fb --- /dev/null +++ b/docsfilter/encoding_test.go @@ -0,0 +1,42 @@ +package docsfilter + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/ozontech/seq-db/seq" +) + +func TestMarshalUnmarshalLIDsFilter(t *testing.T) { + test := func(df DocsFilterBinIn) { + t.Helper() + + rawDocsFilter := marshalDocsFilter(nil, &df) + var out DocsFilterBinOut + tail, err := unmarshalDocsFilter(&out, rawDocsFilter) + require.NoError(t, err) + require.Equal(t, 0, len(tail)) + assert.Equal(t, lidsToUint32s(df.LIDs), out.LIDs) + } + + test(DocsFilterBinIn{LIDs: []seq.LID{0, 1, 2, 3}}) + test(DocsFilterBinIn{LIDs: []seq.LID{10, 15, 22, 18, 105, 1010}}) + test(DocsFilterBinIn{LIDs: []seq.LID{11}}) + + multipleBlocksSize := maxLIDsBlockLen*3 + 15 + multipleBlocksLIDs := make([]seq.LID, 0, multipleBlocksSize) + for i := range multipleBlocksSize { + multipleBlocksLIDs = append(multipleBlocksLIDs, seq.LID(i)) + } + test(DocsFilterBinIn{LIDs: multipleBlocksLIDs}) +} + +func lidsToUint32s(in []seq.LID) []uint32 { + out := make([]uint32, 0, len(in)) + for _, i := range in { + out = append(out, uint32(i)) + } + return out +} diff --git a/docsfilter/filter.go b/docsfilter/filter.go new file mode 100644 index 00000000..a5bc7770 --- /dev/null +++ b/docsfilter/filter.go @@ -0,0 +1,64 @@ +package docsfilter + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "sync" + + "github.com/ozontech/seq-db/parser" +) + +type FilterStatus byte + +const ( + StatusCreated FilterStatus = iota + StatusInProgress + StatusDone + StatusError +) + +type Params struct { + Query string + From int64 + To int64 +} + +type Filter struct { + params Params + + status FilterStatus + + ast parser.SeqQLQuery + + hash string + dirPath string + + processWg *sync.WaitGroup +} + +func NewFilter(params Params) *Filter { + return &Filter{ + params: params, + status: StatusCreated, + processWg: &sync.WaitGroup{}, + } +} + +func (f *Filter) String() string { + return fmt.Sprintf("%s_%d_%d", f.params.Query, f.params.From, f.params.To) +} + +func (f *Filter) Hash() string { + if f.hash == "" { + h := sha256.New() + h.Write([]byte(f.String())) + bs := h.Sum(nil) + f.hash = hex.EncodeToString(bs) + } + return f.hash +} + +func (f *Filter) markAsDone() { + f.status = StatusDone +} diff --git a/docsfilter/metrics.go b/docsfilter/metrics.go new file mode 100644 index 00000000..de45bc40 --- /dev/null +++ b/docsfilter/metrics.go @@ -0,0 +1,26 @@ +package docsfilter + +import ( + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" +) + +var ( + inProgressFilters = promauto.NewGauge(prometheus.GaugeOpts{ + Namespace: "seq_db_store", + Subsystem: "filters", + Name: "in_progress", + Help: "Number of doc filters in progress", + }) + diskUsage = promauto.NewGauge(prometheus.GaugeOpts{ + Namespace: "seq_db_store", + Subsystem: "filters", + Name: "disk_usage_bytes", + }) + storedFilters = promauto.NewGauge(prometheus.GaugeOpts{ + Namespace: "seq_db_store", + Subsystem: "filters", + Name: "stored", + Help: "Number of active doc filters", + }) +) From 3e24d4098aaa1c5a1c871dc284dadcf08aac8b59 Mon Sep 17 00:00:00 2001 From: Daniil Forshev Date: Mon, 6 Apr 2026 18:39:06 +0500 Subject: [PATCH 02/10] fix: review fixes --- docsfilter/docs_filter.go | 413 ------------------------------------ docsfilter/encoding.go | 285 ------------------------- docsfilter/encoding_test.go | 42 ---- docsfilter/filter.go | 64 ------ docsfilter/metrics.go | 26 --- 5 files changed, 830 deletions(-) delete mode 100644 docsfilter/docs_filter.go delete mode 100644 docsfilter/encoding.go delete mode 100644 docsfilter/encoding_test.go delete mode 100644 docsfilter/filter.go delete mode 100644 docsfilter/metrics.go diff --git a/docsfilter/docs_filter.go b/docsfilter/docs_filter.go deleted file mode 100644 index 2a092f7e..00000000 --- a/docsfilter/docs_filter.go +++ /dev/null @@ -1,413 +0,0 @@ -package docsfilter - -import ( - "context" - "fmt" - "math" - "os" - "path" - "runtime" - "strings" - "sync" - "time" - - "go.uber.org/zap" - - "github.com/ozontech/seq-db/frac" - "github.com/ozontech/seq-db/frac/processor" - "github.com/ozontech/seq-db/fracmanager" - "github.com/ozontech/seq-db/logger" - "github.com/ozontech/seq-db/parser" - "github.com/ozontech/seq-db/seq" - "github.com/ozontech/seq-db/util" -) - -const ( - fracInQueueExt = ".queue" - fracDoneExt = ".filter" - tmpExt = ".tmp" -) - -const ( - defaultMaintenanceInterval = 30 * time.Second -) - -type MappingProvider interface { - GetMapping() seq.Mapping -} - -type Config struct { - DataDir string - Workers int - CacheSizeLimit uint64 -} - -type DocsFilter struct { - ctx context.Context - - config Config - filters map[string]*Filter - - fracs map[string][]string - fracsMu *sync.RWMutex - - mp MappingProvider - - rateLimit chan struct{} - createDirOnce *sync.Once - - maintenanceInterval time.Duration -} - -func New( - ctx context.Context, - cfg Config, - params []Params, - mp MappingProvider, -) *DocsFilter { - workers := cfg.Workers - if workers <= 0 { - workers = runtime.GOMAXPROCS(0) - } - - filtersMap := make(map[string]*Filter, len(params)) - - for _, p := range params { - f := NewFilter(p) - filtersMap[string(f.Hash())] = f - } - - return &DocsFilter{ - ctx: ctx, - config: cfg, - filters: filtersMap, - fracs: make(map[string][]string), - fracsMu: &sync.RWMutex{}, - mp: mp, - rateLimit: make(chan struct{}, workers), - createDirOnce: &sync.Once{}, - maintenanceInterval: defaultMaintenanceInterval, - } -} - -func (df *DocsFilter) Start(fracs fracmanager.List) { - df.createDataDir() - - err := df.loadFilters() - if err != nil { - logger.Fatal("failed to load previous docs filters", zap.Error(err)) - } - - err = df.buildQueue(fracs) - if err != nil { - logger.Fatal("failed to build docs filters queue", zap.Error(err)) - } - - go df.maintenance() - - mapping := df.mp.GetMapping() - - for _, f := range df.filters { - ast, err := parser.ParseSeqQL(f.params.Query, mapping) - if err != nil { - panic(fmt.Errorf("BUG: search query must be valid: %s", err)) - } - f.ast = ast - - df.processFilter(f, fracs.FilterInRange(seq.MID(f.params.From), seq.MID(f.params.To))) - } -} - -// RefreshFrac replaces frac's tombstone files with newly found results. Used after active frac is sealed. -func (df *DocsFilter) RefreshFrac(fraction frac.Fraction) { - df.fracsMu.RLock() - fracsFiles, has := df.fracs[fraction.Info().Name()] - df.fracsMu.RUnlock() - - if !has { - return - } - - for _, fileName := range fracsFiles { - filter := df.filters[filterNameFromTombstonesPath(fileName)] - - queueFilePath := path.Join(filter.dirPath, makeFileName(fraction.Info().Name(), fracInQueueExt)) - util.MustWriteFileAtomic(queueFilePath, []byte{}, tmpExt) - - filter.processWg.Add(1) - go func() { - if err := df.processFrac(fraction, filter, false); err != nil { - panic(fmt.Errorf("docs filter refresh frac err: %s", err)) - } - }() - } -} - -// RemoveFrac removes fraction's tombstones. Used after frac is deleted -func (df *DocsFilter) RemoveFrac(fracName string) { - df.fracsMu.RLock() - fracsFiles, has := df.fracs[fracName] - df.fracsMu.RUnlock() - - if !has { - return - } - - df.fracsMu.Lock() - delete(df.fracs, fracName) - df.fracsMu.Unlock() - - for _, fileName := range fracsFiles { - util.RemoveFile(fileName) - } -} - -func filterNameFromTombstonesPath(p string) string { - return path.Base(path.Dir(p)) -} - -func (df *DocsFilter) addDoneFrac(fracName, fracPath string) { - df.fracsMu.Lock() - defer df.fracsMu.Unlock() - - df.fracs[fracName] = append(df.fracs[fracName], fracPath) -} - -// loadFilters loads existing filters -func (df *DocsFilter) loadFilters() error { - des, err := os.ReadDir(df.config.DataDir) - if err != nil { - return err - } - - var anyRemove bool - - for _, de := range des { - if !de.IsDir() { - continue - } - - if _, ok := df.filters[de.Name()]; !ok { - logger.Info("there is filter folder on disk, but not in config. need to delete it.") - err := os.RemoveAll(path.Join(df.config.DataDir, de.Name())) - if err != nil && !os.IsNotExist(err) { - return err - } - anyRemove = true - continue - } - - f := df.filters[de.Name()] - f.status = StatusInProgress - f.dirPath = path.Join(df.config.DataDir, de.Name()) - - filterDes, err := os.ReadDir(f.dirPath) - if err != nil { - return fmt.Errorf("reading directory: %s", err) - } - - var hasFracsInQueue bool - - for _, fde := range filterDes { - if fde.IsDir() { - continue - } - name := fde.Name() - - switch path.Ext(name) { - case fracInQueueExt: - hasFracsInQueue = true - case fracDoneExt: - df.addDoneFrac(fracNameFromFilePath(name), path.Join(f.dirPath, name)) - } - } - - if !hasFracsInQueue { - f.status = StatusDone - } - } - - if anyRemove { - util.MustFsyncFile(df.config.DataDir) - } - - return nil -} - -// buildQueue creates a directory for each of unprocessed filters and creates .queue files -func (df *DocsFilter) buildQueue(fracs fracmanager.List) error { - for _, filter := range df.filters { - if filter.status != StatusCreated { - continue - } - filter.dirPath = path.Join(df.config.DataDir, filter.Hash()) - util.MustCreateDir(filter.dirPath) - - filterFracs := fracs.FilterInRange(seq.MID(filter.params.From), seq.MID(filter.params.To)) - for _, f := range filterFracs { - queueFilePath := path.Join(filter.dirPath, makeFileName(f.Info().Name(), fracInQueueExt)) - util.MustWriteFileAtomic(queueFilePath, []byte{}, tmpExt) - } - } - - return nil -} - -// handleFilter finds docs and writes to fs -func (df *DocsFilter) processFilter(filter *Filter, fracs fracmanager.List) { - if len(fracs) == 0 { - return - } - - fracsByName := make(map[string]frac.Fraction) - for _, f := range fracs { - fracsByName[f.Info().Name()] = f - } - - filterDes, err := os.ReadDir(filter.dirPath) - if err != nil { - panic(fmt.Errorf("BUG: reading directory must be successful: %s", err)) - } - - inProgressFilters.Add(1) - - processFracInQueue := func(name string) error { - f, ok := fracsByName[fracNameFromFilePath(name)] - if !ok { // skip missing fracs - return nil - } - filter.processWg.Add(1) - go func() { - if err := df.processFrac(f, filter, false); err != nil { - panic(fmt.Errorf("docs filter process frac err: %s", err)) - } - }() - return nil - } - _ = util.VisitFilesWithExt(filterDes, fracInQueueExt, processFracInQueue) - - go func() { - filter.processWg.Wait() - filter.markAsDone() - inProgressFilters.Add(-1) - }() -} - -func (df *DocsFilter) processFrac(f frac.Fraction, filter *Filter, refresh bool) error { - defer filter.processWg.Done() - - df.rateLimit <- struct{}{} - defer func() { <-df.rateLimit }() - - qpr, err := f.Search(df.ctx, processor.SearchParams{ - AST: filter.ast.Root, - From: seq.MID(filter.params.From), - To: seq.MID(filter.params.To), - Limit: math.MaxInt64, - }) - if err != nil { - return err - } - - queueFilePath := path.Join(filter.dirPath, makeFileName(f.Info().Name(), fracInQueueExt)) - doneFilePath := path.Join(filter.dirPath, makeFileName(f.Info().Name(), fracDoneExt)) - - if len(qpr.IDs) == 0 { - util.RemoveFile(queueFilePath) - return nil - } - - storeDocsFilter := func(rawDocsFilter []byte) error { - util.MustWriteFileAtomic(doneFilePath, rawDocsFilter, tmpExt) - util.RemoveFile(queueFilePath) - return nil - } - - // TODO: here we doing part of the work twice: - // first time we find LIDs inside f.Search() and then find IDs by these LIDs. - // Then we again find LIDs by earlier found IDs in f.FindLIDs(). - // We did it like this because otherwise we had to do serious f.Search() rewrite. - // For now we're ok with some performance penalty. - lids, err := f.FindLIDs(df.ctx, qpr.IDs.IDs()) - if err != nil { - return err - } - - docsFilterBin := DocsFilterBinIn{LIDs: lids} - if err := writeDocsFilter(&docsFilterBin, storeDocsFilter); err != nil { - return err - } - - if !refresh { - df.addDoneFrac(f.Info().Name(), doneFilePath) - } - - return nil -} - -func (df *DocsFilter) maintenance() { - for { - logger.Info("docs filter maintenance iteration") - df.checkDiskUsage() - time.Sleep(df.maintenanceInterval) - } -} - -func (df *DocsFilter) checkDiskUsage() { - du := int64(0) - - for _, f := range df.filters { - des, err := os.ReadDir(f.dirPath) - if err != nil { - logger.Error("docs filter: can't read filter's dir", - zap.String("filter", f.String()), zap.Error(err)) - return - } - - for _, fde := range des { - if fde.IsDir() { - continue - } - info, err := fde.Info() - if err != nil { - logger.Error("docs filter: can't read tombstones file info", - zap.String("filter", f.String()), zap.Error(err)) - return - } - du += info.Size() - } - } - - diskUsage.Set(float64(du)) - storedFilters.Set(float64(len(df.filters))) -} - -func makeFileName(name, ext string) string { - return name + ext -} - -func fracNameFromFilePath(filterFilePath string) string { - return strings.Split(path.Base(filterFilePath), ".")[0] -} - -var marshalBufferPool util.BufferPool - -func writeDocsFilter(df *DocsFilterBinIn, cb func(compressed []byte) error) error { - rawDocsFilter := marshalBufferPool.Get() - defer marshalBufferPool.Put(rawDocsFilter) - - rawDocsFilter.B = marshalDocsFilter(rawDocsFilter.B, df) - if err := cb(rawDocsFilter.B); err != nil { - return err - } - return nil -} - -// createDataDir creates dir data lazily to avoid creating extra folders. -func (df *DocsFilter) createDataDir() { - df.createDirOnce.Do(func() { - if err := os.MkdirAll(df.config.DataDir, 0o777); err != nil { - panic(err) - } - }) -} diff --git a/docsfilter/encoding.go b/docsfilter/encoding.go deleted file mode 100644 index dccfc62f..00000000 --- a/docsfilter/encoding.go +++ /dev/null @@ -1,285 +0,0 @@ -package docsfilter - -import ( - "encoding/binary" - "errors" - "fmt" - "math" - "unsafe" - - "github.com/ozontech/seq-db/seq" - "github.com/ozontech/seq-db/util" - "github.com/ozontech/seq-db/zstd" -) - -type DocsFilterBinIn struct { - LIDs []seq.LID -} - -type DocsFilterBinOut struct { - LIDs []uint32 -} - -type docsFilterBinVersion uint8 - -const ( - docsFilterBinVersion1 docsFilterBinVersion = iota + 1 -) - -var availableVersions = map[docsFilterBinVersion]struct{}{ - docsFilterBinVersion1: {}, -} - -type lidsCodec byte - -const ( - lidsCodecDelta = 1 - lidsCodecDeltaZstd = 2 -) - -type lidsBlockHeader struct { - Codec lidsCodec - Length uint32 // Number of LIDs in block - MinLID uint32 - MaxLID uint32 - Size uint32 // Size of ids block in bytes. - Offset uint64 // block's offset in file -} - -func (h *lidsBlockHeader) marshal(dst []byte) { - if len(dst) < int(lidsBlockHeaderSizeBytes) { - panic("BUG: marshal lidsBlockHeader: len(dst) is less than header size") - } - - dst[0] = byte(h.Codec) - dst = dst[1:] - binary.BigEndian.PutUint32(dst, h.Length) - dst = dst[sizeOfUint32:] - binary.BigEndian.PutUint32(dst, h.MinLID) - dst = dst[sizeOfUint32:] - binary.BigEndian.PutUint32(dst, h.MaxLID) - dst = dst[sizeOfUint32:] - binary.BigEndian.PutUint32(dst, h.Size) - dst = dst[sizeOfUint32:] - binary.BigEndian.PutUint64(dst, h.Offset) - dst = dst[sizeOfUint64:] -} - -func (h *lidsBlockHeader) unmarshal(src []byte) ([]byte, error) { - if len(src) < int(lidsBlockHeaderSizeBytes) { - return src, errors.New("too few bytes") - } - - h.Codec = lidsCodec(src[0]) - src = src[1:] - h.Length = binary.BigEndian.Uint32(src) - src = src[sizeOfUint32:] - h.MinLID = binary.BigEndian.Uint32(src) - src = src[sizeOfUint32:] - h.MaxLID = binary.BigEndian.Uint32(src) - src = src[sizeOfUint32:] - h.Size = binary.BigEndian.Uint32(src) - src = src[sizeOfUint32:] - h.Offset = binary.BigEndian.Uint64(src) - src = src[sizeOfUint64:] - - return src, nil -} - -func marshalDocsFilter(dst []byte, in *DocsFilterBinIn) []byte { - dst = append(dst, uint8(docsFilterBinVersion1)) - dst = marshalLIDsBlocks(dst, in.LIDs) - return dst -} - -const ( - sizeOfUint32 = unsafe.Sizeof(uint32(0)) - sizeOfUint64 = unsafe.Sizeof(uint64(0)) -) - -const ( - lidsBlockHeaderSizeBytes = 1 + (4 * sizeOfUint32) + sizeOfUint64 - maxLIDsBlockLen = 1024 -) - -var lidsBlockBufPool util.BufferPool - -func marshalLIDsBlocks(dst []byte, in []seq.LID) []byte { - b := lidsBlockBufPool.Get() - defer lidsBlockBufPool.Put(b) - - numberOfBlocks := (len(in) + maxLIDsBlockLen - 1) / maxLIDsBlockLen - dst = binary.BigEndian.AppendUint32(dst, uint32(numberOfBlocks)) - - // reserve space for headers - curHeaderOffset := len(dst) - dst = append(dst, make([]byte, numberOfBlocks*int(lidsBlockHeaderSizeBytes))...) - - var start int - for range numberOfBlocks { - end := min(maxLIDsBlockLen, len(in[start:])) - chunk := in[start : start+end] - - var codec lidsCodec - b.B, codec = marshalLIDsBlock(b.B[:0], chunk) - if len(b.B) > math.MaxUint32 { - panic(fmt.Errorf("unexpected block length %d; want up to %d", len(b.B), math.MaxUint32)) - } - - header := lidsBlockHeader{ - Codec: codec, - Length: uint32(len(chunk)), - MinLID: uint32(chunk[0]), - MaxLID: uint32(chunk[len(chunk)-1]), - Size: uint32(len(b.B)), - Offset: uint64(len(dst)), - } - header.marshal(dst[curHeaderOffset:]) - curHeaderOffset += int(lidsBlockHeaderSizeBytes) - - dst = append(dst, b.B...) - start += end - } - - return dst -} - -func marshalLIDsBlock(dst []byte, in []seq.LID) ([]byte, lidsCodec) { - b := lidsBlockBufPool.Get() - defer lidsBlockBufPool.Put(b) - - prev := seq.LID(0) - for i := range len(in) { - lid := in[i] - deltaLID := lid - prev - prev = lid - b.B = binary.AppendVarint(b.B, int64(deltaLID)) - } - - orig := dst - dst = zstd.CompressLevel(b.B, dst, getCompressLevel(len(b.B))) - - compressRatio := float64(len(dst)-len(orig)) / float64(len(b.B)) - if compressRatio < 1.05 { - orig = append(orig, b.B...) - return orig, lidsCodecDelta - } - - return dst, lidsCodecDeltaZstd -} - -const minLIDsFIlterBytesLen = 10 // 1 byte lidsBinVersion + 8 byte number of LIDs + N (min 1) bytes varint + delta encoded LIDs - -func unmarshalDocsFilter(dst *DocsFilterBinOut, src []byte) (_ []byte, err error) { - if len(src) < minLIDsFIlterBytesLen { - return nil, fmt.Errorf("invalid LIDs filter format; want %d bytes, got %d", minLIDsFIlterBytesLen, len(src)) - } - - version := docsFilterBinVersion(src[0]) - src = src[1:] - if _, ok := availableVersions[version]; !ok { - return nil, fmt.Errorf("invalid LIDs binary version: %d", version) - } - - dst.LIDs, src, err = unmarshalLIDsBlocks(dst.LIDs, src) - if err != nil { - return src, err - } - - return src, nil -} - -func unmarshalLIDsBlocks(dst []uint32, src []byte) ([]uint32, []byte, error) { - numberOfBlocks := binary.BigEndian.Uint32(src) - src = src[sizeOfUint32:] - - var err error - - headers := make([]lidsBlockHeader, 0, numberOfBlocks) - for range numberOfBlocks { - header := lidsBlockHeader{} - src, err = header.unmarshal(src) - if err != nil { - return dst, src, fmt.Errorf("can't unmarshal lids header: %s", err) - } - headers = append(headers, header) - } - - for i := range numberOfBlocks { - dst, src, err = unmarshalLIDsBlock(dst, src, headers[i]) - if err != nil { - return dst, src, err - } - } - - if len(src) > 0 { - return dst, src, fmt.Errorf("unexpected tail when unmarshaling LIDs blocks") - } - - return dst, src, nil -} - -func unmarshalLIDsBlock(dst []uint32, src []byte, header lidsBlockHeader) ([]uint32, []byte, error) { - if len(src) == 0 { - return dst, src, fmt.Errorf("empty LIDs block") - } - - if header.Size == 0 || int(header.Size) > len(src) { - return nil, src, fmt.Errorf("invalid LIDs block length %d; want %d", len(src), header.Size) - } - - block := src[:header.Size] - src = src[header.Size:] - - var err error - - switch header.Codec { - case lidsCodecDeltaZstd: - b := lidsBlockBufPool.Get() - defer lidsBlockBufPool.Put(b) - b.B, err = zstd.Decompress(block, b.B) - if err != nil { - return dst, src, fmt.Errorf("can't decompress ids block: %s", err) - } - dst, err = unmarshalLIDsDelta(dst, b.B, header) - if err != nil { - return dst, src, err - } - return dst, src, nil - case lidsCodecDelta: - dst, err = unmarshalLIDsDelta(dst, block, header) - if err != nil { - return dst, src, err - } - return dst, src, nil - default: - return dst, src, fmt.Errorf("unknown ids codec: %d", header.Codec) - } -} - -func unmarshalLIDsDelta(dst []uint32, block []byte, header lidsBlockHeader) ([]uint32, error) { - prevLID := uint32(0) - for range header.Length { - v, n := binary.Varint(block) - block = block[n:] - lid := prevLID + uint32(v) - prevLID = lid - dst = append(dst, lid) - } - - if len(block) > 0 { - return dst, fmt.Errorf("unexpected tail when unmarshaling LIDs block") - } - - return dst, nil -} - -func getCompressLevel(size int) int { - level := 3 - if size <= 512 { - level = 1 - } else if size <= 4*1024 { - level = 2 - } - return level -} diff --git a/docsfilter/encoding_test.go b/docsfilter/encoding_test.go deleted file mode 100644 index 777285fb..00000000 --- a/docsfilter/encoding_test.go +++ /dev/null @@ -1,42 +0,0 @@ -package docsfilter - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "github.com/ozontech/seq-db/seq" -) - -func TestMarshalUnmarshalLIDsFilter(t *testing.T) { - test := func(df DocsFilterBinIn) { - t.Helper() - - rawDocsFilter := marshalDocsFilter(nil, &df) - var out DocsFilterBinOut - tail, err := unmarshalDocsFilter(&out, rawDocsFilter) - require.NoError(t, err) - require.Equal(t, 0, len(tail)) - assert.Equal(t, lidsToUint32s(df.LIDs), out.LIDs) - } - - test(DocsFilterBinIn{LIDs: []seq.LID{0, 1, 2, 3}}) - test(DocsFilterBinIn{LIDs: []seq.LID{10, 15, 22, 18, 105, 1010}}) - test(DocsFilterBinIn{LIDs: []seq.LID{11}}) - - multipleBlocksSize := maxLIDsBlockLen*3 + 15 - multipleBlocksLIDs := make([]seq.LID, 0, multipleBlocksSize) - for i := range multipleBlocksSize { - multipleBlocksLIDs = append(multipleBlocksLIDs, seq.LID(i)) - } - test(DocsFilterBinIn{LIDs: multipleBlocksLIDs}) -} - -func lidsToUint32s(in []seq.LID) []uint32 { - out := make([]uint32, 0, len(in)) - for _, i := range in { - out = append(out, uint32(i)) - } - return out -} diff --git a/docsfilter/filter.go b/docsfilter/filter.go deleted file mode 100644 index a5bc7770..00000000 --- a/docsfilter/filter.go +++ /dev/null @@ -1,64 +0,0 @@ -package docsfilter - -import ( - "crypto/sha256" - "encoding/hex" - "fmt" - "sync" - - "github.com/ozontech/seq-db/parser" -) - -type FilterStatus byte - -const ( - StatusCreated FilterStatus = iota - StatusInProgress - StatusDone - StatusError -) - -type Params struct { - Query string - From int64 - To int64 -} - -type Filter struct { - params Params - - status FilterStatus - - ast parser.SeqQLQuery - - hash string - dirPath string - - processWg *sync.WaitGroup -} - -func NewFilter(params Params) *Filter { - return &Filter{ - params: params, - status: StatusCreated, - processWg: &sync.WaitGroup{}, - } -} - -func (f *Filter) String() string { - return fmt.Sprintf("%s_%d_%d", f.params.Query, f.params.From, f.params.To) -} - -func (f *Filter) Hash() string { - if f.hash == "" { - h := sha256.New() - h.Write([]byte(f.String())) - bs := h.Sum(nil) - f.hash = hex.EncodeToString(bs) - } - return f.hash -} - -func (f *Filter) markAsDone() { - f.status = StatusDone -} diff --git a/docsfilter/metrics.go b/docsfilter/metrics.go deleted file mode 100644 index de45bc40..00000000 --- a/docsfilter/metrics.go +++ /dev/null @@ -1,26 +0,0 @@ -package docsfilter - -import ( - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promauto" -) - -var ( - inProgressFilters = promauto.NewGauge(prometheus.GaugeOpts{ - Namespace: "seq_db_store", - Subsystem: "filters", - Name: "in_progress", - Help: "Number of doc filters in progress", - }) - diskUsage = promauto.NewGauge(prometheus.GaugeOpts{ - Namespace: "seq_db_store", - Subsystem: "filters", - Name: "disk_usage_bytes", - }) - storedFilters = promauto.NewGauge(prometheus.GaugeOpts{ - Namespace: "seq_db_store", - Subsystem: "filters", - Name: "stored", - Help: "Number of active doc filters", - }) -) From 09a823543d94c2a34123fd8c27c7d6f299f7d633 Mon Sep 17 00:00:00 2001 From: Daniil Forshev Date: Mon, 6 Apr 2026 18:57:21 +0500 Subject: [PATCH 03/10] fix: move config to last pr --- config/config.go | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/config/config.go b/config/config.go index 84e7582d..00df0d94 100644 --- a/config/config.go +++ b/config/config.go @@ -277,13 +277,13 @@ type Config struct { } `config:"tracing"` // Additional filtering options - Filtering Filter `config:"filtering"` - DocsFilter struct { - DataDir string `config:"data_dir"` - Concurrency int `config:"concurrency"` - Filters []Filter `config:"filters"` - CacheSize Bytes `config:"cache_size" default:"100MiB"` - } `config:"docs_filter"` + Filtering struct { + // If a search query time range overlaps with the [from; to] range + // the search query will be `AND`-ed with an additional predicate with the provided query expression + Query string `config:"query"` + From time.Time `config:"from"` + To time.Time `config:"to"` + } `config:"filtering"` // Experimental provides flags // For configuring experimental features. @@ -305,9 +305,3 @@ func (b *Bytes) UnmarshalString(s string) error { *b = Bytes(bytes) return nil } - -type Filter struct { - Query string `config:"query"` - From time.Time `config:"from"` - To time.Time `config:"to"` -} From 9a409998a70fef26413d405efa7805995e566fee Mon Sep 17 00:00:00 2001 From: Daniil Forshev Date: Thu, 5 Mar 2026 17:39:28 +0500 Subject: [PATCH 04/10] feat: use DocsFilter --- cmd/seq-db/seq-db.go | 20 ++++++- frac/active.go | 7 +++ frac/active_index.go | 78 ++++++++++++++++++++++++++- frac/active_indexer_test.go | 1 + frac/fraction_concurrency_test.go | 2 + frac/fraction_test.go | 25 +++++++-- frac/processor/eval_tree.go | 5 ++ frac/processor/fetch.go | 7 ++- frac/processor/search.go | 12 +++++ frac/remote.go | 8 +++ frac/sealed.go | 12 +++++ frac/sealed_index.go | 58 +++++++++++++++++++- fracmanager/fracmanager.go | 4 +- fracmanager/fracmanager_test.go | 14 ++++- fracmanager/fraction_provider.go | 18 ++++++- fracmanager/fraction_provider_test.go | 4 +- storeapi/grpc_v1_test.go | 11 ++-- storeapi/store.go | 19 ++++++- tests/setup/env.go | 3 +- 19 files changed, 285 insertions(+), 23 deletions(-) diff --git a/cmd/seq-db/seq-db.go b/cmd/seq-db/seq-db.go index 27712ee1..7c45d9f4 100644 --- a/cmd/seq-db/seq-db.go +++ b/cmd/seq-db/seq-db.go @@ -21,6 +21,7 @@ import ( "github.com/ozontech/seq-db/buildinfo" "github.com/ozontech/seq-db/config" "github.com/ozontech/seq-db/consts" + "github.com/ozontech/seq-db/docsfilter" "github.com/ozontech/seq-db/frac" "github.com/ozontech/seq-db/frac/common" "github.com/ozontech/seq-db/fracmanager" @@ -316,10 +317,15 @@ func startStore( From: cfg.Filtering.From, }, }, + Filters: docsfilter.Config{ + DataDir: cfg.DocsFilter.DataDir, + Workers: cfg.DocsFilter.Concurrency, + CacheSizeLimit: uint64(cfg.DocsFilter.CacheSize), + }, } s3cli := initS3Client(cfg) - store, err := storeapi.NewStore(ctx, sconfig, s3cli, mp) + store, err := storeapi.NewStore(ctx, sconfig, s3cli, mp, docFilterParamsFromCfg(cfg.DocsFilter.Filters)) if err != nil { logger.Fatal("initializing store", zap.Error(err)) } @@ -361,3 +367,15 @@ func initS3Client(cfg config.Config) *s3.Client { func enableIndexingForAllFields(mappingPath string) bool { return mappingPath == "auto" } + +func docFilterParamsFromCfg(in []config.Filter) []docsfilter.Params { + out := make([]docsfilter.Params, 0, len(in)) + for _, f := range in { + out = append(out, docsfilter.Params{ + Query: f.Query, + From: f.From.UnixNano(), + To: f.To.UnixNano(), + }) + } + return out +} diff --git a/frac/active.go b/frac/active.go index 3e890d01..6ca1f366 100644 --- a/frac/active.go +++ b/frac/active.go @@ -60,6 +60,8 @@ type Active struct { writer *ActiveWriter indexer *ActiveIndexer + + docsFilter DocsFilter } const ( @@ -79,6 +81,7 @@ func NewActive( docsCache *cache.Cache[[]byte], sortCache *cache.Cache[[]byte], cfg *Config, + docsFilter DocsFilter, ) *Active { docsFile, docsStats := mustOpenFile(baseFileName+consts.DocsFileSuffix, config.SkipFsync) @@ -108,6 +111,8 @@ func NewActive( BaseFileName: baseFileName, info: common.NewInfo(baseFileName, uint64(docsStats.Size()), metaSize), Config: cfg, + + docsFilter: docsFilter, } // use of 0 as keys in maps is prohibited – it's system key, so add first element @@ -412,6 +417,8 @@ func (f *Active) createDataProvider(ctx context.Context) *activeDataProvider { docsPositions: f.DocsPositions, idsToLids: f.IDsToLIDs, docsReader: &f.docsReader, + + docsFilter: f.docsFilter, } } diff --git a/frac/active_index.go b/frac/active_index.go index 71831c26..e0f753ed 100644 --- a/frac/active_index.go +++ b/frac/active_index.go @@ -2,6 +2,8 @@ package frac import ( "context" + "math" + "slices" "github.com/ozontech/seq-db/frac/common" "github.com/ozontech/seq-db/frac/processor" @@ -29,6 +31,8 @@ type activeDataProvider struct { docsReader *storage.DocsReader idsIndex *activeIDsIndex + + docsFilter DocsFilter } func (dp *activeDataProvider) release() { @@ -77,7 +81,10 @@ func (dp *activeDataProvider) Fetch(ids []seq.ID) ([][]byte, error) { indexes := []activeFetchIndex{{ blocksOffsets: dp.blocksOffsets, docsPositions: dp.docsPositions, + idsToLids: dp.idsToLids, docsReader: dp.docsReader, + docsFilter: dp.docsFilter, + fracName: dp.info.Name(), }} for _, fi := range indexes { @@ -117,6 +124,8 @@ func (dp *activeDataProvider) Search(params processor.SearchParams) (*seq.QPR, e indexes := []activeSearchIndex{{ activeIDsIndex: dp.getIDsIndex(), activeTokenIndex: dp.getTokenIndex(), + docsFilter: dp.docsFilter, + fracName: dp.info.Name(), }} m.Stop() @@ -178,6 +187,35 @@ func (p *activeIDsIndex) LessOrEqual(lid seq.LID, id seq.ID) bool { type activeSearchIndex struct { *activeIDsIndex *activeTokenIndex + docsFilter DocsFilter + fracName string +} + +func (si *activeSearchIndex) GetTombstones(minLID, maxLID uint32, reverse bool) (node.Node, error) { + // active fraction doesn't meet min and max lid + minLID, maxLID = uint32(0), uint32(math.MaxUint32) + + iterator, err := si.docsFilter.GetTombstonesIteratorByFrac(si.fracName, minLID, maxLID, reverse) + if err != nil { + return nil, err + } + + res := make([]uint32, 0) + for { + // traverse iterator to inverse and sort lids + lid := iterator.Next() + if lid.IsNull() { + break + } + if inversed, ok := si.activeIDsIndex.inverser.Inverse(lid.Unpack()); ok { + res = append(res, uint32(inversed)) + } + } + + // we need to sort inversed values since they may be out of order after replay of active fraction + slices.Sort(res) + + return node.NewStatic(res, reverse), nil } type activeTokenIndex struct { @@ -223,19 +261,55 @@ func inverseLIDs(unmapped []uint32, inv *inverser, minLID, maxLID uint32) []uint type activeFetchIndex struct { blocksOffsets []uint64 docsPositions *DocsPositions + idsToLids *ActiveLIDs docsReader *storage.DocsReader + docsFilter DocsFilter + fracName string } func (di *activeFetchIndex) GetBlocksOffsets(num uint32) uint64 { return di.blocksOffsets[num] } -func (di *activeFetchIndex) GetDocPos(ids []seq.ID) []seq.DocPos { +func (di *activeFetchIndex) GetDocPos(ids []seq.ID) ([]seq.DocPos, error) { + allLids := make([]uint32, len(ids)) + for i, id := range ids { + if lid, ok := di.idsToLids.Get(id); ok { + allLids[i] = uint32(lid) + } + } + + minLID, maxLID := uint32(0), uint32(math.MaxUint32) + tombstonesIterator, err := di.docsFilter.GetTombstonesIteratorByFrac(di.fracName, minLID, maxLID, false) + if err != nil { + return nil, err + } + + filteredLIDs := make(map[uint32]struct{}) + for { + lid := tombstonesIterator.Next() + if lid.IsNull() { + break + } + filteredLIDs[lid.Unpack()] = struct{}{} + } + docsPos := make([]seq.DocPos, len(ids)) for i, id := range ids { docsPos[i] = di.docsPositions.GetSync(id) } - return docsPos + + if len(filteredLIDs) == 0 { + return docsPos, nil + } + + for i, lid := range allLids { + if _, ok := filteredLIDs[lid]; ok { + docsPos[i] = seq.DocPosNotFound + } + } + + return docsPos, nil } func (di *activeFetchIndex) ReadDocs(blockOffset uint64, docOffsets []uint64) ([][]byte, error) { diff --git a/frac/active_indexer_test.go b/frac/active_indexer_test.go index 0b75b59e..b1164f11 100644 --- a/frac/active_indexer_test.go +++ b/frac/active_indexer_test.go @@ -90,6 +90,7 @@ func BenchmarkIndexer(b *testing.B) { cache.NewCache[[]byte](nil, nil), cache.NewCache[[]byte](nil, nil), &Config{}, + testDocsFilter{}, ) processor := getTestProcessor() diff --git a/frac/fraction_concurrency_test.go b/frac/fraction_concurrency_test.go index 70af8f46..d1f4446b 100644 --- a/frac/fraction_concurrency_test.go +++ b/frac/fraction_concurrency_test.go @@ -52,6 +52,7 @@ func TestConcurrentAppendAndQuery(t *testing.T) { cache.NewCache[[]byte](nil, nil), cache.NewCache[[]byte](nil, nil), &Config{}, + testDocsFilter{}, ) mapping := seq.Mapping{ @@ -368,6 +369,7 @@ func seal(active *Active) (*Sealed, error) { indexCache, cache.NewCache[[]byte](nil, nil), &Config{}, + testDocsFilter{}, ) active.Release() return sealed, nil diff --git a/frac/fraction_test.go b/frac/fraction_test.go index 8e44f566..83a8d27b 100644 --- a/frac/fraction_test.go +++ b/frac/fraction_test.go @@ -27,6 +27,7 @@ import ( "github.com/ozontech/seq-db/frac/sealed/seqids" "github.com/ozontech/seq-db/frac/sealed/token" "github.com/ozontech/seq-db/indexer" + "github.com/ozontech/seq-db/node" "github.com/ozontech/seq-db/parser" "github.com/ozontech/seq-db/seq" "github.com/ozontech/seq-db/storage" @@ -34,6 +35,16 @@ import ( "github.com/ozontech/seq-db/tokenizer" ) +type testDocsFilter struct{} + +func (testDocsFilter) GetFilteredLIDsByFrac(_ string) ([]uint32, error) { + return []uint32{}, nil +} +func (testDocsFilter) GetTombstonesIteratorByFrac(fracName string, minLID, maxLID uint32, reverse bool) (node.Node, error) { + return node.NewStatic([]uint32{}, false), nil +} +func (testDocsFilter) RemoveFrac(_ string) {} + type FractionTestSuite struct { suite.Suite tmpDir string @@ -2039,6 +2050,7 @@ func (s *FractionTestSuite) newActive(bulks ...[]string) *Active { cache.NewCache[[]byte](nil, nil), cache.NewCache[[]byte](nil, nil), s.config, + testDocsFilter{}, ) var wg sync.WaitGroup @@ -2101,6 +2113,7 @@ func (s *FractionTestSuite) newSealed(bulks ...[]string) *Sealed { indexCache, cache.NewCache[[]byte](nil, nil), s.config, + testDocsFilter{}, ) active.Release() return sealed @@ -2177,7 +2190,9 @@ func (s *ActiveReplayedFractionTestSuite) Replay(frac *Active) Fraction { storage.NewReadLimiter(1, nil), cache.NewCache[[]byte](nil, nil), cache.NewCache[[]byte](nil, nil), - &Config{}) + &Config{}, + testDocsFilter{}, + ) err := replayedFrac.Replay(context.Background()) s.Require().NoError(err, "replay failed") return replayedFrac @@ -2288,7 +2303,9 @@ func (s *SealedLoadedFractionTestSuite) newSealedLoaded(bulks ...[]string) *Seal indexCache, cache.NewCache[[]byte](nil, nil), nil, - s.config) + s.config, + testDocsFilter{}, + ) s.fraction = sealed return sealed } @@ -2356,7 +2373,9 @@ func (s *RemoteFractionTestSuite) SetupTest() { cache.NewCache[[]byte](nil, nil), sealed.info, s.config, - s3cli) + s3cli, + testDocsFilter{}, + ) s.fraction = remoteFrac } } diff --git a/frac/processor/eval_tree.go b/frac/processor/eval_tree.go index 0f260053..2e9fb1a6 100644 --- a/frac/processor/eval_tree.go +++ b/frac/processor/eval_tree.go @@ -88,6 +88,11 @@ func evalLeaf( return node.BuildORTree(lidsTids), nil } +func evalTombstones(root, tombstonesIterator node.Node, stats *searchStats) node.Node { + stats.NodesTotal++ + return node.NewNAnd(tombstonesIterator, root) +} + type Aggregator interface { // Next iterates to count the next lid. Next(lid node.LID) error diff --git a/frac/processor/fetch.go b/frac/processor/fetch.go index 41d46152..e2267780 100644 --- a/frac/processor/fetch.go +++ b/frac/processor/fetch.go @@ -7,13 +7,16 @@ import ( type fetchIndex interface { GetBlocksOffsets(uint32) uint64 - GetDocPos([]seq.ID) []seq.DocPos + GetDocPos([]seq.ID) ([]seq.DocPos, error) ReadDocs(blockOffset uint64, docOffsets []uint64) ([][]byte, error) } func IndexFetch(ids []seq.ID, sw *stopwatch.Stopwatch, fetchIndex fetchIndex, res [][]byte) error { m := sw.Start("get_docs_pos") - docsPos := fetchIndex.GetDocPos(ids) + docsPos, err := fetchIndex.GetDocPos(ids) + if err != nil { + return err + } blocks, offsets, index := seq.GroupDocsOffsets(docsPos) m.Stop() diff --git a/frac/processor/search.go b/frac/processor/search.go index 21fb4e8a..38270e8a 100644 --- a/frac/processor/search.go +++ b/frac/processor/search.go @@ -38,6 +38,7 @@ type tokenIndex interface { type searchIndex interface { tokenIndex idsIndex + GetTombstones(minLID, maxLID uint32, reverse bool) (node.Node, error) } func IndexSearch( @@ -93,6 +94,17 @@ func IndexSearch( } } + m = sw.Start("get_tombstones") + tombstones, err := index.GetTombstones(minLID, maxLID, params.Order.IsReverse()) + m.Stop() + if err != nil { + return nil, err + } + + m = sw.Start("eval_tombstones") + evalTree = evalTombstones(evalTree, tombstones, stats) + m.Stop() + m = sw.Start("iterate_eval_tree") total, ids, histogram, aggs, err := iterateEvalTree(ctx, params, index, evalTree, aggSupplier, sw) m.Stop() diff --git a/frac/remote.go b/frac/remote.go index e15aa73f..6f9cdb62 100644 --- a/frac/remote.go +++ b/frac/remote.go @@ -55,6 +55,8 @@ type Remote struct { s3cli *s3.Client readLimiter *storage.ReadLimiter + + docsFilter DocsFilter } func NewRemote( @@ -66,6 +68,7 @@ func NewRemote( info *common.Info, config *Config, s3cli *s3.Client, + docsFilter DocsFilter, ) *Remote { f := &Remote{ ctx: ctx, @@ -81,6 +84,8 @@ func NewRemote( Config: config, s3cli: s3cli, + + docsFilter: docsFilter, } // Fast path if fraction-info cache exists AND it has valid index size. @@ -170,6 +175,7 @@ func (f *Remote) createDataProvider(ctx context.Context) (*sealedDataProvider, e &f.blocksData.IDsTable, f.info.BinaryDataVer, ), + docsFilter: f.docsFilter, }, nil } @@ -201,6 +207,8 @@ func (f *Remote) Suicide() { zap.Error(err), ) } + + go f.docsFilter.RemoveFrac(f.info.Name()) } func (f *Remote) String() string { diff --git a/frac/sealed.go b/frac/sealed.go index ddae8ccf..8161e061 100644 --- a/frac/sealed.go +++ b/frac/sealed.go @@ -51,6 +51,8 @@ type Sealed struct { // shit for testing PartialSuicideMode PSD + + docsFilter DocsFilter } type PSD int // emulates hard shutdown on different stages of fraction deletion, used for tests @@ -68,6 +70,7 @@ func NewSealed( docsCache *cache.Cache[[]byte], info *common.Info, config *Config, + docsFilter DocsFilter, ) *Sealed { f := &Sealed{ loadMu: &sync.RWMutex{}, @@ -81,6 +84,8 @@ func NewSealed( Config: config, PartialSuicideMode: Off, + + docsFilter: docsFilter, } // fast path if fraction-info cache exists AND it has valid index size @@ -130,6 +135,7 @@ func NewSealedPreloaded( indexCache *IndexCache, docsCache *cache.Cache[[]byte], config *Config, + docsFilter DocsFilter, ) *Sealed { f := &Sealed{ blocksData: preloaded.BlocksData, @@ -144,6 +150,8 @@ func NewSealedPreloaded( info: preloaded.Info, BaseFileName: baseFile, Config: config, + + docsFilter: docsFilter, } // put the token table built during sealing into the cache of the sealed fraction @@ -292,6 +300,8 @@ func (f *Sealed) Suicide() { zap.Error(err), ) } + + go f.docsFilter.RemoveFrac(f.info.Name()) } func (f *Sealed) String() string { @@ -343,6 +353,8 @@ func (f *Sealed) createDataProvider(ctx context.Context) *sealedDataProvider { &f.blocksData.IDsTable, f.info.BinaryDataVer, ), + + docsFilter: f.docsFilter, } } diff --git a/frac/sealed_index.go b/frac/sealed_index.go index 3899124a..6b25ed79 100644 --- a/frac/sealed_index.go +++ b/frac/sealed_index.go @@ -22,6 +22,11 @@ import ( "github.com/ozontech/seq-db/util" ) +type DocsFilter interface { + GetTombstonesIteratorByFrac(fracName string, minLID, maxLID uint32, reverse bool) (node.Node, error) + RemoveFrac(fracName string) +} + type sealedDataProvider struct { ctx context.Context info *common.Info @@ -42,6 +47,8 @@ type sealedDataProvider struct { // fractionTypeLabel can be either 'sealed' or 'remote'. // This value is used in metrics to distinguish between operations over local and remote fractions. fractionTypeLabel string + + docsFilter DocsFilter } func (dp *sealedDataProvider) getIDsIndex() *sealedIDsIndex { @@ -54,9 +61,11 @@ func (dp *sealedDataProvider) getIDsIndex() *sealedIDsIndex { func (dp *sealedDataProvider) getFetchIndex() *sealedFetchIndex { return &sealedFetchIndex{ + fracName: dp.info.Name(), idsIndex: dp.getIDsIndex(), docsReader: dp.docsReader, blocksOffsets: dp.blocksOffsets, + docsFilter: dp.docsFilter, } } @@ -74,6 +83,7 @@ func (dp *sealedDataProvider) getSearchIndex() *sealedSearchIndex { return &sealedSearchIndex{ sealedIDsIndex: dp.getIDsIndex(), sealedTokenIndex: dp.getTokenIndex(), + docsFilter: dp.docsFilter, } } @@ -259,17 +269,56 @@ func (ti *sealedTokenIndex) GetLIDsFromTIDs(tids []uint32, stats lids.Counter, m } type sealedFetchIndex struct { + fracName string idsIndex *sealedIDsIndex docsReader *storage.DocsReader blocksOffsets []uint64 + docsFilter DocsFilter } func (fi *sealedFetchIndex) GetBlocksOffsets(num uint32) uint64 { return fi.blocksOffsets[num] } -func (fi *sealedFetchIndex) GetDocPos(ids []seq.ID) []seq.DocPos { - return fi.getDocPosByLIDs(fi.findLIDs(ids)) +func (fi *sealedFetchIndex) GetDocPos(ids []seq.ID) ([]seq.DocPos, error) { + allLids := fi.findLIDs(ids) + + minLID, maxLID := uint32(0), uint32(math.MaxUint32) + if len(allLids) > 0 { + // allLids can be not sorted + minVal, maxVal := allLids[0], allLids[0] + for i := 1; i < len(allLids); i++ { + minVal = min(minVal, allLids[i]) + maxVal = max(maxVal, allLids[i]) + } + minLID, maxLID = uint32(minVal), uint32(maxVal) + } + + tombstonesIterator, err := fi.docsFilter.GetTombstonesIteratorByFrac(fi.fracName, minLID, maxLID, false) + if err != nil { + return nil, err + } + + filteredLIDs := make(map[uint32]struct{}) + for { + lid := tombstonesIterator.Next() + if lid.IsNull() { + break + } + filteredLIDs[lid.Unpack()] = struct{}{} + } + + if len(filteredLIDs) == 0 { + return fi.getDocPosByLIDs(allLids), nil + } + + for i, lid := range allLids { + if _, ok := filteredLIDs[uint32(lid)]; ok { + allLids[i] = 0 + } + } + + return fi.getDocPosByLIDs(allLids), nil } func (fi *sealedFetchIndex) ReadDocs(blockOffset uint64, docOffsets []uint64) ([][]byte, error) { @@ -324,4 +373,9 @@ func (fi *sealedFetchIndex) getDocPosByLIDs(localIDs []seq.LID) []seq.DocPos { type sealedSearchIndex struct { *sealedIDsIndex *sealedTokenIndex + docsFilter DocsFilter +} + +func (si *sealedSearchIndex) GetTombstones(minLID, maxLID uint32, reverse bool) (node.Node, error) { + return si.docsFilter.GetTombstonesIteratorByFrac(si.fracName, minLID, maxLID, reverse) } diff --git a/fracmanager/fracmanager.go b/fracmanager/fracmanager.go index bd5b4f19..0cad4883 100644 --- a/fracmanager/fracmanager.go +++ b/fracmanager/fracmanager.go @@ -35,13 +35,13 @@ var defaultStorageState = StorageState{ // - stats updating // // Returns the manager instance and a stop function to gracefully shutdown -func New(ctx context.Context, cfg *Config, s3cli *s3.Client) (*FracManager, func(), error) { +func New(ctx context.Context, cfg *Config, s3cli *s3.Client, docsFilter DocsFilter) (*FracManager, func(), error) { FillConfigWithDefault(cfg) readLimiter := storage.NewReadLimiter(config.ReaderWorkers, storeBytesRead) idx, stopIdx := frac.NewActiveIndexer(config.IndexWorkers, config.IndexWorkers) cache := NewCacheMaintainer(cfg.CacheSize, cfg.SortCacheSize, newDefaultCacheMetrics()) - provider := newFractionProvider(cfg, s3cli, cache, readLimiter, idx) + provider := newFractionProvider(cfg, s3cli, cache, readLimiter, idx, docsFilter) infoCache := NewFracInfoCache(filepath.Join(cfg.DataDir, consts.FracCacheFileSuffix)) // Load existing fractions into registry diff --git a/fracmanager/fracmanager_test.go b/fracmanager/fracmanager_test.go index 89437904..a2c2b8ca 100644 --- a/fracmanager/fracmanager_test.go +++ b/fracmanager/fracmanager_test.go @@ -8,9 +8,21 @@ import ( "github.com/ozontech/seq-db/frac" "github.com/ozontech/seq-db/indexer" + "github.com/ozontech/seq-db/node" "github.com/ozontech/seq-db/seq" ) +type testDocsFilter struct{} + +func (testDocsFilter) GetFilteredLIDsByFrac(_ string) ([]uint32, error) { + return nil, nil +} +func (testDocsFilter) GetTombstonesIteratorByFrac(fracName string, minLID, maxLID uint32, reverse bool) (node.Node, error) { + return node.NewStatic([]uint32{}, reverse), nil +} +func (testDocsFilter) RefreshFrac(_ frac.Fraction) {} +func (testDocsFilter) RemoveFrac(_ string) {} + func setupDataDir(t testing.TB, cfg *Config) *Config { if cfg == nil { cfg = &Config{ @@ -25,7 +37,7 @@ func setupDataDir(t testing.TB, cfg *Config) *Config { func setupFracManager(t testing.TB, cfg *Config) (*Config, *FracManager, func()) { cfg = setupDataDir(t, cfg) - fm, stop, err := New(t.Context(), cfg, nil) + fm, stop, err := New(t.Context(), cfg, nil, testDocsFilter{}) assert.NoError(t, err) return cfg, fm, stop } diff --git a/fracmanager/fraction_provider.go b/fracmanager/fraction_provider.go index e2915598..f970912f 100644 --- a/fracmanager/fraction_provider.go +++ b/fracmanager/fraction_provider.go @@ -13,12 +13,19 @@ import ( "github.com/ozontech/seq-db/frac/common" "github.com/ozontech/seq-db/frac/sealed" "github.com/ozontech/seq-db/frac/sealed/sealing" + "github.com/ozontech/seq-db/node" "github.com/ozontech/seq-db/storage" "github.com/ozontech/seq-db/storage/s3" ) const fileBasePattern = "seq-db-" +type DocsFilter interface { + GetTombstonesIteratorByFrac(fracName string, minLID, maxLID uint32, reverse bool) (node.Node, error) + RefreshFrac(frac frac.Fraction) + RemoveFrac(fracName string) +} + // fractionProvider is a factory for creating different types of fractions // Contains all necessary dependencies for creating and managing fractions type fractionProvider struct { @@ -28,11 +35,13 @@ type fractionProvider struct { activeIndexer *frac.ActiveIndexer // Indexer for active fractions readLimiter *storage.ReadLimiter // Read rate limiter ulidEntropy io.Reader // Entropy source for ULID generation + docsFilter DocsFilter } func newFractionProvider( cfg *Config, s3cli *s3.Client, cp *CacheMaintainer, readLimiter *storage.ReadLimiter, indexer *frac.ActiveIndexer, + docsFilter DocsFilter, ) *fractionProvider { return &fractionProvider{ s3cli: s3cli, @@ -41,6 +50,7 @@ func newFractionProvider( activeIndexer: indexer, readLimiter: readLimiter, ulidEntropy: ulid.Monotonic(rand.New(rand.NewSource(time.Now().UnixNano())), 0), + docsFilter: docsFilter, } } @@ -52,6 +62,7 @@ func (fp *fractionProvider) NewActive(name string) *frac.Active { fp.cacheProvider.CreateDocBlockCache(), fp.cacheProvider.CreateSortDocsCache(), &fp.config.Fraction, + fp.docsFilter, ) } @@ -63,6 +74,7 @@ func (fp *fractionProvider) NewSealed(name string, cachedInfo *common.Info) *fra fp.cacheProvider.CreateDocBlockCache(), cachedInfo, // Preloaded meta information &fp.config.Fraction, + fp.docsFilter, ) } @@ -74,6 +86,7 @@ func (fp *fractionProvider) NewSealedPreloaded(name string, preloadedData *seale fp.cacheProvider.CreateIndexCache(), fp.cacheProvider.CreateDocBlockCache(), &fp.config.Fraction, + fp.docsFilter, ) } @@ -87,6 +100,7 @@ func (fp *fractionProvider) NewRemote(ctx context.Context, name string, cachedIn cachedInfo, &fp.config.Fraction, fp.s3cli, + fp.docsFilter, ) } @@ -117,7 +131,9 @@ func (fp *fractionProvider) Seal(active *frac.Active) (*frac.Sealed, error) { return nil, err } - return fp.NewSealedPreloaded(active.BaseFileName, preloaded), nil + sealedFrac := fp.NewSealedPreloaded(active.BaseFileName, preloaded) + fp.docsFilter.RefreshFrac(sealedFrac) + return sealedFrac, nil } // Offload uploads fraction to S3 storage and returns a remote fraction diff --git a/fracmanager/fraction_provider_test.go b/fracmanager/fraction_provider_test.go index 5dffeeee..80617aab 100644 --- a/fracmanager/fraction_provider_test.go +++ b/fracmanager/fraction_provider_test.go @@ -38,7 +38,7 @@ func setupFractionProvider(t testing.TB, cfg *Config) (*fractionProvider, func() s3cli, stopS3 := setupS3Client(t) idx, stopIdx := frac.NewActiveIndexer(1, 1) cache := NewCacheMaintainer(uint64(units.MB), uint64(units.MB), nil) - provider := newFractionProvider(cfg, s3cli, cache, rl, idx) + provider := newFractionProvider(cfg, s3cli, cache, rl, idx, testDocsFilter{}) return provider, func() { stopIdx() stopS3() @@ -46,7 +46,7 @@ func setupFractionProvider(t testing.TB, cfg *Config) (*fractionProvider, func() } func TestFractionID(t *testing.T) { - fp := newFractionProvider(nil, nil, nil, nil, nil) + fp := newFractionProvider(nil, nil, nil, nil, nil, nil) ulid1 := fp.nextFractionID() ulid2 := fp.nextFractionID() assert.NotEqual(t, ulid1, ulid2, "ULIDs should be different") diff --git a/storeapi/grpc_v1_test.go b/storeapi/grpc_v1_test.go index ced60053..9b4a5453 100644 --- a/storeapi/grpc_v1_test.go +++ b/storeapi/grpc_v1_test.go @@ -12,6 +12,7 @@ import ( "github.com/ozontech/seq-db/asyncsearcher" "github.com/ozontech/seq-db/consts" + "github.com/ozontech/seq-db/docsfilter" "github.com/ozontech/seq-db/fracmanager" "github.com/ozontech/seq-db/indexer" "github.com/ozontech/seq-db/mappingprovider" @@ -67,11 +68,16 @@ func getTestGrpc(t *testing.T) (*GrpcV1, func(), func()) { dataDir := common.GetTestTmpDir(t) common.RecreateDir(dataDir) + mappingProvider, err := mappingprovider.New("", mappingprovider.WithMapping(seq.TestMapping)) + assert.NoError(t, err) + + df := docsfilter.New(t.Context(), docsfilter.Config{}, nil, mappingProvider) + fm, stop, err := fracmanager.New(t.Context(), &fracmanager.Config{ FracSize: 500, TotalSize: 5000, DataDir: dataDir, - }, nil) + }, nil, df) assert.NoError(t, err) config := APIConfig{ @@ -92,9 +98,6 @@ func getTestGrpc(t *testing.T) (*GrpcV1, func(), func()) { }, } - mappingProvider, err := mappingprovider.New("", mappingprovider.WithMapping(seq.TestMapping)) - assert.NoError(t, err) - g := NewGrpcV1(config, fm, mappingProvider) release := func() { diff --git a/storeapi/store.go b/storeapi/store.go index 857be4e2..c41cdf21 100644 --- a/storeapi/store.go +++ b/storeapi/store.go @@ -9,6 +9,7 @@ import ( "go.uber.org/atomic" "github.com/ozontech/seq-db/consts" + "github.com/ozontech/seq-db/docsfilter" "github.com/ozontech/seq-db/fracmanager" "github.com/ozontech/seq-db/logger" "github.com/ozontech/seq-db/metric" @@ -35,6 +36,7 @@ type Store struct { type StoreConfig struct { API APIConfig FracManager fracmanager.Config + Filters docsfilter.Config } func (c *StoreConfig) setDefaults() error { @@ -44,19 +46,32 @@ func (c *StoreConfig) setDefaults() error { if c.API.Search.Async.DataDir == "" { c.API.Search.Async.DataDir = path.Join(c.FracManager.DataDir, "async_searches") } + if c.Filters.DataDir == "" { + c.Filters.DataDir = path.Join(c.FracManager.DataDir, "filters") + } return nil } -func NewStore(ctx context.Context, c StoreConfig, s3cli *s3.Client, mappingProvider MappingProvider) (*Store, error) { +func NewStore( + ctx context.Context, + c StoreConfig, + s3cli *s3.Client, + mappingProvider MappingProvider, + docFilterParams []docsfilter.Params, +) (*Store, error) { if err := c.setDefaults(); err != nil { return nil, err } - fracManager, stop, err := fracmanager.New(ctx, &c.FracManager, s3cli) + df := docsfilter.New(ctx, c.Filters, docFilterParams, mappingProvider) + + fracManager, stop, err := fracmanager.New(ctx, &c.FracManager, s3cli, df) if err != nil { return nil, fmt.Errorf("loading fractions error: %w", err) } + df.Start(fracManager.Fractions()) + return &Store{ Config: c, // We will set grpcAddr later in Start() diff --git a/tests/setup/env.go b/tests/setup/env.go index fbc66018..5144db92 100644 --- a/tests/setup/env.go +++ b/tests/setup/env.go @@ -22,6 +22,7 @@ import ( "github.com/ozontech/seq-db/buildinfo" "github.com/ozontech/seq-db/consts" + "github.com/ozontech/seq-db/docsfilter" "github.com/ozontech/seq-db/frac/common" "github.com/ozontech/seq-db/fracmanager" "github.com/ozontech/seq-db/logger" @@ -275,7 +276,7 @@ func (cfg *TestingEnvConfig) MakeStores( logger.Fatal("can't create mapping", zap.Error(err)) } - store, err := storeapi.NewStore(context.Background(), confs[i], s3cli, mappingProvider) + store, err := storeapi.NewStore(context.Background(), confs[i], s3cli, mappingProvider, []docsfilter.Params{}) if err != nil { panic(err) } From 1700881d3622d18a4ecf277621c2a936906f1b8b Mon Sep 17 00:00:00 2001 From: Daniil Forshev Date: Mon, 23 Mar 2026 16:35:28 +0500 Subject: [PATCH 05/10] chore(docs filter): add integration test --- tests/integration_tests/integration_test.go | 62 +++++++++++++++++++++ tests/setup/env.go | 6 +- 2 files changed, 67 insertions(+), 1 deletion(-) diff --git a/tests/integration_tests/integration_test.go b/tests/integration_tests/integration_test.go index cab874cf..0f12abed 100644 --- a/tests/integration_tests/integration_test.go +++ b/tests/integration_tests/integration_test.go @@ -27,6 +27,7 @@ import ( "github.com/ozontech/seq-db/asyncsearcher" "github.com/ozontech/seq-db/consts" + "github.com/ozontech/seq-db/docsfilter" "github.com/ozontech/seq-db/pkg/seqproxyapi/v1" "github.com/ozontech/seq-db/pkg/storeapi" "github.com/ozontech/seq-db/proxy/search" @@ -1751,3 +1752,64 @@ func (s *IntegrationTestSuite) TestPaginationWithOffsetId() { r.Equal(totalDocs, len(fetchedDocs), "count of unique docs does not match") } } + +func (s *IntegrationTestSuite) TestDocsFilter() { + t := s.T() + r := require.New(t) + + cfg := *s.Config + env := setup.NewTestingEnv(&cfg) + + docs := []string{ + `{"service":"visible", "message":"doc1"}`, + `{"service":"hidden", "message":"doc2"}`, + `{"service":"visible", "message":"doc3"}`, + `{"service":"hidden", "message":"doc4"}`, + } + setup.Bulk(t, env.IngestorBulkAddr(), docs) + env.WaitIdle() + env.SealAll() + + // bulk docs one more time to have sealed and active fracs + setup.Bulk(t, env.IngestorBulkAddr(), docs) + + // save hidden doc ids to test fetch later + qpr, _, _, err := env.Search(`service:hidden`, 10, setup.WithTotal(true)) + r.NoError(err) + hiddenDocIDs := qpr.IDs.IDs() + + env.WaitIdle() + env.StopAll() + + cfg.DocsFilters = []docsfilter.Params{ + { + Query: "service:hidden", + From: 0, + To: time.Now().UnixNano(), + }, + } + env = setup.NewTestingEnv(&cfg) + defer env.StopAll() + + // we don't have a convenient way to wait for doc filters processing, so just wait enough time for now + time.Sleep(2 * time.Second) + + // test search + + qpr, _, _, err = env.Search(`service:hidden`, 10, setup.WithTotal(true)) + r.NoError(err) + r.Equal(uint64(0), qpr.Total) + + qpr, _, _, err = env.Search(`service:*`, 10, setup.WithTotal(true)) + r.NoError(err) + r.Equal(uint64(4), qpr.Total) + + // test fetch + + fetchedDocs, err := env.Fetch(hiddenDocIDs) + r.NoError(err) + r.Len(fetchedDocs, len(hiddenDocIDs)) + for _, doc := range fetchedDocs { + r.Len(doc, 0) // fetch hiddenID returns nothing + } +} diff --git a/tests/setup/env.go b/tests/setup/env.go index 5144db92..bae64b06 100644 --- a/tests/setup/env.go +++ b/tests/setup/env.go @@ -49,6 +49,7 @@ type TestingEnvConfig struct { HotModeEnabled bool QueryRateLimit *float64 FracManagerConfig *fracmanager.Config + DocsFilters []docsfilter.Params Mapping seq.Mapping IndexAllFields bool @@ -123,6 +124,9 @@ func (cfg *TestingEnvConfig) GetStoreConfig(replicaID string, cold bool) storeap LogThreshold: 0, }, }, + Filters: docsfilter.Config{ + DataDir: filepath.Join(cfg.DataDir, replicaID, "filters"), + }, } } @@ -276,7 +280,7 @@ func (cfg *TestingEnvConfig) MakeStores( logger.Fatal("can't create mapping", zap.Error(err)) } - store, err := storeapi.NewStore(context.Background(), confs[i], s3cli, mappingProvider, []docsfilter.Params{}) + store, err := storeapi.NewStore(context.Background(), confs[i], s3cli, mappingProvider, cfg.DocsFilters) if err != nil { panic(err) } From 7bbe0ea185c8534624351dfcb8466d8c1fbe3025 Mon Sep 17 00:00:00 2001 From: Daniil Forshev Date: Tue, 7 Apr 2026 15:19:26 +0500 Subject: [PATCH 06/10] fix: fix after rebase --- cmd/seq-db/seq-db.go | 15 +++++++------- config/config.go | 20 ++++++++++++------- filtermanager/filter_manager.go | 7 ++++++- frac/active.go | 8 ++++---- frac/active_index.go | 20 +++++++++---------- frac/active_indexer_test.go | 2 +- frac/fraction_concurrency_test.go | 4 ++-- frac/fraction_test.go | 19 ++++++++---------- frac/processor/eval_tree.go | 4 ++-- frac/processor/search.go | 10 +++++----- frac/remote.go | 10 +++++----- frac/sealed.go | 14 ++++++------- frac/sealed_index.go | 22 ++++++++++----------- fracmanager/fracmanager.go | 4 ++-- fracmanager/fracmanager_test.go | 13 +++++------- fracmanager/fraction_provider.go | 20 +++++++++---------- fracmanager/fraction_provider_test.go | 2 +- storeapi/grpc_v1_test.go | 6 +++--- storeapi/store.go | 12 +++++------ tests/integration_tests/integration_test.go | 8 ++++---- tests/setup/env.go | 6 +++--- 21 files changed, 116 insertions(+), 110 deletions(-) diff --git a/cmd/seq-db/seq-db.go b/cmd/seq-db/seq-db.go index 7c45d9f4..bffb4aec 100644 --- a/cmd/seq-db/seq-db.go +++ b/cmd/seq-db/seq-db.go @@ -21,7 +21,7 @@ import ( "github.com/ozontech/seq-db/buildinfo" "github.com/ozontech/seq-db/config" "github.com/ozontech/seq-db/consts" - "github.com/ozontech/seq-db/docsfilter" + "github.com/ozontech/seq-db/filtermanager" "github.com/ozontech/seq-db/frac" "github.com/ozontech/seq-db/frac/common" "github.com/ozontech/seq-db/fracmanager" @@ -34,6 +34,7 @@ import ( "github.com/ozontech/seq-db/proxy/search" "github.com/ozontech/seq-db/proxy/stores" "github.com/ozontech/seq-db/proxyapi" + "github.com/ozontech/seq-db/seq" "github.com/ozontech/seq-db/storage/s3" "github.com/ozontech/seq-db/storeapi" "github.com/ozontech/seq-db/tracing" @@ -317,7 +318,7 @@ func startStore( From: cfg.Filtering.From, }, }, - Filters: docsfilter.Config{ + Filters: filtermanager.Config{ DataDir: cfg.DocsFilter.DataDir, Workers: cfg.DocsFilter.Concurrency, CacheSizeLimit: uint64(cfg.DocsFilter.CacheSize), @@ -368,13 +369,13 @@ func enableIndexingForAllFields(mappingPath string) bool { return mappingPath == "auto" } -func docFilterParamsFromCfg(in []config.Filter) []docsfilter.Params { - out := make([]docsfilter.Params, 0, len(in)) +func docFilterParamsFromCfg(in []config.Filter) []filtermanager.Params { + out := make([]filtermanager.Params, 0, len(in)) for _, f := range in { - out = append(out, docsfilter.Params{ + out = append(out, filtermanager.Params{ Query: f.Query, - From: f.From.UnixNano(), - To: f.To.UnixNano(), + From: seq.MID(f.From.UnixNano()), + To: seq.MID(f.To.UnixNano()), }) } return out diff --git a/config/config.go b/config/config.go index 00df0d94..a17e8485 100644 --- a/config/config.go +++ b/config/config.go @@ -277,13 +277,13 @@ type Config struct { } `config:"tracing"` // Additional filtering options - Filtering struct { - // If a search query time range overlaps with the [from; to] range - // the search query will be `AND`-ed with an additional predicate with the provided query expression - Query string `config:"query"` - From time.Time `config:"from"` - To time.Time `config:"to"` - } `config:"filtering"` + Filtering Filter `config:"filtering"` + DocsFilter struct { + DataDir string `config:"data_dir"` + Concurrency int `config:"concurrency"` + Filters []Filter `config:"filters"` + CacheSize Bytes `config:"cache_size" default:"100MiB"` + } `config:"docs_filter"` // Experimental provides flags // For configuring experimental features. @@ -295,6 +295,12 @@ type Config struct { } `config:"experimental"` } +type Filter struct { + Query string `config:"query"` + From time.Time `config:"from"` + To time.Time `config:"to"` +} + type Bytes units.Base2Bytes func (b *Bytes) UnmarshalString(s string) error { diff --git a/filtermanager/filter_manager.go b/filtermanager/filter_manager.go index 30805d17..d677d3d2 100644 --- a/filtermanager/filter_manager.go +++ b/filtermanager/filter_manager.go @@ -100,6 +100,7 @@ func New( fracsMu: &sync.RWMutex{}, mp: mp, rateLimit: make(chan struct{}, workers), + maintenanceWG: &sync.WaitGroup{}, maintenanceInterval: defaultMaintenanceInterval, cacheCleanInterval: defaultCacheCleanInterval, cacheGCDelay: defaultCacheGCDelay, @@ -147,7 +148,11 @@ func (fm *FilterManager) Stop() { fm.maintenanceWG.Wait() } -func (fm *FilterManager) GetHideFlagIteratorByFrac(fracName string, minLID, maxLID uint32, reverse bool) (node.Node, error) { +func (fm *FilterManager) GetHideFlagIteratorByFrac( + fracName string, + minLID, maxLID uint32, + reverse bool, +) (node.Node, error) { fm.fracsMu.RLock() defer fm.fracsMu.RUnlock() diff --git a/frac/active.go b/frac/active.go index 6ca1f366..a2db933c 100644 --- a/frac/active.go +++ b/frac/active.go @@ -61,7 +61,7 @@ type Active struct { writer *ActiveWriter indexer *ActiveIndexer - docsFilter DocsFilter + filterManager FilterManager } const ( @@ -81,7 +81,7 @@ func NewActive( docsCache *cache.Cache[[]byte], sortCache *cache.Cache[[]byte], cfg *Config, - docsFilter DocsFilter, + filterManager FilterManager, ) *Active { docsFile, docsStats := mustOpenFile(baseFileName+consts.DocsFileSuffix, config.SkipFsync) @@ -112,7 +112,7 @@ func NewActive( info: common.NewInfo(baseFileName, uint64(docsStats.Size()), metaSize), Config: cfg, - docsFilter: docsFilter, + filterManager: filterManager, } // use of 0 as keys in maps is prohibited – it's system key, so add first element @@ -418,7 +418,7 @@ func (f *Active) createDataProvider(ctx context.Context) *activeDataProvider { idsToLids: f.IDsToLIDs, docsReader: &f.docsReader, - docsFilter: f.docsFilter, + filterManager: f.filterManager, } } diff --git a/frac/active_index.go b/frac/active_index.go index e0f753ed..75496af1 100644 --- a/frac/active_index.go +++ b/frac/active_index.go @@ -32,7 +32,7 @@ type activeDataProvider struct { idsIndex *activeIDsIndex - docsFilter DocsFilter + filterManager FilterManager } func (dp *activeDataProvider) release() { @@ -83,7 +83,7 @@ func (dp *activeDataProvider) Fetch(ids []seq.ID) ([][]byte, error) { docsPositions: dp.docsPositions, idsToLids: dp.idsToLids, docsReader: dp.docsReader, - docsFilter: dp.docsFilter, + filterManager: dp.filterManager, fracName: dp.info.Name(), }} @@ -124,7 +124,7 @@ func (dp *activeDataProvider) Search(params processor.SearchParams) (*seq.QPR, e indexes := []activeSearchIndex{{ activeIDsIndex: dp.getIDsIndex(), activeTokenIndex: dp.getTokenIndex(), - docsFilter: dp.docsFilter, + filterManager: dp.filterManager, fracName: dp.info.Name(), }} m.Stop() @@ -187,15 +187,15 @@ func (p *activeIDsIndex) LessOrEqual(lid seq.LID, id seq.ID) bool { type activeSearchIndex struct { *activeIDsIndex *activeTokenIndex - docsFilter DocsFilter - fracName string + filterManager FilterManager + fracName string } -func (si *activeSearchIndex) GetTombstones(minLID, maxLID uint32, reverse bool) (node.Node, error) { +func (si *activeSearchIndex) GetHideFlags(minLID, maxLID uint32, reverse bool) (node.Node, error) { // active fraction doesn't meet min and max lid minLID, maxLID = uint32(0), uint32(math.MaxUint32) - iterator, err := si.docsFilter.GetTombstonesIteratorByFrac(si.fracName, minLID, maxLID, reverse) + iterator, err := si.filterManager.GetHideFlagIteratorByFrac(si.fracName, minLID, maxLID, reverse) if err != nil { return nil, err } @@ -263,7 +263,7 @@ type activeFetchIndex struct { docsPositions *DocsPositions idsToLids *ActiveLIDs docsReader *storage.DocsReader - docsFilter DocsFilter + filterManager FilterManager fracName string } @@ -280,14 +280,14 @@ func (di *activeFetchIndex) GetDocPos(ids []seq.ID) ([]seq.DocPos, error) { } minLID, maxLID := uint32(0), uint32(math.MaxUint32) - tombstonesIterator, err := di.docsFilter.GetTombstonesIteratorByFrac(di.fracName, minLID, maxLID, false) + hideFlagsIterator, err := di.filterManager.GetHideFlagIteratorByFrac(di.fracName, minLID, maxLID, false) if err != nil { return nil, err } filteredLIDs := make(map[uint32]struct{}) for { - lid := tombstonesIterator.Next() + lid := hideFlagsIterator.Next() if lid.IsNull() { break } diff --git a/frac/active_indexer_test.go b/frac/active_indexer_test.go index b1164f11..d1c7655b 100644 --- a/frac/active_indexer_test.go +++ b/frac/active_indexer_test.go @@ -90,7 +90,7 @@ func BenchmarkIndexer(b *testing.B) { cache.NewCache[[]byte](nil, nil), cache.NewCache[[]byte](nil, nil), &Config{}, - testDocsFilter{}, + testFilterManager{}, ) processor := getTestProcessor() diff --git a/frac/fraction_concurrency_test.go b/frac/fraction_concurrency_test.go index d1f4446b..0ad35e10 100644 --- a/frac/fraction_concurrency_test.go +++ b/frac/fraction_concurrency_test.go @@ -52,7 +52,7 @@ func TestConcurrentAppendAndQuery(t *testing.T) { cache.NewCache[[]byte](nil, nil), cache.NewCache[[]byte](nil, nil), &Config{}, - testDocsFilter{}, + testFilterManager{}, ) mapping := seq.Mapping{ @@ -369,7 +369,7 @@ func seal(active *Active) (*Sealed, error) { indexCache, cache.NewCache[[]byte](nil, nil), &Config{}, - testDocsFilter{}, + testFilterManager{}, ) active.Release() return sealed, nil diff --git a/frac/fraction_test.go b/frac/fraction_test.go index 83a8d27b..c01a65c8 100644 --- a/frac/fraction_test.go +++ b/frac/fraction_test.go @@ -35,15 +35,12 @@ import ( "github.com/ozontech/seq-db/tokenizer" ) -type testDocsFilter struct{} +type testFilterManager struct{} -func (testDocsFilter) GetFilteredLIDsByFrac(_ string) ([]uint32, error) { - return []uint32{}, nil -} -func (testDocsFilter) GetTombstonesIteratorByFrac(fracName string, minLID, maxLID uint32, reverse bool) (node.Node, error) { +func (testFilterManager) GetHideFlagIteratorByFrac(fracName string, minLID, maxLID uint32, reverse bool) (node.Node, error) { return node.NewStatic([]uint32{}, false), nil } -func (testDocsFilter) RemoveFrac(_ string) {} +func (testFilterManager) RemoveFrac(_ string) {} type FractionTestSuite struct { suite.Suite @@ -2050,7 +2047,7 @@ func (s *FractionTestSuite) newActive(bulks ...[]string) *Active { cache.NewCache[[]byte](nil, nil), cache.NewCache[[]byte](nil, nil), s.config, - testDocsFilter{}, + testFilterManager{}, ) var wg sync.WaitGroup @@ -2113,7 +2110,7 @@ func (s *FractionTestSuite) newSealed(bulks ...[]string) *Sealed { indexCache, cache.NewCache[[]byte](nil, nil), s.config, - testDocsFilter{}, + testFilterManager{}, ) active.Release() return sealed @@ -2191,7 +2188,7 @@ func (s *ActiveReplayedFractionTestSuite) Replay(frac *Active) Fraction { cache.NewCache[[]byte](nil, nil), cache.NewCache[[]byte](nil, nil), &Config{}, - testDocsFilter{}, + testFilterManager{}, ) err := replayedFrac.Replay(context.Background()) s.Require().NoError(err, "replay failed") @@ -2304,7 +2301,7 @@ func (s *SealedLoadedFractionTestSuite) newSealedLoaded(bulks ...[]string) *Seal cache.NewCache[[]byte](nil, nil), nil, s.config, - testDocsFilter{}, + testFilterManager{}, ) s.fraction = sealed return sealed @@ -2374,7 +2371,7 @@ func (s *RemoteFractionTestSuite) SetupTest() { sealed.info, s.config, s3cli, - testDocsFilter{}, + testFilterManager{}, ) s.fraction = remoteFrac } diff --git a/frac/processor/eval_tree.go b/frac/processor/eval_tree.go index 2e9fb1a6..7bdd80d9 100644 --- a/frac/processor/eval_tree.go +++ b/frac/processor/eval_tree.go @@ -88,9 +88,9 @@ func evalLeaf( return node.BuildORTree(lidsTids), nil } -func evalTombstones(root, tombstonesIterator node.Node, stats *searchStats) node.Node { +func evalHideFlags(root, hideFlagsIterator node.Node, stats *searchStats) node.Node { stats.NodesTotal++ - return node.NewNAnd(tombstonesIterator, root) + return node.NewNAnd(hideFlagsIterator, root) } type Aggregator interface { diff --git a/frac/processor/search.go b/frac/processor/search.go index 38270e8a..e4cfb0a6 100644 --- a/frac/processor/search.go +++ b/frac/processor/search.go @@ -38,7 +38,7 @@ type tokenIndex interface { type searchIndex interface { tokenIndex idsIndex - GetTombstones(minLID, maxLID uint32, reverse bool) (node.Node, error) + GetHideFlags(minLID, maxLID uint32, reverse bool) (node.Node, error) } func IndexSearch( @@ -94,15 +94,15 @@ func IndexSearch( } } - m = sw.Start("get_tombstones") - tombstones, err := index.GetTombstones(minLID, maxLID, params.Order.IsReverse()) + m = sw.Start("get_hide_flags") + hideFlags, err := index.GetHideFlags(minLID, maxLID, params.Order.IsReverse()) m.Stop() if err != nil { return nil, err } - m = sw.Start("eval_tombstones") - evalTree = evalTombstones(evalTree, tombstones, stats) + m = sw.Start("eval_hide_flags") + evalTree = evalHideFlags(evalTree, hideFlags, stats) m.Stop() m = sw.Start("iterate_eval_tree") diff --git a/frac/remote.go b/frac/remote.go index 6f9cdb62..cf79c3ce 100644 --- a/frac/remote.go +++ b/frac/remote.go @@ -56,7 +56,7 @@ type Remote struct { s3cli *s3.Client readLimiter *storage.ReadLimiter - docsFilter DocsFilter + filterManager FilterManager } func NewRemote( @@ -68,7 +68,7 @@ func NewRemote( info *common.Info, config *Config, s3cli *s3.Client, - docsFilter DocsFilter, + filterManager FilterManager, ) *Remote { f := &Remote{ ctx: ctx, @@ -85,7 +85,7 @@ func NewRemote( s3cli: s3cli, - docsFilter: docsFilter, + filterManager: filterManager, } // Fast path if fraction-info cache exists AND it has valid index size. @@ -175,7 +175,7 @@ func (f *Remote) createDataProvider(ctx context.Context) (*sealedDataProvider, e &f.blocksData.IDsTable, f.info.BinaryDataVer, ), - docsFilter: f.docsFilter, + filterManager: f.filterManager, }, nil } @@ -208,7 +208,7 @@ func (f *Remote) Suicide() { ) } - go f.docsFilter.RemoveFrac(f.info.Name()) + go f.filterManager.RemoveFrac(f.info.Name()) } func (f *Remote) String() string { diff --git a/frac/sealed.go b/frac/sealed.go index 8161e061..a5caa144 100644 --- a/frac/sealed.go +++ b/frac/sealed.go @@ -52,7 +52,7 @@ type Sealed struct { // shit for testing PartialSuicideMode PSD - docsFilter DocsFilter + filterManager FilterManager } type PSD int // emulates hard shutdown on different stages of fraction deletion, used for tests @@ -70,7 +70,7 @@ func NewSealed( docsCache *cache.Cache[[]byte], info *common.Info, config *Config, - docsFilter DocsFilter, + filterManager FilterManager, ) *Sealed { f := &Sealed{ loadMu: &sync.RWMutex{}, @@ -85,7 +85,7 @@ func NewSealed( PartialSuicideMode: Off, - docsFilter: docsFilter, + filterManager: filterManager, } // fast path if fraction-info cache exists AND it has valid index size @@ -135,7 +135,7 @@ func NewSealedPreloaded( indexCache *IndexCache, docsCache *cache.Cache[[]byte], config *Config, - docsFilter DocsFilter, + filterManager FilterManager, ) *Sealed { f := &Sealed{ blocksData: preloaded.BlocksData, @@ -151,7 +151,7 @@ func NewSealedPreloaded( BaseFileName: baseFile, Config: config, - docsFilter: docsFilter, + filterManager: filterManager, } // put the token table built during sealing into the cache of the sealed fraction @@ -301,7 +301,7 @@ func (f *Sealed) Suicide() { ) } - go f.docsFilter.RemoveFrac(f.info.Name()) + go f.filterManager.RemoveFrac(f.info.Name()) } func (f *Sealed) String() string { @@ -354,7 +354,7 @@ func (f *Sealed) createDataProvider(ctx context.Context) *sealedDataProvider { f.info.BinaryDataVer, ), - docsFilter: f.docsFilter, + filterManager: f.filterManager, } } diff --git a/frac/sealed_index.go b/frac/sealed_index.go index 6b25ed79..6ba35921 100644 --- a/frac/sealed_index.go +++ b/frac/sealed_index.go @@ -22,8 +22,8 @@ import ( "github.com/ozontech/seq-db/util" ) -type DocsFilter interface { - GetTombstonesIteratorByFrac(fracName string, minLID, maxLID uint32, reverse bool) (node.Node, error) +type FilterManager interface { + GetHideFlagIteratorByFrac(fracName string, minLID, maxLID uint32, reverse bool) (node.Node, error) RemoveFrac(fracName string) } @@ -48,7 +48,7 @@ type sealedDataProvider struct { // This value is used in metrics to distinguish between operations over local and remote fractions. fractionTypeLabel string - docsFilter DocsFilter + filterManager FilterManager } func (dp *sealedDataProvider) getIDsIndex() *sealedIDsIndex { @@ -65,7 +65,7 @@ func (dp *sealedDataProvider) getFetchIndex() *sealedFetchIndex { idsIndex: dp.getIDsIndex(), docsReader: dp.docsReader, blocksOffsets: dp.blocksOffsets, - docsFilter: dp.docsFilter, + filterManager: dp.filterManager, } } @@ -83,7 +83,7 @@ func (dp *sealedDataProvider) getSearchIndex() *sealedSearchIndex { return &sealedSearchIndex{ sealedIDsIndex: dp.getIDsIndex(), sealedTokenIndex: dp.getTokenIndex(), - docsFilter: dp.docsFilter, + filterManager: dp.filterManager, } } @@ -273,7 +273,7 @@ type sealedFetchIndex struct { idsIndex *sealedIDsIndex docsReader *storage.DocsReader blocksOffsets []uint64 - docsFilter DocsFilter + filterManager FilterManager } func (fi *sealedFetchIndex) GetBlocksOffsets(num uint32) uint64 { @@ -294,14 +294,14 @@ func (fi *sealedFetchIndex) GetDocPos(ids []seq.ID) ([]seq.DocPos, error) { minLID, maxLID = uint32(minVal), uint32(maxVal) } - tombstonesIterator, err := fi.docsFilter.GetTombstonesIteratorByFrac(fi.fracName, minLID, maxLID, false) + hideFlagsIterator, err := fi.filterManager.GetHideFlagIteratorByFrac(fi.fracName, minLID, maxLID, false) if err != nil { return nil, err } filteredLIDs := make(map[uint32]struct{}) for { - lid := tombstonesIterator.Next() + lid := hideFlagsIterator.Next() if lid.IsNull() { break } @@ -373,9 +373,9 @@ func (fi *sealedFetchIndex) getDocPosByLIDs(localIDs []seq.LID) []seq.DocPos { type sealedSearchIndex struct { *sealedIDsIndex *sealedTokenIndex - docsFilter DocsFilter + filterManager FilterManager } -func (si *sealedSearchIndex) GetTombstones(minLID, maxLID uint32, reverse bool) (node.Node, error) { - return si.docsFilter.GetTombstonesIteratorByFrac(si.fracName, minLID, maxLID, reverse) +func (si *sealedSearchIndex) GetHideFlags(minLID, maxLID uint32, reverse bool) (node.Node, error) { + return si.filterManager.GetHideFlagIteratorByFrac(si.fracName, minLID, maxLID, reverse) } diff --git a/fracmanager/fracmanager.go b/fracmanager/fracmanager.go index 0cad4883..1b28bd71 100644 --- a/fracmanager/fracmanager.go +++ b/fracmanager/fracmanager.go @@ -35,13 +35,13 @@ var defaultStorageState = StorageState{ // - stats updating // // Returns the manager instance and a stop function to gracefully shutdown -func New(ctx context.Context, cfg *Config, s3cli *s3.Client, docsFilter DocsFilter) (*FracManager, func(), error) { +func New(ctx context.Context, cfg *Config, s3cli *s3.Client, filterManager FilterManager) (*FracManager, func(), error) { FillConfigWithDefault(cfg) readLimiter := storage.NewReadLimiter(config.ReaderWorkers, storeBytesRead) idx, stopIdx := frac.NewActiveIndexer(config.IndexWorkers, config.IndexWorkers) cache := NewCacheMaintainer(cfg.CacheSize, cfg.SortCacheSize, newDefaultCacheMetrics()) - provider := newFractionProvider(cfg, s3cli, cache, readLimiter, idx, docsFilter) + provider := newFractionProvider(cfg, s3cli, cache, readLimiter, idx, filterManager) infoCache := NewFracInfoCache(filepath.Join(cfg.DataDir, consts.FracCacheFileSuffix)) // Load existing fractions into registry diff --git a/fracmanager/fracmanager_test.go b/fracmanager/fracmanager_test.go index a2c2b8ca..7ff69f6d 100644 --- a/fracmanager/fracmanager_test.go +++ b/fracmanager/fracmanager_test.go @@ -12,16 +12,13 @@ import ( "github.com/ozontech/seq-db/seq" ) -type testDocsFilter struct{} +type testFilterManager struct{} -func (testDocsFilter) GetFilteredLIDsByFrac(_ string) ([]uint32, error) { - return nil, nil -} -func (testDocsFilter) GetTombstonesIteratorByFrac(fracName string, minLID, maxLID uint32, reverse bool) (node.Node, error) { +func (testFilterManager) GetHideFlagIteratorByFrac(fracName string, minLID, maxLID uint32, reverse bool) (node.Node, error) { return node.NewStatic([]uint32{}, reverse), nil } -func (testDocsFilter) RefreshFrac(_ frac.Fraction) {} -func (testDocsFilter) RemoveFrac(_ string) {} +func (testFilterManager) RefreshFrac(_ frac.Fraction) {} +func (testFilterManager) RemoveFrac(_ string) {} func setupDataDir(t testing.TB, cfg *Config) *Config { if cfg == nil { @@ -37,7 +34,7 @@ func setupDataDir(t testing.TB, cfg *Config) *Config { func setupFracManager(t testing.TB, cfg *Config) (*Config, *FracManager, func()) { cfg = setupDataDir(t, cfg) - fm, stop, err := New(t.Context(), cfg, nil, testDocsFilter{}) + fm, stop, err := New(t.Context(), cfg, nil, testFilterManager{}) assert.NoError(t, err) return cfg, fm, stop } diff --git a/fracmanager/fraction_provider.go b/fracmanager/fraction_provider.go index f970912f..ce041055 100644 --- a/fracmanager/fraction_provider.go +++ b/fracmanager/fraction_provider.go @@ -20,8 +20,8 @@ import ( const fileBasePattern = "seq-db-" -type DocsFilter interface { - GetTombstonesIteratorByFrac(fracName string, minLID, maxLID uint32, reverse bool) (node.Node, error) +type FilterManager interface { + GetHideFlagIteratorByFrac(fracName string, minLID, maxLID uint32, reverse bool) (node.Node, error) RefreshFrac(frac frac.Fraction) RemoveFrac(fracName string) } @@ -35,13 +35,13 @@ type fractionProvider struct { activeIndexer *frac.ActiveIndexer // Indexer for active fractions readLimiter *storage.ReadLimiter // Read rate limiter ulidEntropy io.Reader // Entropy source for ULID generation - docsFilter DocsFilter + filterManager FilterManager } func newFractionProvider( cfg *Config, s3cli *s3.Client, cp *CacheMaintainer, readLimiter *storage.ReadLimiter, indexer *frac.ActiveIndexer, - docsFilter DocsFilter, + filterManager FilterManager, ) *fractionProvider { return &fractionProvider{ s3cli: s3cli, @@ -50,7 +50,7 @@ func newFractionProvider( activeIndexer: indexer, readLimiter: readLimiter, ulidEntropy: ulid.Monotonic(rand.New(rand.NewSource(time.Now().UnixNano())), 0), - docsFilter: docsFilter, + filterManager: filterManager, } } @@ -62,7 +62,7 @@ func (fp *fractionProvider) NewActive(name string) *frac.Active { fp.cacheProvider.CreateDocBlockCache(), fp.cacheProvider.CreateSortDocsCache(), &fp.config.Fraction, - fp.docsFilter, + fp.filterManager, ) } @@ -74,7 +74,7 @@ func (fp *fractionProvider) NewSealed(name string, cachedInfo *common.Info) *fra fp.cacheProvider.CreateDocBlockCache(), cachedInfo, // Preloaded meta information &fp.config.Fraction, - fp.docsFilter, + fp.filterManager, ) } @@ -86,7 +86,7 @@ func (fp *fractionProvider) NewSealedPreloaded(name string, preloadedData *seale fp.cacheProvider.CreateIndexCache(), fp.cacheProvider.CreateDocBlockCache(), &fp.config.Fraction, - fp.docsFilter, + fp.filterManager, ) } @@ -100,7 +100,7 @@ func (fp *fractionProvider) NewRemote(ctx context.Context, name string, cachedIn cachedInfo, &fp.config.Fraction, fp.s3cli, - fp.docsFilter, + fp.filterManager, ) } @@ -132,7 +132,7 @@ func (fp *fractionProvider) Seal(active *frac.Active) (*frac.Sealed, error) { } sealedFrac := fp.NewSealedPreloaded(active.BaseFileName, preloaded) - fp.docsFilter.RefreshFrac(sealedFrac) + fp.filterManager.RefreshFrac(sealedFrac) return sealedFrac, nil } diff --git a/fracmanager/fraction_provider_test.go b/fracmanager/fraction_provider_test.go index 80617aab..b965de33 100644 --- a/fracmanager/fraction_provider_test.go +++ b/fracmanager/fraction_provider_test.go @@ -38,7 +38,7 @@ func setupFractionProvider(t testing.TB, cfg *Config) (*fractionProvider, func() s3cli, stopS3 := setupS3Client(t) idx, stopIdx := frac.NewActiveIndexer(1, 1) cache := NewCacheMaintainer(uint64(units.MB), uint64(units.MB), nil) - provider := newFractionProvider(cfg, s3cli, cache, rl, idx, testDocsFilter{}) + provider := newFractionProvider(cfg, s3cli, cache, rl, idx, testFilterManager{}) return provider, func() { stopIdx() stopS3() diff --git a/storeapi/grpc_v1_test.go b/storeapi/grpc_v1_test.go index 9b4a5453..29b68f8d 100644 --- a/storeapi/grpc_v1_test.go +++ b/storeapi/grpc_v1_test.go @@ -12,7 +12,7 @@ import ( "github.com/ozontech/seq-db/asyncsearcher" "github.com/ozontech/seq-db/consts" - "github.com/ozontech/seq-db/docsfilter" + "github.com/ozontech/seq-db/filtermanager" "github.com/ozontech/seq-db/fracmanager" "github.com/ozontech/seq-db/indexer" "github.com/ozontech/seq-db/mappingprovider" @@ -71,13 +71,13 @@ func getTestGrpc(t *testing.T) (*GrpcV1, func(), func()) { mappingProvider, err := mappingprovider.New("", mappingprovider.WithMapping(seq.TestMapping)) assert.NoError(t, err) - df := docsfilter.New(t.Context(), docsfilter.Config{}, nil, mappingProvider) + filterManager := filtermanager.New(t.Context(), filtermanager.Config{}, nil, mappingProvider) fm, stop, err := fracmanager.New(t.Context(), &fracmanager.Config{ FracSize: 500, TotalSize: 5000, DataDir: dataDir, - }, nil, df) + }, nil, filterManager) assert.NoError(t, err) config := APIConfig{ diff --git a/storeapi/store.go b/storeapi/store.go index c41cdf21..6b1e2b27 100644 --- a/storeapi/store.go +++ b/storeapi/store.go @@ -9,7 +9,7 @@ import ( "go.uber.org/atomic" "github.com/ozontech/seq-db/consts" - "github.com/ozontech/seq-db/docsfilter" + "github.com/ozontech/seq-db/filtermanager" "github.com/ozontech/seq-db/fracmanager" "github.com/ozontech/seq-db/logger" "github.com/ozontech/seq-db/metric" @@ -36,7 +36,7 @@ type Store struct { type StoreConfig struct { API APIConfig FracManager fracmanager.Config - Filters docsfilter.Config + Filters filtermanager.Config } func (c *StoreConfig) setDefaults() error { @@ -57,20 +57,20 @@ func NewStore( c StoreConfig, s3cli *s3.Client, mappingProvider MappingProvider, - docFilterParams []docsfilter.Params, + docFilterParams []filtermanager.Params, ) (*Store, error) { if err := c.setDefaults(); err != nil { return nil, err } - df := docsfilter.New(ctx, c.Filters, docFilterParams, mappingProvider) + filterManager := filtermanager.New(ctx, c.Filters, docFilterParams, mappingProvider) - fracManager, stop, err := fracmanager.New(ctx, &c.FracManager, s3cli, df) + fracManager, stop, err := fracmanager.New(ctx, &c.FracManager, s3cli, filterManager) if err != nil { return nil, fmt.Errorf("loading fractions error: %w", err) } - df.Start(fracManager.Fractions()) + filterManager.Start(ctx, fracManager.Fractions()) return &Store{ Config: c, diff --git a/tests/integration_tests/integration_test.go b/tests/integration_tests/integration_test.go index 0f12abed..d17dd061 100644 --- a/tests/integration_tests/integration_test.go +++ b/tests/integration_tests/integration_test.go @@ -27,7 +27,7 @@ import ( "github.com/ozontech/seq-db/asyncsearcher" "github.com/ozontech/seq-db/consts" - "github.com/ozontech/seq-db/docsfilter" + "github.com/ozontech/seq-db/filtermanager" "github.com/ozontech/seq-db/pkg/seqproxyapi/v1" "github.com/ozontech/seq-db/pkg/storeapi" "github.com/ozontech/seq-db/proxy/search" @@ -1753,7 +1753,7 @@ func (s *IntegrationTestSuite) TestPaginationWithOffsetId() { } } -func (s *IntegrationTestSuite) TestDocsFilter() { +func (s *IntegrationTestSuite) TestFilterManager() { t := s.T() r := require.New(t) @@ -1781,11 +1781,11 @@ func (s *IntegrationTestSuite) TestDocsFilter() { env.WaitIdle() env.StopAll() - cfg.DocsFilters = []docsfilter.Params{ + cfg.DocsFilters = []filtermanager.Params{ { Query: "service:hidden", From: 0, - To: time.Now().UnixNano(), + To: seq.MID(time.Now().UnixNano()), }, } env = setup.NewTestingEnv(&cfg) diff --git a/tests/setup/env.go b/tests/setup/env.go index bae64b06..3d003c25 100644 --- a/tests/setup/env.go +++ b/tests/setup/env.go @@ -22,7 +22,7 @@ import ( "github.com/ozontech/seq-db/buildinfo" "github.com/ozontech/seq-db/consts" - "github.com/ozontech/seq-db/docsfilter" + "github.com/ozontech/seq-db/filtermanager" "github.com/ozontech/seq-db/frac/common" "github.com/ozontech/seq-db/fracmanager" "github.com/ozontech/seq-db/logger" @@ -49,7 +49,7 @@ type TestingEnvConfig struct { HotModeEnabled bool QueryRateLimit *float64 FracManagerConfig *fracmanager.Config - DocsFilters []docsfilter.Params + DocsFilters []filtermanager.Params Mapping seq.Mapping IndexAllFields bool @@ -124,7 +124,7 @@ func (cfg *TestingEnvConfig) GetStoreConfig(replicaID string, cold bool) storeap LogThreshold: 0, }, }, - Filters: docsfilter.Config{ + Filters: filtermanager.Config{ DataDir: filepath.Join(cfg.DataDir, replicaID, "filters"), }, } From 326c25e615cad8eb5167674a5e87adf70a580ae3 Mon Sep 17 00:00:00 2001 From: Daniil Forshev Date: Tue, 7 Apr 2026 15:43:46 +0500 Subject: [PATCH 07/10] fix(filter manager): review fixes --- cache/cache.go | 14 +++++++ cmd/seq-db/seq-db.go | 2 +- config/config.go | 8 ++-- filtermanager/filter_manager.go | 42 +++++++++++++-------- filtermanager/loader.go | 6 +-- frac/active_index.go | 36 +++++++++--------- frac/fraction_test.go | 4 +- frac/processor/search.go | 12 +++--- frac/sealed_index.go | 14 +++---- fracmanager/fracmanager_test.go | 4 +- fracmanager/fraction_provider.go | 4 +- tests/integration_tests/integration_test.go | 12 +++++- 12 files changed, 95 insertions(+), 63 deletions(-) diff --git a/cache/cache.go b/cache/cache.go index 451c38e4..ed8b221c 100644 --- a/cache/cache.go +++ b/cache/cache.go @@ -126,6 +126,20 @@ func (c *Cache[V]) Cleanup() uint64 { return totalFreed } +func (c *Cache[V]) Evict(key uint32) { + c.mu.Lock() + defer c.mu.Unlock() + + e, ok := c.payload[key] + if !ok { + // no key in cache + return + } + + delete(c.payload, key) + e.deleted = true +} + // Recreates the payload map. If len is too small, fraction is probably out of date and useless func (c *Cache[V]) recreatePayload() { if c.maxPayloadSize < recreateThreshold { // not large enough diff --git a/cmd/seq-db/seq-db.go b/cmd/seq-db/seq-db.go index bffb4aec..ae6be9cf 100644 --- a/cmd/seq-db/seq-db.go +++ b/cmd/seq-db/seq-db.go @@ -320,7 +320,7 @@ func startStore( }, Filters: filtermanager.Config{ DataDir: cfg.DocsFilter.DataDir, - Workers: cfg.DocsFilter.Concurrency, + Workers: cfg.DocsFilter.Workers, CacheSizeLimit: uint64(cfg.DocsFilter.CacheSize), }, } diff --git a/config/config.go b/config/config.go index a17e8485..6b57deff 100644 --- a/config/config.go +++ b/config/config.go @@ -279,10 +279,10 @@ type Config struct { // Additional filtering options Filtering Filter `config:"filtering"` DocsFilter struct { - DataDir string `config:"data_dir"` - Concurrency int `config:"concurrency"` - Filters []Filter `config:"filters"` - CacheSize Bytes `config:"cache_size" default:"100MiB"` + DataDir string `config:"data_dir"` + Workers int `config:"workers" default:"1"` + Filters []Filter `config:"filters"` + CacheSize Bytes `config:"cache_size" default:"100MiB"` } `config:"docs_filter"` // Experimental provides flags diff --git a/filtermanager/filter_manager.go b/filtermanager/filter_manager.go index d677d3d2..baba0d8c 100644 --- a/filtermanager/filter_manager.go +++ b/filtermanager/filter_manager.go @@ -3,6 +3,7 @@ package filtermanager import ( "context" "fmt" + "hash/fnv" "math" "os" "path" @@ -152,13 +153,13 @@ func (fm *FilterManager) GetHideFlagIteratorByFrac( fracName string, minLID, maxLID uint32, reverse bool, -) (node.Node, error) { +) (node.Node, bool, error) { fm.fracsMu.RLock() defer fm.fracsMu.RUnlock() fracFiles, has := fm.fracs[fracName] if !has { - return &EmptyIterator{}, nil + return &EmptyIterator{}, has, nil } iterators := make([]node.Node, 0, len(fracFiles)) @@ -166,7 +167,7 @@ func (fm *FilterManager) GetHideFlagIteratorByFrac( loader, err := newLoader(f, fm.headersCache) if err != nil { logger.Error("can't open filtered lids file", zap.String("path", f), zap.Error(err)) - return nil, err + return nil, has, err } if reverse { iterators = append(iterators, (*IteratorAsc)(NewIterator(loader, minLID, maxLID))) @@ -175,20 +176,24 @@ func (fm *FilterManager) GetHideFlagIteratorByFrac( } } - return NewNMergedIterators(iterators), nil + return NewNMergedIterators(iterators), has, nil } // RefreshFrac replaces frac's filter files with newly found results. Used after active frac is sealed. func (fm *FilterManager) RefreshFrac(fraction frac.Fraction) { - fm.fracsMu.RLock() + fm.fracsMu.Lock() fracsFiles, has := fm.fracs[fraction.Info().Name()] - fm.fracsMu.RUnlock() + delete(fm.fracs, fraction.Info().Name()) + fm.fracsMu.Unlock() if !has { return } for _, fileName := range fracsFiles { + fm.headersCache.Evict(hashFilePath(fileName)) + util.RemoveFile(fileName) + filter := fm.filters[filterNameFromPath(fileName)] queueFilePath := path.Join(filter.dirPath, makeFileName(fraction.Info().Name(), fracInQueueExt)) @@ -197,7 +202,7 @@ func (fm *FilterManager) RefreshFrac(fraction frac.Fraction) { fm.rateLimit <- struct{}{} go func() { defer func() { <-fm.rateLimit }() - if err := fm.processFrac(fraction, filter, false); err != nil { + if err := fm.processFrac(fraction, filter); err != nil { panic(fmt.Errorf("docs filter refresh frac err: %s", err)) } }() @@ -219,6 +224,7 @@ func (fm *FilterManager) RemoveFrac(fracName string) { fm.fracsMu.Unlock() for _, fileName := range fracsFiles { + fm.headersCache.Evict(hashFilePath(fileName)) util.RemoveFile(fileName) } } @@ -306,7 +312,7 @@ func (fm *FilterManager) buildQueue(fracs fracmanager.List) error { tmpDir := path.Join(fm.config.DataDir, fmt.Sprintf("%s%s", filter.Hash(), tmpDirSuffix)) util.MustCreateDir(tmpDir) - filterFracs := fracs.FilterInRange(seq.MID(filter.params.From), seq.MID(filter.params.To)) + filterFracs := fracs.FilterInRange(filter.params.From, filter.params.To) for _, f := range filterFracs { queueFilePath := path.Join(tmpDir, makeFileName(f.Info().Name(), fracInQueueExt)) util.MustWriteFileAtomic(queueFilePath, []byte{}, 0o666, tmpExt) @@ -324,7 +330,7 @@ func (fm *FilterManager) buildQueue(fracs fracmanager.List) error { return nil } -// handleFilter finds docs and writes to fs +// processFilter finds docs and writes to fs func (fm *FilterManager) processFilter(ctx context.Context, filter *Filter, fracs fracmanager.List) { if len(fracs) == 0 { return @@ -354,7 +360,7 @@ func (fm *FilterManager) processFilter(ctx context.Context, filter *Filter, frac case fm.rateLimit <- struct{}{}: filter.processWg.Go(func() { defer func() { <-fm.rateLimit }() - if err := fm.processFrac(f, filter, false); err != nil { + if err := fm.processFrac(f, filter); err != nil { panic(fmt.Errorf("docs filter process frac err: %s", err)) } }) @@ -370,11 +376,11 @@ func (fm *FilterManager) processFilter(ctx context.Context, filter *Filter, frac }() } -func (fm *FilterManager) processFrac(f frac.Fraction, filter *Filter, refresh bool) error { +func (fm *FilterManager) processFrac(f frac.Fraction, filter *Filter) error { qpr, err := f.Search(fm.ctx, processor.SearchParams{ AST: filter.ast.Root, - From: seq.MID(filter.params.From), - To: seq.MID(filter.params.To), + From: filter.params.From, + To: filter.params.To, Limit: math.MaxInt64, }) if err != nil { @@ -404,9 +410,7 @@ func (fm *FilterManager) processFrac(f frac.Fraction, filter *Filter, refresh bo return err } - if !refresh { - fm.addDoneFrac(f.Info().Name(), doneFilePath) - } + fm.addDoneFrac(f.Info().Name(), doneFilePath) return nil } @@ -477,6 +481,12 @@ func fracNameFromFilePath(filterFilePath string) string { return strings.Split(path.Base(filterFilePath), ".")[0] } +func hashFilePath(filePath string) uint32 { + hash := fnv.New32a() + hash.Write([]byte(filterNameFromPath(filePath) + fracNameFromFilePath(filePath))) + return hash.Sum32() +} + var marshalBufferPool util.BufferPool func writeDocsFilter(df *DocsFilterBinIn, queueFilePath, doneFilePath string) error { diff --git a/filtermanager/loader.go b/filtermanager/loader.go index dff30926..0c795dd9 100644 --- a/filtermanager/loader.go +++ b/filtermanager/loader.go @@ -3,7 +3,6 @@ package filtermanager import ( "encoding/binary" "fmt" - "hash/fnv" "io" "os" @@ -22,13 +21,10 @@ type loader struct { } func newLoader(filePath string, headersCache *cache.Cache[[]lidsBlockHeader]) (*loader, error) { - hash := fnv.New32a() - hash.Write([]byte(filterNameFromPath(filePath) + fracNameFromFilePath(filePath))) - return &loader{ filePath: filePath, headersCache: headersCache, - cashKey: hash.Sum32(), + cashKey: hashFilePath(filePath), }, nil } diff --git a/frac/active_index.go b/frac/active_index.go index 75496af1..4c0130c4 100644 --- a/frac/active_index.go +++ b/frac/active_index.go @@ -191,13 +191,13 @@ type activeSearchIndex struct { fracName string } -func (si *activeSearchIndex) GetHideFlags(minLID, maxLID uint32, reverse bool) (node.Node, error) { +func (si *activeSearchIndex) GetHideFlags(minLID, maxLID uint32, reverse bool) (node.Node, bool, error) { // active fraction doesn't meet min and max lid minLID, maxLID = uint32(0), uint32(math.MaxUint32) - iterator, err := si.filterManager.GetHideFlagIteratorByFrac(si.fracName, minLID, maxLID, reverse) + iterator, has, err := si.filterManager.GetHideFlagIteratorByFrac(si.fracName, minLID, maxLID, reverse) if err != nil { - return nil, err + return nil, false, err } res := make([]uint32, 0) @@ -215,7 +215,7 @@ func (si *activeSearchIndex) GetHideFlags(minLID, maxLID uint32, reverse bool) ( // we need to sort inversed values since they may be out of order after replay of active fraction slices.Sort(res) - return node.NewStatic(res, reverse), nil + return node.NewStatic(res, reverse), has, nil } type activeTokenIndex struct { @@ -272,19 +272,28 @@ func (di *activeFetchIndex) GetBlocksOffsets(num uint32) uint64 { } func (di *activeFetchIndex) GetDocPos(ids []seq.ID) ([]seq.DocPos, error) { - allLids := make([]uint32, len(ids)) + docsPos := make([]seq.DocPos, len(ids)) for i, id := range ids { - if lid, ok := di.idsToLids.Get(id); ok { - allLids[i] = uint32(lid) - } + docsPos[i] = di.docsPositions.GetSync(id) } minLID, maxLID := uint32(0), uint32(math.MaxUint32) - hideFlagsIterator, err := di.filterManager.GetHideFlagIteratorByFrac(di.fracName, minLID, maxLID, false) + hideFlagsIterator, has, err := di.filterManager.GetHideFlagIteratorByFrac(di.fracName, minLID, maxLID, false) if err != nil { return nil, err } + if !has { + return docsPos, nil + } + + allLids := make([]uint32, len(ids)) + for i, id := range ids { + if lid, ok := di.idsToLids.Get(id); ok { + allLids[i] = uint32(lid) + } + } + filteredLIDs := make(map[uint32]struct{}) for { lid := hideFlagsIterator.Next() @@ -294,15 +303,6 @@ func (di *activeFetchIndex) GetDocPos(ids []seq.ID) ([]seq.DocPos, error) { filteredLIDs[lid.Unpack()] = struct{}{} } - docsPos := make([]seq.DocPos, len(ids)) - for i, id := range ids { - docsPos[i] = di.docsPositions.GetSync(id) - } - - if len(filteredLIDs) == 0 { - return docsPos, nil - } - for i, lid := range allLids { if _, ok := filteredLIDs[lid]; ok { docsPos[i] = seq.DocPosNotFound diff --git a/frac/fraction_test.go b/frac/fraction_test.go index c01a65c8..f40b8c9d 100644 --- a/frac/fraction_test.go +++ b/frac/fraction_test.go @@ -37,8 +37,8 @@ import ( type testFilterManager struct{} -func (testFilterManager) GetHideFlagIteratorByFrac(fracName string, minLID, maxLID uint32, reverse bool) (node.Node, error) { - return node.NewStatic([]uint32{}, false), nil +func (testFilterManager) GetHideFlagIteratorByFrac(fracName string, minLID, maxLID uint32, reverse bool) (node.Node, bool, error) { + return node.NewStatic([]uint32{}, false), false, nil } func (testFilterManager) RemoveFrac(_ string) {} diff --git a/frac/processor/search.go b/frac/processor/search.go index e4cfb0a6..064a0b31 100644 --- a/frac/processor/search.go +++ b/frac/processor/search.go @@ -38,7 +38,7 @@ type tokenIndex interface { type searchIndex interface { tokenIndex idsIndex - GetHideFlags(minLID, maxLID uint32, reverse bool) (node.Node, error) + GetHideFlags(minLID, maxLID uint32, reverse bool) (node.Node, bool, error) } func IndexSearch( @@ -95,15 +95,17 @@ func IndexSearch( } m = sw.Start("get_hide_flags") - hideFlags, err := index.GetHideFlags(minLID, maxLID, params.Order.IsReverse()) + hideFlags, hasHideFlags, err := index.GetHideFlags(minLID, maxLID, params.Order.IsReverse()) m.Stop() if err != nil { return nil, err } - m = sw.Start("eval_hide_flags") - evalTree = evalHideFlags(evalTree, hideFlags, stats) - m.Stop() + if hasHideFlags { + m = sw.Start("eval_hide_flags") + evalTree = evalHideFlags(evalTree, hideFlags, stats) + m.Stop() + } m = sw.Start("iterate_eval_tree") total, ids, histogram, aggs, err := iterateEvalTree(ctx, params, index, evalTree, aggSupplier, sw) diff --git a/frac/sealed_index.go b/frac/sealed_index.go index 6ba35921..c1a0e3c5 100644 --- a/frac/sealed_index.go +++ b/frac/sealed_index.go @@ -23,7 +23,7 @@ import ( ) type FilterManager interface { - GetHideFlagIteratorByFrac(fracName string, minLID, maxLID uint32, reverse bool) (node.Node, error) + GetHideFlagIteratorByFrac(fracName string, minLID, maxLID uint32, reverse bool) (node.Node, bool, error) RemoveFrac(fracName string) } @@ -294,11 +294,15 @@ func (fi *sealedFetchIndex) GetDocPos(ids []seq.ID) ([]seq.DocPos, error) { minLID, maxLID = uint32(minVal), uint32(maxVal) } - hideFlagsIterator, err := fi.filterManager.GetHideFlagIteratorByFrac(fi.fracName, minLID, maxLID, false) + hideFlagsIterator, has, err := fi.filterManager.GetHideFlagIteratorByFrac(fi.fracName, minLID, maxLID, false) if err != nil { return nil, err } + if !has { + return fi.getDocPosByLIDs(allLids), nil + } + filteredLIDs := make(map[uint32]struct{}) for { lid := hideFlagsIterator.Next() @@ -308,10 +312,6 @@ func (fi *sealedFetchIndex) GetDocPos(ids []seq.ID) ([]seq.DocPos, error) { filteredLIDs[lid.Unpack()] = struct{}{} } - if len(filteredLIDs) == 0 { - return fi.getDocPosByLIDs(allLids), nil - } - for i, lid := range allLids { if _, ok := filteredLIDs[uint32(lid)]; ok { allLids[i] = 0 @@ -376,6 +376,6 @@ type sealedSearchIndex struct { filterManager FilterManager } -func (si *sealedSearchIndex) GetHideFlags(minLID, maxLID uint32, reverse bool) (node.Node, error) { +func (si *sealedSearchIndex) GetHideFlags(minLID, maxLID uint32, reverse bool) (node.Node, bool, error) { return si.filterManager.GetHideFlagIteratorByFrac(si.fracName, minLID, maxLID, reverse) } diff --git a/fracmanager/fracmanager_test.go b/fracmanager/fracmanager_test.go index 7ff69f6d..9eca4add 100644 --- a/fracmanager/fracmanager_test.go +++ b/fracmanager/fracmanager_test.go @@ -14,8 +14,8 @@ import ( type testFilterManager struct{} -func (testFilterManager) GetHideFlagIteratorByFrac(fracName string, minLID, maxLID uint32, reverse bool) (node.Node, error) { - return node.NewStatic([]uint32{}, reverse), nil +func (testFilterManager) GetHideFlagIteratorByFrac(fracName string, minLID, maxLID uint32, reverse bool) (node.Node, bool, error) { + return node.NewStatic([]uint32{}, reverse), false, nil } func (testFilterManager) RefreshFrac(_ frac.Fraction) {} func (testFilterManager) RemoveFrac(_ string) {} diff --git a/fracmanager/fraction_provider.go b/fracmanager/fraction_provider.go index ce041055..f2a3a694 100644 --- a/fracmanager/fraction_provider.go +++ b/fracmanager/fraction_provider.go @@ -21,7 +21,7 @@ import ( const fileBasePattern = "seq-db-" type FilterManager interface { - GetHideFlagIteratorByFrac(fracName string, minLID, maxLID uint32, reverse bool) (node.Node, error) + GetHideFlagIteratorByFrac(fracName string, minLID, maxLID uint32, reverse bool) (node.Node, bool, error) RefreshFrac(frac frac.Fraction) RemoveFrac(fracName string) } @@ -132,7 +132,7 @@ func (fp *fractionProvider) Seal(active *frac.Active) (*frac.Sealed, error) { } sealedFrac := fp.NewSealedPreloaded(active.BaseFileName, preloaded) - fp.filterManager.RefreshFrac(sealedFrac) + go fp.filterManager.RefreshFrac(sealedFrac) return sealedFrac, nil } diff --git a/tests/integration_tests/integration_test.go b/tests/integration_tests/integration_test.go index d17dd061..5f7b1127 100644 --- a/tests/integration_tests/integration_test.go +++ b/tests/integration_tests/integration_test.go @@ -1792,7 +1792,7 @@ func (s *IntegrationTestSuite) TestFilterManager() { defer env.StopAll() // we don't have a convenient way to wait for doc filters processing, so just wait enough time for now - time.Sleep(2 * time.Second) + time.Sleep(1 * time.Second) // test search @@ -1812,4 +1812,14 @@ func (s *IntegrationTestSuite) TestFilterManager() { for _, doc := range fetchedDocs { r.Len(doc, 0) // fetch hiddenID returns nothing } + + // refresh frac + + env.WaitIdle() + env.SealAll() + time.Sleep(1 * time.Second) + + qpr, _, _, err = env.Search(`service:hidden`, 10, setup.WithTotal(true)) + r.NoError(err) + r.Equal(uint64(0), qpr.Total) } From 229885df75b3bed5a0ef6cb347776bbb4a04fcca Mon Sep 17 00:00:00 2001 From: Daniil Forshev Date: Thu, 9 Apr 2026 17:14:54 +0500 Subject: [PATCH 08/10] fix(filter manager): graceful shutdown --- filtermanager/filter_manager.go | 167 ++++++++++++++++++------------- frac/remote.go | 2 +- frac/sealed.go | 2 +- fracmanager/fraction_provider.go | 2 +- storeapi/store.go | 16 +-- 5 files changed, 110 insertions(+), 79 deletions(-) diff --git a/filtermanager/filter_manager.go b/filtermanager/filter_manager.go index baba0d8c..90bc8cf0 100644 --- a/filtermanager/filter_manager.go +++ b/filtermanager/filter_manager.go @@ -2,6 +2,7 @@ package filtermanager import ( "context" + "errors" "fmt" "hash/fnv" "math" @@ -50,7 +51,8 @@ type Config struct { } type FilterManager struct { - ctx context.Context + ctx context.Context + ctxCancel context.CancelFunc config Config filters map[string]*Filter @@ -62,9 +64,8 @@ type FilterManager struct { rateLimit chan struct{} - maintenanceWG *sync.WaitGroup + bgWG *sync.WaitGroup maintenanceInterval time.Duration - maintenanceStop context.CancelFunc cacheCleanInterval time.Duration cacheGCDelay time.Duration @@ -79,6 +80,8 @@ func New( params []Params, mp MappingProvider, ) *FilterManager { + fmCtx, ctxCancel := context.WithCancel(ctx) + workers := cfg.Workers if workers <= 0 { workers = runtime.GOMAXPROCS(0) @@ -94,14 +97,15 @@ func New( cacheCleaner := cache.NewCleaner(cfg.CacheSizeLimit, nil) return &FilterManager{ - ctx: ctx, + ctx: fmCtx, + ctxCancel: ctxCancel, config: cfg, filters: filtersMap, fracs: make(map[string][]string), fracsMu: &sync.RWMutex{}, mp: mp, rateLimit: make(chan struct{}, workers), - maintenanceWG: &sync.WaitGroup{}, + bgWG: &sync.WaitGroup{}, maintenanceInterval: defaultMaintenanceInterval, cacheCleanInterval: defaultCacheCleanInterval, cacheGCDelay: defaultCacheGCDelay, @@ -110,28 +114,28 @@ func New( } } -func (fm *FilterManager) Start(ctx context.Context, fracs fracmanager.List) { +func (fm *FilterManager) Start(fracs fracmanager.List) { fm.createDataDir() err := fm.loadFilters() if err != nil { - logger.Fatal("failed to load previous docs filters", zap.Error(err)) + logger.Fatal("failed to load previous doc filters", zap.Error(err)) } err = fm.buildQueue(fracs) if err != nil { - logger.Fatal("failed to build docs filters queue", zap.Error(err)) + logger.Fatal("failed to build filter manager queue", zap.Error(err)) } - ctx, cancel := context.WithCancel(ctx) - fm.maintenanceStop = cancel - fm.startMaintenance(ctx) - - go fm.cacheCleanLoop() + fm.startMaintenance() + fm.cacheCleanLoop() mapping := fm.mp.GetMapping() + fm.bgWG.Add(1) go func() { + defer fm.bgWG.Done() + for _, f := range fm.filters { ast, err := parser.ParseSeqQL(f.params.Query, mapping) if err != nil { @@ -139,14 +143,15 @@ func (fm *FilterManager) Start(ctx context.Context, fracs fracmanager.List) { } f.ast = ast - fm.processFilter(ctx, f, fracs.FilterInRange(f.params.From, f.params.To)) + fm.processFilter(f, fracs.FilterInRange(f.params.From, f.params.To)) } }() } func (fm *FilterManager) Stop() { - fm.maintenanceStop() - fm.maintenanceWG.Wait() + fm.ctxCancel() + fm.bgWG.Wait() + logger.Info("filter manager stopped") } func (fm *FilterManager) GetHideFlagIteratorByFrac( @@ -190,43 +195,59 @@ func (fm *FilterManager) RefreshFrac(fraction frac.Fraction) { return } - for _, fileName := range fracsFiles { - fm.headersCache.Evict(hashFilePath(fileName)) - util.RemoveFile(fileName) + fm.bgWG.Add(1) + go func() { + defer fm.bgWG.Done() - filter := fm.filters[filterNameFromPath(fileName)] + for _, fileName := range fracsFiles { + fm.headersCache.Evict(hashFilePath(fileName)) + util.RemoveFile(fileName) - queueFilePath := path.Join(filter.dirPath, makeFileName(fraction.Info().Name(), fracInQueueExt)) - util.MustWriteFileAtomic(queueFilePath, []byte{}, 0o666, tmpExt) + filter := fm.filters[filterNameFromPath(fileName)] - fm.rateLimit <- struct{}{} - go func() { - defer func() { <-fm.rateLimit }() - if err := fm.processFrac(fraction, filter); err != nil { - panic(fmt.Errorf("docs filter refresh frac err: %s", err)) + queueFilePath := path.Join(filter.dirPath, makeFileName(fraction.Info().Name(), fracInQueueExt)) + util.MustWriteFileAtomic(queueFilePath, []byte{}, 0o666, tmpExt) + + select { + case <-fm.ctx.Done(): + // do not return because we have to create a .queue file for each of frac files to handle it on startup + continue + case fm.rateLimit <- struct{}{}: + go func() { + defer func() { <-fm.rateLimit }() + if err := fm.processFrac(fraction, filter); err != nil { + if errors.Is(err, context.Canceled) { + logger.Info("filter manager refresh frac context cancelled") + return + } + panic(fmt.Errorf("filter manager refresh frac err: %s", err)) + } + }() } - }() - } + } + }() } // RemoveFrac removes fraction's filter files. Used after frac is deleted func (fm *FilterManager) RemoveFrac(fracName string) { - fm.fracsMu.RLock() - fracsFiles, has := fm.fracs[fracName] - fm.fracsMu.RUnlock() + fm.bgWG.Go(func() { + fm.fracsMu.RLock() + fracsFiles, has := fm.fracs[fracName] + fm.fracsMu.RUnlock() - if !has { - return - } + if !has { + return + } - fm.fracsMu.Lock() - delete(fm.fracs, fracName) - fm.fracsMu.Unlock() + fm.fracsMu.Lock() + delete(fm.fracs, fracName) + fm.fracsMu.Unlock() - for _, fileName := range fracsFiles { - fm.headersCache.Evict(hashFilePath(fileName)) - util.RemoveFile(fileName) - } + for _, fileName := range fracsFiles { + fm.headersCache.Evict(hashFilePath(fileName)) + util.RemoveFile(fileName) + } + }) } func filterNameFromPath(p string) string { @@ -331,7 +352,7 @@ func (fm *FilterManager) buildQueue(fracs fracmanager.List) error { } // processFilter finds docs and writes to fs -func (fm *FilterManager) processFilter(ctx context.Context, filter *Filter, fracs fracmanager.List) { +func (fm *FilterManager) processFilter(filter *Filter, fracs fracmanager.List) { if len(fracs) == 0 { return } @@ -355,15 +376,21 @@ func (fm *FilterManager) processFilter(ctx context.Context, filter *Filter, frac } select { - case <-ctx.Done(): + case <-fm.ctx.Done(): return nil case fm.rateLimit <- struct{}{}: - filter.processWg.Go(func() { + filter.processWg.Add(1) + go func() { + defer filter.processWg.Done() defer func() { <-fm.rateLimit }() if err := fm.processFrac(f, filter); err != nil { - panic(fmt.Errorf("docs filter process frac err: %s", err)) + if errors.Is(err, context.Canceled) { + logger.Info("filter manager refresh frac context cancelled") + return + } + panic(fmt.Errorf("filter manager process frac err: %s", err)) } - }) + }() } return nil } @@ -415,33 +442,33 @@ func (fm *FilterManager) processFrac(f frac.Fraction, filter *Filter) error { return nil } -func (fm *FilterManager) startMaintenance(ctx context.Context) { - fm.maintenanceWG.Go(func() { - logger.Info("start docs filter maintenance") - util.RunEvery(ctx.Done(), fm.maintenanceInterval, func() { - logger.Info("docs filter maintenance iteration") +func (fm *FilterManager) startMaintenance() { + fm.bgWG.Go(func() { + logger.Info("start filter manager maintenance") + util.RunEvery(fm.ctx.Done(), fm.maintenanceInterval, func() { + logger.Info("filter manager maintenance iteration") fm.checkDiskUsage() }) }) } func (fm *FilterManager) cacheCleanLoop() { - runs := 0 - gcRunsCount := int(fm.cacheGCDelay / fm.cacheCleanInterval) - - for { - runs++ - fm.headersCacheCleaner.Cleanup(&cache.CleanStat{}) - fm.headersCacheCleaner.Rotate() - - if runs >= gcRunsCount { - runs = 0 - fm.headersCacheCleaner.CleanEmptyGenerations() - fm.headersCacheCleaner.ReleaseBuckets() - } - - time.Sleep(fm.cacheCleanInterval) - } + fm.bgWG.Go(func() { + runs := 0 + gcRunsCount := int(fm.cacheGCDelay / fm.cacheCleanInterval) + + util.RunEvery(fm.ctx.Done(), fm.cacheCleanInterval, func() { + runs++ + fm.headersCacheCleaner.Cleanup(&cache.CleanStat{}) + fm.headersCacheCleaner.Rotate() + + if runs >= gcRunsCount { + runs = 0 + fm.headersCacheCleaner.CleanEmptyGenerations() + fm.headersCacheCleaner.ReleaseBuckets() + } + }) + }) } func (fm *FilterManager) checkDiskUsage() { @@ -450,7 +477,7 @@ func (fm *FilterManager) checkDiskUsage() { for _, f := range fm.filters { des, err := os.ReadDir(f.dirPath) if err != nil { - logger.Error("docs filter: can't read filter's dir", + logger.Error("filter manager: can't read filter's dir", zap.String("filter", f.String()), zap.Error(err)) return } @@ -461,7 +488,7 @@ func (fm *FilterManager) checkDiskUsage() { } info, err := fde.Info() if err != nil { - logger.Error("docs filter: can't read filter file info", + logger.Error("filter manager: can't read filter file info", zap.String("filter", f.String()), zap.Error(err)) return } diff --git a/frac/remote.go b/frac/remote.go index cf79c3ce..44510f47 100644 --- a/frac/remote.go +++ b/frac/remote.go @@ -208,7 +208,7 @@ func (f *Remote) Suicide() { ) } - go f.filterManager.RemoveFrac(f.info.Name()) + f.filterManager.RemoveFrac(f.info.Name()) } func (f *Remote) String() string { diff --git a/frac/sealed.go b/frac/sealed.go index a5caa144..0d324348 100644 --- a/frac/sealed.go +++ b/frac/sealed.go @@ -301,7 +301,7 @@ func (f *Sealed) Suicide() { ) } - go f.filterManager.RemoveFrac(f.info.Name()) + f.filterManager.RemoveFrac(f.info.Name()) } func (f *Sealed) String() string { diff --git a/fracmanager/fraction_provider.go b/fracmanager/fraction_provider.go index f2a3a694..169a5f2d 100644 --- a/fracmanager/fraction_provider.go +++ b/fracmanager/fraction_provider.go @@ -132,7 +132,7 @@ func (fp *fractionProvider) Seal(active *frac.Active) (*frac.Sealed, error) { } sealedFrac := fp.NewSealedPreloaded(active.BaseFileName, preloaded) - go fp.filterManager.RefreshFrac(sealedFrac) + fp.filterManager.RefreshFrac(sealedFrac) return sealedFrac, nil } diff --git a/storeapi/store.go b/storeapi/store.go index 6b1e2b27..b6ac70e6 100644 --- a/storeapi/store.go +++ b/storeapi/store.go @@ -30,6 +30,8 @@ type Store struct { FracManager *fracmanager.FracManager fracManagerStop func() + filterManagerStop func() + isStopped atomic.Bool } @@ -70,16 +72,17 @@ func NewStore( return nil, fmt.Errorf("loading fractions error: %w", err) } - filterManager.Start(ctx, fracManager.Fractions()) + filterManager.Start(fracManager.Fractions()) return &Store{ Config: c, // We will set grpcAddr later in Start() - grpcAddr: "", - grpcServer: newGRPCServer(c.API, fracManager, mappingProvider), - FracManager: fracManager, - fracManagerStop: stop, - isStopped: atomic.Bool{}, + grpcAddr: "", + grpcServer: newGRPCServer(c.API, fracManager, mappingProvider), + FracManager: fracManager, + fracManagerStop: stop, + filterManagerStop: filterManager.Stop, + isStopped: atomic.Bool{}, }, nil } @@ -103,6 +106,7 @@ func (s *Store) Stop() { s.grpcServer.Stop(ctx) s.fracManagerStop() + s.filterManagerStop() logger.Info("store stopped") } From 99725b1f101e353b46dd4ea670673b20cc618380 Mon Sep 17 00:00:00 2001 From: Daniil Forshev Date: Thu, 9 Apr 2026 17:49:07 +0500 Subject: [PATCH 09/10] fix: naming fixes --- cmd/seq-db/seq-db.go | 22 +- config/config.go | 17 +- filtermanager/filter_manager.go | 535 ----------------- frac/active.go | 8 +- frac/active_index.go | 46 +- frac/active_indexer_test.go | 2 +- frac/fraction_concurrency_test.go | 4 +- frac/fraction_test.go | 16 +- frac/processor/eval_tree.go | 4 +- frac/processor/search.go | 12 +- frac/remote.go | 10 +- frac/sealed.go | 14 +- frac/sealed_index.go | 44 +- fracmanager/fracmanager.go | 4 +- fracmanager/fracmanager_test.go | 10 +- fracmanager/fraction_provider.go | 44 +- fracmanager/fraction_provider_test.go | 2 +- .../encoding.go | 30 +- .../encoding_test.go | 20 +- .../iterator.go | 2 +- .../iterator_asc.go | 4 +- .../iterator_asc_test.go | 8 +- .../iterator_desc.go | 4 +- .../iterator_desc_test.go | 8 +- {filtermanager => skipmaskmanager}/loader.go | 6 +- .../loader_test.go | 8 +- .../merged_iterator.go | 2 +- .../merged_iterator_test.go | 2 +- {filtermanager => skipmaskmanager}/metrics.go | 18 +- .../filter.go => skipmaskmanager/skip_mask.go | 38 +- skipmaskmanager/skip_mask_manager.go | 552 ++++++++++++++++++ storeapi/grpc_v1_test.go | 6 +- storeapi/store.go | 36 +- tests/integration_tests/integration_test.go | 31 +- tests/setup/env.go | 10 +- 35 files changed, 813 insertions(+), 766 deletions(-) delete mode 100644 filtermanager/filter_manager.go rename {filtermanager => skipmaskmanager}/encoding.go (87%) rename {filtermanager => skipmaskmanager}/encoding_test.go (58%) rename {filtermanager => skipmaskmanager}/iterator.go (97%) rename {filtermanager => skipmaskmanager}/iterator_asc.go (93%) rename {filtermanager => skipmaskmanager}/iterator_asc_test.go (88%) rename {filtermanager => skipmaskmanager}/iterator_desc.go (93%) rename {filtermanager => skipmaskmanager}/iterator_desc_test.go (87%) rename {filtermanager => skipmaskmanager}/loader.go (96%) rename {filtermanager => skipmaskmanager}/loader_test.go (79%) rename {filtermanager => skipmaskmanager}/merged_iterator.go (96%) rename {filtermanager => skipmaskmanager}/merged_iterator_test.go (98%) rename {filtermanager => skipmaskmanager}/metrics.go (50%) rename filtermanager/filter.go => skipmaskmanager/skip_mask.go (51%) create mode 100644 skipmaskmanager/skip_mask_manager.go diff --git a/cmd/seq-db/seq-db.go b/cmd/seq-db/seq-db.go index ae6be9cf..b0a7c47e 100644 --- a/cmd/seq-db/seq-db.go +++ b/cmd/seq-db/seq-db.go @@ -21,7 +21,6 @@ import ( "github.com/ozontech/seq-db/buildinfo" "github.com/ozontech/seq-db/config" "github.com/ozontech/seq-db/consts" - "github.com/ozontech/seq-db/filtermanager" "github.com/ozontech/seq-db/frac" "github.com/ozontech/seq-db/frac/common" "github.com/ozontech/seq-db/fracmanager" @@ -35,6 +34,7 @@ import ( "github.com/ozontech/seq-db/proxy/stores" "github.com/ozontech/seq-db/proxyapi" "github.com/ozontech/seq-db/seq" + "github.com/ozontech/seq-db/skipmaskmanager" "github.com/ozontech/seq-db/storage/s3" "github.com/ozontech/seq-db/storeapi" "github.com/ozontech/seq-db/tracing" @@ -318,15 +318,15 @@ func startStore( From: cfg.Filtering.From, }, }, - Filters: filtermanager.Config{ - DataDir: cfg.DocsFilter.DataDir, - Workers: cfg.DocsFilter.Workers, - CacheSizeLimit: uint64(cfg.DocsFilter.CacheSize), + SkipMaskManagerConfig: skipmaskmanager.Config{ + DataDir: cfg.SkipMaskManager.DataDir, + Workers: cfg.SkipMaskManager.Workers, + CacheSizeLimit: uint64(cfg.SkipMaskManager.CacheSize), }, } s3cli := initS3Client(cfg) - store, err := storeapi.NewStore(ctx, sconfig, s3cli, mp, docFilterParamsFromCfg(cfg.DocsFilter.Filters)) + store, err := storeapi.NewStore(ctx, sconfig, s3cli, mp, skipMaskParamsFromCfg(cfg.SkipMaskManager.SkipMasks)) if err != nil { logger.Fatal("initializing store", zap.Error(err)) } @@ -369,13 +369,13 @@ func enableIndexingForAllFields(mappingPath string) bool { return mappingPath == "auto" } -func docFilterParamsFromCfg(in []config.Filter) []filtermanager.Params { - out := make([]filtermanager.Params, 0, len(in)) +func skipMaskParamsFromCfg(in []config.SkipMaskParams) []skipmaskmanager.SkipMaskParams { + out := make([]skipmaskmanager.SkipMaskParams, 0, len(in)) for _, f := range in { - out = append(out, filtermanager.Params{ + out = append(out, skipmaskmanager.SkipMaskParams{ Query: f.Query, - From: seq.MID(f.From.UnixNano()), - To: seq.MID(f.To.UnixNano()), + From: seq.TimeToMID(f.From), + To: seq.TimeToMID(f.To), }) } return out diff --git a/config/config.go b/config/config.go index 6b57deff..28a67dc0 100644 --- a/config/config.go +++ b/config/config.go @@ -277,13 +277,14 @@ type Config struct { } `config:"tracing"` // Additional filtering options - Filtering Filter `config:"filtering"` - DocsFilter struct { - DataDir string `config:"data_dir"` - Workers int `config:"workers" default:"1"` - Filters []Filter `config:"filters"` - CacheSize Bytes `config:"cache_size" default:"100MiB"` - } `config:"docs_filter"` + Filtering SkipMaskParams `config:"filtering"` + + SkipMaskManager struct { + DataDir string `config:"data_dir"` + Workers int `config:"workers" default:"1"` + SkipMasks []SkipMaskParams `config:"skip_masks"` + CacheSize Bytes `config:"cache_size" default:"100MiB"` + } `config:"skip_mask_manager"` // Experimental provides flags // For configuring experimental features. @@ -295,7 +296,7 @@ type Config struct { } `config:"experimental"` } -type Filter struct { +type SkipMaskParams struct { Query string `config:"query"` From time.Time `config:"from"` To time.Time `config:"to"` diff --git a/filtermanager/filter_manager.go b/filtermanager/filter_manager.go deleted file mode 100644 index 90bc8cf0..00000000 --- a/filtermanager/filter_manager.go +++ /dev/null @@ -1,535 +0,0 @@ -package filtermanager - -import ( - "context" - "errors" - "fmt" - "hash/fnv" - "math" - "os" - "path" - "runtime" - "strings" - "sync" - "time" - - "go.uber.org/zap" - - "github.com/ozontech/seq-db/cache" - "github.com/ozontech/seq-db/frac" - "github.com/ozontech/seq-db/frac/processor" - "github.com/ozontech/seq-db/fracmanager" - "github.com/ozontech/seq-db/logger" - "github.com/ozontech/seq-db/node" - "github.com/ozontech/seq-db/parser" - "github.com/ozontech/seq-db/seq" - "github.com/ozontech/seq-db/util" -) - -const ( - fracInQueueExt = ".queue" - fracDoneExt = ".filter" - tmpExt = ".tmp" - - tmpDirSuffix = "_tmp" -) - -const ( - defaultMaintenanceInterval = 30 * time.Second - defaultCacheCleanInterval = 10 * time.Millisecond - defaultCacheGCDelay = 1 * time.Second -) - -type MappingProvider interface { - GetMapping() seq.Mapping -} - -type Config struct { - DataDir string - Workers int - CacheSizeLimit uint64 -} - -type FilterManager struct { - ctx context.Context - ctxCancel context.CancelFunc - - config Config - filters map[string]*Filter - - fracs map[string][]string - fracsMu *sync.RWMutex - - mp MappingProvider - - rateLimit chan struct{} - - bgWG *sync.WaitGroup - maintenanceInterval time.Duration - - cacheCleanInterval time.Duration - cacheGCDelay time.Duration - - headersCache *cache.Cache[[]lidsBlockHeader] - headersCacheCleaner *cache.Cleaner -} - -func New( - ctx context.Context, - cfg Config, - params []Params, - mp MappingProvider, -) *FilterManager { - fmCtx, ctxCancel := context.WithCancel(ctx) - - workers := cfg.Workers - if workers <= 0 { - workers = runtime.GOMAXPROCS(0) - } - - filtersMap := make(map[string]*Filter, len(params)) - - for _, p := range params { - f := NewFilter(p) - filtersMap[f.Hash()] = f - } - - cacheCleaner := cache.NewCleaner(cfg.CacheSizeLimit, nil) - - return &FilterManager{ - ctx: fmCtx, - ctxCancel: ctxCancel, - config: cfg, - filters: filtersMap, - fracs: make(map[string][]string), - fracsMu: &sync.RWMutex{}, - mp: mp, - rateLimit: make(chan struct{}, workers), - bgWG: &sync.WaitGroup{}, - maintenanceInterval: defaultMaintenanceInterval, - cacheCleanInterval: defaultCacheCleanInterval, - cacheGCDelay: defaultCacheGCDelay, - headersCache: cache.NewCache[[]lidsBlockHeader](cacheCleaner, nil), - headersCacheCleaner: cacheCleaner, - } -} - -func (fm *FilterManager) Start(fracs fracmanager.List) { - fm.createDataDir() - - err := fm.loadFilters() - if err != nil { - logger.Fatal("failed to load previous doc filters", zap.Error(err)) - } - - err = fm.buildQueue(fracs) - if err != nil { - logger.Fatal("failed to build filter manager queue", zap.Error(err)) - } - - fm.startMaintenance() - fm.cacheCleanLoop() - - mapping := fm.mp.GetMapping() - - fm.bgWG.Add(1) - go func() { - defer fm.bgWG.Done() - - for _, f := range fm.filters { - ast, err := parser.ParseSeqQL(f.params.Query, mapping) - if err != nil { - panic(fmt.Errorf("BUG: search query must be valid: %s", err)) - } - f.ast = ast - - fm.processFilter(f, fracs.FilterInRange(f.params.From, f.params.To)) - } - }() -} - -func (fm *FilterManager) Stop() { - fm.ctxCancel() - fm.bgWG.Wait() - logger.Info("filter manager stopped") -} - -func (fm *FilterManager) GetHideFlagIteratorByFrac( - fracName string, - minLID, maxLID uint32, - reverse bool, -) (node.Node, bool, error) { - fm.fracsMu.RLock() - defer fm.fracsMu.RUnlock() - - fracFiles, has := fm.fracs[fracName] - if !has { - return &EmptyIterator{}, has, nil - } - - iterators := make([]node.Node, 0, len(fracFiles)) - for _, f := range fracFiles { - loader, err := newLoader(f, fm.headersCache) - if err != nil { - logger.Error("can't open filtered lids file", zap.String("path", f), zap.Error(err)) - return nil, has, err - } - if reverse { - iterators = append(iterators, (*IteratorAsc)(NewIterator(loader, minLID, maxLID))) - } else { - iterators = append(iterators, (*IteratorDesc)(NewIterator(loader, minLID, maxLID))) - } - } - - return NewNMergedIterators(iterators), has, nil -} - -// RefreshFrac replaces frac's filter files with newly found results. Used after active frac is sealed. -func (fm *FilterManager) RefreshFrac(fraction frac.Fraction) { - fm.fracsMu.Lock() - fracsFiles, has := fm.fracs[fraction.Info().Name()] - delete(fm.fracs, fraction.Info().Name()) - fm.fracsMu.Unlock() - - if !has { - return - } - - fm.bgWG.Add(1) - go func() { - defer fm.bgWG.Done() - - for _, fileName := range fracsFiles { - fm.headersCache.Evict(hashFilePath(fileName)) - util.RemoveFile(fileName) - - filter := fm.filters[filterNameFromPath(fileName)] - - queueFilePath := path.Join(filter.dirPath, makeFileName(fraction.Info().Name(), fracInQueueExt)) - util.MustWriteFileAtomic(queueFilePath, []byte{}, 0o666, tmpExt) - - select { - case <-fm.ctx.Done(): - // do not return because we have to create a .queue file for each of frac files to handle it on startup - continue - case fm.rateLimit <- struct{}{}: - go func() { - defer func() { <-fm.rateLimit }() - if err := fm.processFrac(fraction, filter); err != nil { - if errors.Is(err, context.Canceled) { - logger.Info("filter manager refresh frac context cancelled") - return - } - panic(fmt.Errorf("filter manager refresh frac err: %s", err)) - } - }() - } - } - }() -} - -// RemoveFrac removes fraction's filter files. Used after frac is deleted -func (fm *FilterManager) RemoveFrac(fracName string) { - fm.bgWG.Go(func() { - fm.fracsMu.RLock() - fracsFiles, has := fm.fracs[fracName] - fm.fracsMu.RUnlock() - - if !has { - return - } - - fm.fracsMu.Lock() - delete(fm.fracs, fracName) - fm.fracsMu.Unlock() - - for _, fileName := range fracsFiles { - fm.headersCache.Evict(hashFilePath(fileName)) - util.RemoveFile(fileName) - } - }) -} - -func filterNameFromPath(p string) string { - return path.Base(path.Dir(p)) -} - -func (fm *FilterManager) addDoneFrac(fracName, fracPath string) { - fm.fracsMu.Lock() - defer fm.fracsMu.Unlock() - - fm.fracs[fracName] = append(fm.fracs[fracName], fracPath) -} - -// loadFilters loads existing filters -func (fm *FilterManager) loadFilters() error { - des, err := os.ReadDir(fm.config.DataDir) - if err != nil { - return err - } - - var anyRemove bool - - for _, de := range des { - if !de.IsDir() { - continue - } - - if _, ok := fm.filters[de.Name()]; !ok { - logger.Info("there is filter folder on disk, but not in config. need to delete it.") - err := os.RemoveAll(path.Join(fm.config.DataDir, de.Name())) - if err != nil && !os.IsNotExist(err) { - return err - } - anyRemove = true - continue - } - - f := fm.filters[de.Name()] - f.status = StatusInProgress - f.dirPath = path.Join(fm.config.DataDir, de.Name()) - - filterDes, err := os.ReadDir(f.dirPath) - if err != nil { - return fmt.Errorf("reading directory: %s", err) - } - - var hasFracsInQueue bool - - for _, fde := range filterDes { - if fde.IsDir() { - continue - } - name := fde.Name() - - switch path.Ext(name) { - case fracInQueueExt: - hasFracsInQueue = true - case fracDoneExt: - fm.addDoneFrac(fracNameFromFilePath(name), path.Join(f.dirPath, name)) - } - } - - if !hasFracsInQueue { - f.status = StatusDone - } - } - - if anyRemove { - util.MustFsyncFile(fm.config.DataDir) - } - - return nil -} - -// buildQueue creates a directory for each of unprocessed filters and creates .queue files -func (fm *FilterManager) buildQueue(fracs fracmanager.List) error { - for _, filter := range fm.filters { - if filter.status != StatusCreated { - continue - } - - // create tmp dir - tmpDir := path.Join(fm.config.DataDir, fmt.Sprintf("%s%s", filter.Hash(), tmpDirSuffix)) - util.MustCreateDir(tmpDir) - - filterFracs := fracs.FilterInRange(filter.params.From, filter.params.To) - for _, f := range filterFracs { - queueFilePath := path.Join(tmpDir, makeFileName(f.Info().Name(), fracInQueueExt)) - util.MustWriteFileAtomic(queueFilePath, []byte{}, 0o666, tmpExt) - } - - // rename tmp dir - dir := path.Join(fm.config.DataDir, filter.Hash()) - if err := os.Rename(tmpDir, dir); err != nil { - return err - } - util.MustFsyncFile(fm.config.DataDir) - filter.dirPath = dir - } - - return nil -} - -// processFilter finds docs and writes to fs -func (fm *FilterManager) processFilter(filter *Filter, fracs fracmanager.List) { - if len(fracs) == 0 { - return - } - - fracsByName := make(map[string]frac.Fraction) - for _, f := range fracs { - fracsByName[f.Info().Name()] = f - } - - filterDes, err := os.ReadDir(filter.dirPath) - if err != nil { - panic(fmt.Errorf("BUG: reading directory must be successful: %s", err)) - } - - inProgressFilters.Add(1) - - processFracInQueue := func(name string) error { - f, ok := fracsByName[fracNameFromFilePath(name)] - if !ok { // skip missing fracs - return nil - } - - select { - case <-fm.ctx.Done(): - return nil - case fm.rateLimit <- struct{}{}: - filter.processWg.Add(1) - go func() { - defer filter.processWg.Done() - defer func() { <-fm.rateLimit }() - if err := fm.processFrac(f, filter); err != nil { - if errors.Is(err, context.Canceled) { - logger.Info("filter manager refresh frac context cancelled") - return - } - panic(fmt.Errorf("filter manager process frac err: %s", err)) - } - }() - } - return nil - } - _ = util.VisitFilesWithExt(filterDes, fracInQueueExt, processFracInQueue) - - go func() { - filter.processWg.Wait() - filter.markAsDone() - inProgressFilters.Add(-1) - }() -} - -func (fm *FilterManager) processFrac(f frac.Fraction, filter *Filter) error { - qpr, err := f.Search(fm.ctx, processor.SearchParams{ - AST: filter.ast.Root, - From: filter.params.From, - To: filter.params.To, - Limit: math.MaxInt64, - }) - if err != nil { - return err - } - - queueFilePath := path.Join(filter.dirPath, makeFileName(f.Info().Name(), fracInQueueExt)) - doneFilePath := path.Join(filter.dirPath, makeFileName(f.Info().Name(), fracDoneExt)) - - if len(qpr.IDs) == 0 { - util.RemoveFile(queueFilePath) - return nil - } - - // TODO: here we doing part of the work twice: - // first time we find LIDs inside f.Search() and then find IDs by these LIDs. - // Then we again find LIDs by earlier found IDs in f.FindLIDs(). - // We did it like this because otherwise we had to do serious f.Search() rewrite. - // For now we're ok with some performance penalty. - lids, err := f.FindLIDs(fm.ctx, qpr.IDs.IDs()) - if err != nil { - return err - } - - docsFilterBin := DocsFilterBinIn{LIDs: lids} - if err := writeDocsFilter(&docsFilterBin, queueFilePath, doneFilePath); err != nil { - return err - } - - fm.addDoneFrac(f.Info().Name(), doneFilePath) - - return nil -} - -func (fm *FilterManager) startMaintenance() { - fm.bgWG.Go(func() { - logger.Info("start filter manager maintenance") - util.RunEvery(fm.ctx.Done(), fm.maintenanceInterval, func() { - logger.Info("filter manager maintenance iteration") - fm.checkDiskUsage() - }) - }) -} - -func (fm *FilterManager) cacheCleanLoop() { - fm.bgWG.Go(func() { - runs := 0 - gcRunsCount := int(fm.cacheGCDelay / fm.cacheCleanInterval) - - util.RunEvery(fm.ctx.Done(), fm.cacheCleanInterval, func() { - runs++ - fm.headersCacheCleaner.Cleanup(&cache.CleanStat{}) - fm.headersCacheCleaner.Rotate() - - if runs >= gcRunsCount { - runs = 0 - fm.headersCacheCleaner.CleanEmptyGenerations() - fm.headersCacheCleaner.ReleaseBuckets() - } - }) - }) -} - -func (fm *FilterManager) checkDiskUsage() { - du := int64(0) - - for _, f := range fm.filters { - des, err := os.ReadDir(f.dirPath) - if err != nil { - logger.Error("filter manager: can't read filter's dir", - zap.String("filter", f.String()), zap.Error(err)) - return - } - - for _, fde := range des { - if fde.IsDir() { - continue - } - info, err := fde.Info() - if err != nil { - logger.Error("filter manager: can't read filter file info", - zap.String("filter", f.String()), zap.Error(err)) - return - } - du += info.Size() - } - } - - diskUsage.Set(float64(du)) - storedFilters.Set(float64(len(fm.filters))) -} - -func makeFileName(name, ext string) string { - return name + ext -} - -func fracNameFromFilePath(filterFilePath string) string { - return strings.Split(path.Base(filterFilePath), ".")[0] -} - -func hashFilePath(filePath string) uint32 { - hash := fnv.New32a() - hash.Write([]byte(filterNameFromPath(filePath) + fracNameFromFilePath(filePath))) - return hash.Sum32() -} - -var marshalBufferPool util.BufferPool - -func writeDocsFilter(df *DocsFilterBinIn, queueFilePath, doneFilePath string) error { - rawDocsFilter := marshalBufferPool.Get() - defer marshalBufferPool.Put(rawDocsFilter) - - rawDocsFilter.B = marshalDocsFilter(rawDocsFilter.B, df) - util.MustWriteFileAtomic(doneFilePath, rawDocsFilter.B, 0o666, tmpExt) - util.RemoveFile(queueFilePath) - - return nil -} - -// createDataDir creates data dir. -func (fm *FilterManager) createDataDir() { - if err := os.MkdirAll(fm.config.DataDir, 0o777); err != nil { - panic(err) - } -} diff --git a/frac/active.go b/frac/active.go index a2db933c..7c3691c1 100644 --- a/frac/active.go +++ b/frac/active.go @@ -61,7 +61,7 @@ type Active struct { writer *ActiveWriter indexer *ActiveIndexer - filterManager FilterManager + skipMaskProvider skipMaskProvider } const ( @@ -81,7 +81,7 @@ func NewActive( docsCache *cache.Cache[[]byte], sortCache *cache.Cache[[]byte], cfg *Config, - filterManager FilterManager, + skipMaskProvider skipMaskProvider, ) *Active { docsFile, docsStats := mustOpenFile(baseFileName+consts.DocsFileSuffix, config.SkipFsync) @@ -112,7 +112,7 @@ func NewActive( info: common.NewInfo(baseFileName, uint64(docsStats.Size()), metaSize), Config: cfg, - filterManager: filterManager, + skipMaskProvider: skipMaskProvider, } // use of 0 as keys in maps is prohibited – it's system key, so add first element @@ -418,7 +418,7 @@ func (f *Active) createDataProvider(ctx context.Context) *activeDataProvider { idsToLids: f.IDsToLIDs, docsReader: &f.docsReader, - filterManager: f.filterManager, + skipMaskProvider: f.skipMaskProvider, } } diff --git a/frac/active_index.go b/frac/active_index.go index 4c0130c4..27e4e464 100644 --- a/frac/active_index.go +++ b/frac/active_index.go @@ -32,7 +32,7 @@ type activeDataProvider struct { idsIndex *activeIDsIndex - filterManager FilterManager + skipMaskProvider skipMaskProvider } func (dp *activeDataProvider) release() { @@ -79,12 +79,12 @@ func (dp *activeDataProvider) Fetch(ids []seq.ID) ([][]byte, error) { res := make([][]byte, len(ids)) indexes := []activeFetchIndex{{ - blocksOffsets: dp.blocksOffsets, - docsPositions: dp.docsPositions, - idsToLids: dp.idsToLids, - docsReader: dp.docsReader, - filterManager: dp.filterManager, - fracName: dp.info.Name(), + blocksOffsets: dp.blocksOffsets, + docsPositions: dp.docsPositions, + idsToLids: dp.idsToLids, + docsReader: dp.docsReader, + skipMaskProvider: dp.skipMaskProvider, + fracName: dp.info.Name(), }} for _, fi := range indexes { @@ -124,7 +124,7 @@ func (dp *activeDataProvider) Search(params processor.SearchParams) (*seq.QPR, e indexes := []activeSearchIndex{{ activeIDsIndex: dp.getIDsIndex(), activeTokenIndex: dp.getTokenIndex(), - filterManager: dp.filterManager, + skipMaskProvider: dp.skipMaskProvider, fracName: dp.info.Name(), }} m.Stop() @@ -187,15 +187,15 @@ func (p *activeIDsIndex) LessOrEqual(lid seq.LID, id seq.ID) bool { type activeSearchIndex struct { *activeIDsIndex *activeTokenIndex - filterManager FilterManager - fracName string + skipMaskProvider skipMaskProvider + fracName string } -func (si *activeSearchIndex) GetHideFlags(minLID, maxLID uint32, reverse bool) (node.Node, bool, error) { +func (si *activeSearchIndex) GetSkipLIDs(minLID, maxLID uint32, reverse bool) (node.Node, bool, error) { // active fraction doesn't meet min and max lid minLID, maxLID = uint32(0), uint32(math.MaxUint32) - iterator, has, err := si.filterManager.GetHideFlagIteratorByFrac(si.fracName, minLID, maxLID, reverse) + iterator, has, err := si.skipMaskProvider.GetIDsIteratorByFrac(si.fracName, minLID, maxLID, reverse) if err != nil { return nil, false, err } @@ -259,12 +259,12 @@ func inverseLIDs(unmapped []uint32, inv *inverser, minLID, maxLID uint32) []uint } type activeFetchIndex struct { - blocksOffsets []uint64 - docsPositions *DocsPositions - idsToLids *ActiveLIDs - docsReader *storage.DocsReader - filterManager FilterManager - fracName string + blocksOffsets []uint64 + docsPositions *DocsPositions + idsToLids *ActiveLIDs + docsReader *storage.DocsReader + skipMaskProvider skipMaskProvider + fracName string } func (di *activeFetchIndex) GetBlocksOffsets(num uint32) uint64 { @@ -278,7 +278,7 @@ func (di *activeFetchIndex) GetDocPos(ids []seq.ID) ([]seq.DocPos, error) { } minLID, maxLID := uint32(0), uint32(math.MaxUint32) - hideFlagsIterator, has, err := di.filterManager.GetHideFlagIteratorByFrac(di.fracName, minLID, maxLID, false) + skipLIDsIterator, has, err := di.skipMaskProvider.GetIDsIteratorByFrac(di.fracName, minLID, maxLID, false) if err != nil { return nil, err } @@ -294,17 +294,17 @@ func (di *activeFetchIndex) GetDocPos(ids []seq.ID) ([]seq.DocPos, error) { } } - filteredLIDs := make(map[uint32]struct{}) + skipLIDs := make(map[uint32]struct{}) for { - lid := hideFlagsIterator.Next() + lid := skipLIDsIterator.Next() if lid.IsNull() { break } - filteredLIDs[lid.Unpack()] = struct{}{} + skipLIDs[lid.Unpack()] = struct{}{} } for i, lid := range allLids { - if _, ok := filteredLIDs[lid]; ok { + if _, ok := skipLIDs[lid]; ok { docsPos[i] = seq.DocPosNotFound } } diff --git a/frac/active_indexer_test.go b/frac/active_indexer_test.go index d1c7655b..a1200a7c 100644 --- a/frac/active_indexer_test.go +++ b/frac/active_indexer_test.go @@ -90,7 +90,7 @@ func BenchmarkIndexer(b *testing.B) { cache.NewCache[[]byte](nil, nil), cache.NewCache[[]byte](nil, nil), &Config{}, - testFilterManager{}, + testSkipMaskProvider{}, ) processor := getTestProcessor() diff --git a/frac/fraction_concurrency_test.go b/frac/fraction_concurrency_test.go index 0ad35e10..a5c19b22 100644 --- a/frac/fraction_concurrency_test.go +++ b/frac/fraction_concurrency_test.go @@ -52,7 +52,7 @@ func TestConcurrentAppendAndQuery(t *testing.T) { cache.NewCache[[]byte](nil, nil), cache.NewCache[[]byte](nil, nil), &Config{}, - testFilterManager{}, + testSkipMaskProvider{}, ) mapping := seq.Mapping{ @@ -369,7 +369,7 @@ func seal(active *Active) (*Sealed, error) { indexCache, cache.NewCache[[]byte](nil, nil), &Config{}, - testFilterManager{}, + testSkipMaskProvider{}, ) active.Release() return sealed, nil diff --git a/frac/fraction_test.go b/frac/fraction_test.go index f40b8c9d..ec5f3d85 100644 --- a/frac/fraction_test.go +++ b/frac/fraction_test.go @@ -35,12 +35,12 @@ import ( "github.com/ozontech/seq-db/tokenizer" ) -type testFilterManager struct{} +type testSkipMaskProvider struct{} -func (testFilterManager) GetHideFlagIteratorByFrac(fracName string, minLID, maxLID uint32, reverse bool) (node.Node, bool, error) { +func (testSkipMaskProvider) GetIDsIteratorByFrac(fracName string, minLID, maxLID uint32, reverse bool) (node.Node, bool, error) { return node.NewStatic([]uint32{}, false), false, nil } -func (testFilterManager) RemoveFrac(_ string) {} +func (testSkipMaskProvider) RemoveFrac(_ string) {} type FractionTestSuite struct { suite.Suite @@ -2047,7 +2047,7 @@ func (s *FractionTestSuite) newActive(bulks ...[]string) *Active { cache.NewCache[[]byte](nil, nil), cache.NewCache[[]byte](nil, nil), s.config, - testFilterManager{}, + testSkipMaskProvider{}, ) var wg sync.WaitGroup @@ -2110,7 +2110,7 @@ func (s *FractionTestSuite) newSealed(bulks ...[]string) *Sealed { indexCache, cache.NewCache[[]byte](nil, nil), s.config, - testFilterManager{}, + testSkipMaskProvider{}, ) active.Release() return sealed @@ -2188,7 +2188,7 @@ func (s *ActiveReplayedFractionTestSuite) Replay(frac *Active) Fraction { cache.NewCache[[]byte](nil, nil), cache.NewCache[[]byte](nil, nil), &Config{}, - testFilterManager{}, + testSkipMaskProvider{}, ) err := replayedFrac.Replay(context.Background()) s.Require().NoError(err, "replay failed") @@ -2301,7 +2301,7 @@ func (s *SealedLoadedFractionTestSuite) newSealedLoaded(bulks ...[]string) *Seal cache.NewCache[[]byte](nil, nil), nil, s.config, - testFilterManager{}, + testSkipMaskProvider{}, ) s.fraction = sealed return sealed @@ -2371,7 +2371,7 @@ func (s *RemoteFractionTestSuite) SetupTest() { sealed.info, s.config, s3cli, - testFilterManager{}, + testSkipMaskProvider{}, ) s.fraction = remoteFrac } diff --git a/frac/processor/eval_tree.go b/frac/processor/eval_tree.go index 7bdd80d9..b152b3bc 100644 --- a/frac/processor/eval_tree.go +++ b/frac/processor/eval_tree.go @@ -88,9 +88,9 @@ func evalLeaf( return node.BuildORTree(lidsTids), nil } -func evalHideFlags(root, hideFlagsIterator node.Node, stats *searchStats) node.Node { +func evalSkipLIDs(root, skipLIDsIterator node.Node, stats *searchStats) node.Node { stats.NodesTotal++ - return node.NewNAnd(hideFlagsIterator, root) + return node.NewNAnd(skipLIDsIterator, root) } type Aggregator interface { diff --git a/frac/processor/search.go b/frac/processor/search.go index 064a0b31..3c08eae0 100644 --- a/frac/processor/search.go +++ b/frac/processor/search.go @@ -38,7 +38,7 @@ type tokenIndex interface { type searchIndex interface { tokenIndex idsIndex - GetHideFlags(minLID, maxLID uint32, reverse bool) (node.Node, bool, error) + GetSkipLIDs(minLID, maxLID uint32, reverse bool) (node.Node, bool, error) } func IndexSearch( @@ -94,16 +94,16 @@ func IndexSearch( } } - m = sw.Start("get_hide_flags") - hideFlags, hasHideFlags, err := index.GetHideFlags(minLID, maxLID, params.Order.IsReverse()) + m = sw.Start("get_skip_lids") + skipLIDs, hasSkipLIDs, err := index.GetSkipLIDs(minLID, maxLID, params.Order.IsReverse()) m.Stop() if err != nil { return nil, err } - if hasHideFlags { - m = sw.Start("eval_hide_flags") - evalTree = evalHideFlags(evalTree, hideFlags, stats) + if hasSkipLIDs { + m = sw.Start("eval_skip_lids") + evalTree = evalSkipLIDs(evalTree, skipLIDs, stats) m.Stop() } diff --git a/frac/remote.go b/frac/remote.go index 44510f47..7658e80e 100644 --- a/frac/remote.go +++ b/frac/remote.go @@ -56,7 +56,7 @@ type Remote struct { s3cli *s3.Client readLimiter *storage.ReadLimiter - filterManager FilterManager + skipMaskProvider skipMaskProvider } func NewRemote( @@ -68,7 +68,7 @@ func NewRemote( info *common.Info, config *Config, s3cli *s3.Client, - filterManager FilterManager, + skipMaskProvider skipMaskProvider, ) *Remote { f := &Remote{ ctx: ctx, @@ -85,7 +85,7 @@ func NewRemote( s3cli: s3cli, - filterManager: filterManager, + skipMaskProvider: skipMaskProvider, } // Fast path if fraction-info cache exists AND it has valid index size. @@ -175,7 +175,7 @@ func (f *Remote) createDataProvider(ctx context.Context) (*sealedDataProvider, e &f.blocksData.IDsTable, f.info.BinaryDataVer, ), - filterManager: f.filterManager, + skipMaskProvider: f.skipMaskProvider, }, nil } @@ -208,7 +208,7 @@ func (f *Remote) Suicide() { ) } - f.filterManager.RemoveFrac(f.info.Name()) + f.skipMaskProvider.RemoveFrac(f.info.Name()) } func (f *Remote) String() string { diff --git a/frac/sealed.go b/frac/sealed.go index 0d324348..bda4fc72 100644 --- a/frac/sealed.go +++ b/frac/sealed.go @@ -52,7 +52,7 @@ type Sealed struct { // shit for testing PartialSuicideMode PSD - filterManager FilterManager + skipMaskProvider skipMaskProvider } type PSD int // emulates hard shutdown on different stages of fraction deletion, used for tests @@ -70,7 +70,7 @@ func NewSealed( docsCache *cache.Cache[[]byte], info *common.Info, config *Config, - filterManager FilterManager, + skipMaskProvider skipMaskProvider, ) *Sealed { f := &Sealed{ loadMu: &sync.RWMutex{}, @@ -85,7 +85,7 @@ func NewSealed( PartialSuicideMode: Off, - filterManager: filterManager, + skipMaskProvider: skipMaskProvider, } // fast path if fraction-info cache exists AND it has valid index size @@ -135,7 +135,7 @@ func NewSealedPreloaded( indexCache *IndexCache, docsCache *cache.Cache[[]byte], config *Config, - filterManager FilterManager, + skipMaskProvider skipMaskProvider, ) *Sealed { f := &Sealed{ blocksData: preloaded.BlocksData, @@ -151,7 +151,7 @@ func NewSealedPreloaded( BaseFileName: baseFile, Config: config, - filterManager: filterManager, + skipMaskProvider: skipMaskProvider, } // put the token table built during sealing into the cache of the sealed fraction @@ -301,7 +301,7 @@ func (f *Sealed) Suicide() { ) } - f.filterManager.RemoveFrac(f.info.Name()) + f.skipMaskProvider.RemoveFrac(f.info.Name()) } func (f *Sealed) String() string { @@ -354,7 +354,7 @@ func (f *Sealed) createDataProvider(ctx context.Context) *sealedDataProvider { f.info.BinaryDataVer, ), - filterManager: f.filterManager, + skipMaskProvider: f.skipMaskProvider, } } diff --git a/frac/sealed_index.go b/frac/sealed_index.go index c1a0e3c5..7c62713c 100644 --- a/frac/sealed_index.go +++ b/frac/sealed_index.go @@ -22,8 +22,8 @@ import ( "github.com/ozontech/seq-db/util" ) -type FilterManager interface { - GetHideFlagIteratorByFrac(fracName string, minLID, maxLID uint32, reverse bool) (node.Node, bool, error) +type skipMaskProvider interface { + GetIDsIteratorByFrac(fracName string, minLID, maxLID uint32, reverse bool) (node.Node, bool, error) RemoveFrac(fracName string) } @@ -48,7 +48,7 @@ type sealedDataProvider struct { // This value is used in metrics to distinguish between operations over local and remote fractions. fractionTypeLabel string - filterManager FilterManager + skipMaskProvider skipMaskProvider } func (dp *sealedDataProvider) getIDsIndex() *sealedIDsIndex { @@ -61,11 +61,11 @@ func (dp *sealedDataProvider) getIDsIndex() *sealedIDsIndex { func (dp *sealedDataProvider) getFetchIndex() *sealedFetchIndex { return &sealedFetchIndex{ - fracName: dp.info.Name(), - idsIndex: dp.getIDsIndex(), - docsReader: dp.docsReader, - blocksOffsets: dp.blocksOffsets, - filterManager: dp.filterManager, + fracName: dp.info.Name(), + idsIndex: dp.getIDsIndex(), + docsReader: dp.docsReader, + blocksOffsets: dp.blocksOffsets, + skipMaskProvider: dp.skipMaskProvider, } } @@ -83,7 +83,7 @@ func (dp *sealedDataProvider) getSearchIndex() *sealedSearchIndex { return &sealedSearchIndex{ sealedIDsIndex: dp.getIDsIndex(), sealedTokenIndex: dp.getTokenIndex(), - filterManager: dp.filterManager, + skipMaskProvider: dp.skipMaskProvider, } } @@ -269,11 +269,11 @@ func (ti *sealedTokenIndex) GetLIDsFromTIDs(tids []uint32, stats lids.Counter, m } type sealedFetchIndex struct { - fracName string - idsIndex *sealedIDsIndex - docsReader *storage.DocsReader - blocksOffsets []uint64 - filterManager FilterManager + fracName string + idsIndex *sealedIDsIndex + docsReader *storage.DocsReader + blocksOffsets []uint64 + skipMaskProvider skipMaskProvider } func (fi *sealedFetchIndex) GetBlocksOffsets(num uint32) uint64 { @@ -294,7 +294,7 @@ func (fi *sealedFetchIndex) GetDocPos(ids []seq.ID) ([]seq.DocPos, error) { minLID, maxLID = uint32(minVal), uint32(maxVal) } - hideFlagsIterator, has, err := fi.filterManager.GetHideFlagIteratorByFrac(fi.fracName, minLID, maxLID, false) + skipLIDsIterator, has, err := fi.skipMaskProvider.GetIDsIteratorByFrac(fi.fracName, minLID, maxLID, false) if err != nil { return nil, err } @@ -303,17 +303,17 @@ func (fi *sealedFetchIndex) GetDocPos(ids []seq.ID) ([]seq.DocPos, error) { return fi.getDocPosByLIDs(allLids), nil } - filteredLIDs := make(map[uint32]struct{}) + skipLIDs := make(map[uint32]struct{}) for { - lid := hideFlagsIterator.Next() + lid := skipLIDsIterator.Next() if lid.IsNull() { break } - filteredLIDs[lid.Unpack()] = struct{}{} + skipLIDs[lid.Unpack()] = struct{}{} } for i, lid := range allLids { - if _, ok := filteredLIDs[uint32(lid)]; ok { + if _, ok := skipLIDs[uint32(lid)]; ok { allLids[i] = 0 } } @@ -373,9 +373,9 @@ func (fi *sealedFetchIndex) getDocPosByLIDs(localIDs []seq.LID) []seq.DocPos { type sealedSearchIndex struct { *sealedIDsIndex *sealedTokenIndex - filterManager FilterManager + skipMaskProvider skipMaskProvider } -func (si *sealedSearchIndex) GetHideFlags(minLID, maxLID uint32, reverse bool) (node.Node, bool, error) { - return si.filterManager.GetHideFlagIteratorByFrac(si.fracName, minLID, maxLID, reverse) +func (si *sealedSearchIndex) GetSkipLIDs(minLID, maxLID uint32, reverse bool) (node.Node, bool, error) { + return si.skipMaskProvider.GetIDsIteratorByFrac(si.fracName, minLID, maxLID, reverse) } diff --git a/fracmanager/fracmanager.go b/fracmanager/fracmanager.go index 1b28bd71..b35066a9 100644 --- a/fracmanager/fracmanager.go +++ b/fracmanager/fracmanager.go @@ -35,13 +35,13 @@ var defaultStorageState = StorageState{ // - stats updating // // Returns the manager instance and a stop function to gracefully shutdown -func New(ctx context.Context, cfg *Config, s3cli *s3.Client, filterManager FilterManager) (*FracManager, func(), error) { +func New(ctx context.Context, cfg *Config, s3cli *s3.Client, skipMaskProvider skipMaskProvider) (*FracManager, func(), error) { FillConfigWithDefault(cfg) readLimiter := storage.NewReadLimiter(config.ReaderWorkers, storeBytesRead) idx, stopIdx := frac.NewActiveIndexer(config.IndexWorkers, config.IndexWorkers) cache := NewCacheMaintainer(cfg.CacheSize, cfg.SortCacheSize, newDefaultCacheMetrics()) - provider := newFractionProvider(cfg, s3cli, cache, readLimiter, idx, filterManager) + provider := newFractionProvider(cfg, s3cli, cache, readLimiter, idx, skipMaskProvider) infoCache := NewFracInfoCache(filepath.Join(cfg.DataDir, consts.FracCacheFileSuffix)) // Load existing fractions into registry diff --git a/fracmanager/fracmanager_test.go b/fracmanager/fracmanager_test.go index 9eca4add..663dedec 100644 --- a/fracmanager/fracmanager_test.go +++ b/fracmanager/fracmanager_test.go @@ -12,13 +12,13 @@ import ( "github.com/ozontech/seq-db/seq" ) -type testFilterManager struct{} +type testSkipMaskProvider struct{} -func (testFilterManager) GetHideFlagIteratorByFrac(fracName string, minLID, maxLID uint32, reverse bool) (node.Node, bool, error) { +func (testSkipMaskProvider) GetIDsIteratorByFrac(fracName string, minLID, maxLID uint32, reverse bool) (node.Node, bool, error) { return node.NewStatic([]uint32{}, reverse), false, nil } -func (testFilterManager) RefreshFrac(_ frac.Fraction) {} -func (testFilterManager) RemoveFrac(_ string) {} +func (testSkipMaskProvider) RefreshFrac(_ frac.Fraction) {} +func (testSkipMaskProvider) RemoveFrac(_ string) {} func setupDataDir(t testing.TB, cfg *Config) *Config { if cfg == nil { @@ -34,7 +34,7 @@ func setupDataDir(t testing.TB, cfg *Config) *Config { func setupFracManager(t testing.TB, cfg *Config) (*Config, *FracManager, func()) { cfg = setupDataDir(t, cfg) - fm, stop, err := New(t.Context(), cfg, nil, testFilterManager{}) + fm, stop, err := New(t.Context(), cfg, nil, testSkipMaskProvider{}) assert.NoError(t, err) return cfg, fm, stop } diff --git a/fracmanager/fraction_provider.go b/fracmanager/fraction_provider.go index 169a5f2d..73deb907 100644 --- a/fracmanager/fraction_provider.go +++ b/fracmanager/fraction_provider.go @@ -20,8 +20,8 @@ import ( const fileBasePattern = "seq-db-" -type FilterManager interface { - GetHideFlagIteratorByFrac(fracName string, minLID, maxLID uint32, reverse bool) (node.Node, bool, error) +type skipMaskProvider interface { + GetIDsIteratorByFrac(fracName string, minLID, maxLID uint32, reverse bool) (node.Node, bool, error) RefreshFrac(frac frac.Fraction) RemoveFrac(fracName string) } @@ -29,28 +29,28 @@ type FilterManager interface { // fractionProvider is a factory for creating different types of fractions // Contains all necessary dependencies for creating and managing fractions type fractionProvider struct { - s3cli *s3.Client // Client for S3 storage operations - config *Config // Fraction manager configuration - cacheProvider *CacheMaintainer // Cache provider for data access optimization - activeIndexer *frac.ActiveIndexer // Indexer for active fractions - readLimiter *storage.ReadLimiter // Read rate limiter - ulidEntropy io.Reader // Entropy source for ULID generation - filterManager FilterManager + s3cli *s3.Client // Client for S3 storage operations + config *Config // Fraction manager configuration + cacheProvider *CacheMaintainer // Cache provider for data access optimization + activeIndexer *frac.ActiveIndexer // Indexer for active fractions + readLimiter *storage.ReadLimiter // Read rate limiter + ulidEntropy io.Reader // Entropy source for ULID generation + skipMaskProvider skipMaskProvider } func newFractionProvider( cfg *Config, s3cli *s3.Client, cp *CacheMaintainer, readLimiter *storage.ReadLimiter, indexer *frac.ActiveIndexer, - filterManager FilterManager, + skipMaskProvider skipMaskProvider, ) *fractionProvider { return &fractionProvider{ - s3cli: s3cli, - config: cfg, - cacheProvider: cp, - activeIndexer: indexer, - readLimiter: readLimiter, - ulidEntropy: ulid.Monotonic(rand.New(rand.NewSource(time.Now().UnixNano())), 0), - filterManager: filterManager, + s3cli: s3cli, + config: cfg, + cacheProvider: cp, + activeIndexer: indexer, + readLimiter: readLimiter, + ulidEntropy: ulid.Monotonic(rand.New(rand.NewSource(time.Now().UnixNano())), 0), + skipMaskProvider: skipMaskProvider, } } @@ -62,7 +62,7 @@ func (fp *fractionProvider) NewActive(name string) *frac.Active { fp.cacheProvider.CreateDocBlockCache(), fp.cacheProvider.CreateSortDocsCache(), &fp.config.Fraction, - fp.filterManager, + fp.skipMaskProvider, ) } @@ -74,7 +74,7 @@ func (fp *fractionProvider) NewSealed(name string, cachedInfo *common.Info) *fra fp.cacheProvider.CreateDocBlockCache(), cachedInfo, // Preloaded meta information &fp.config.Fraction, - fp.filterManager, + fp.skipMaskProvider, ) } @@ -86,7 +86,7 @@ func (fp *fractionProvider) NewSealedPreloaded(name string, preloadedData *seale fp.cacheProvider.CreateIndexCache(), fp.cacheProvider.CreateDocBlockCache(), &fp.config.Fraction, - fp.filterManager, + fp.skipMaskProvider, ) } @@ -100,7 +100,7 @@ func (fp *fractionProvider) NewRemote(ctx context.Context, name string, cachedIn cachedInfo, &fp.config.Fraction, fp.s3cli, - fp.filterManager, + fp.skipMaskProvider, ) } @@ -132,7 +132,7 @@ func (fp *fractionProvider) Seal(active *frac.Active) (*frac.Sealed, error) { } sealedFrac := fp.NewSealedPreloaded(active.BaseFileName, preloaded) - fp.filterManager.RefreshFrac(sealedFrac) + fp.skipMaskProvider.RefreshFrac(sealedFrac) return sealedFrac, nil } diff --git a/fracmanager/fraction_provider_test.go b/fracmanager/fraction_provider_test.go index b965de33..aae4e820 100644 --- a/fracmanager/fraction_provider_test.go +++ b/fracmanager/fraction_provider_test.go @@ -38,7 +38,7 @@ func setupFractionProvider(t testing.TB, cfg *Config) (*fractionProvider, func() s3cli, stopS3 := setupS3Client(t) idx, stopIdx := frac.NewActiveIndexer(1, 1) cache := NewCacheMaintainer(uint64(units.MB), uint64(units.MB), nil) - provider := newFractionProvider(cfg, s3cli, cache, rl, idx, testFilterManager{}) + provider := newFractionProvider(cfg, s3cli, cache, rl, idx, testSkipMaskProvider{}) return provider, func() { stopIdx() stopS3() diff --git a/filtermanager/encoding.go b/skipmaskmanager/encoding.go similarity index 87% rename from filtermanager/encoding.go rename to skipmaskmanager/encoding.go index 26a6fa46..69b6e5a1 100644 --- a/filtermanager/encoding.go +++ b/skipmaskmanager/encoding.go @@ -1,4 +1,4 @@ -package filtermanager +package skipmaskmanager import ( "encoding/binary" @@ -12,22 +12,22 @@ import ( "github.com/ozontech/seq-db/zstd" ) -type DocsFilterBinIn struct { +type SkipMaskBinIn struct { LIDs []seq.LID } -type DocsFilterBinOut struct { +type SkipMaskBinOut struct { LIDs []uint32 } -type docsFilterBinVersion uint8 +type skipMaskBinVersion uint8 const ( - docsFilterBinVersion1 docsFilterBinVersion = iota + 1 + skipMaskBinVersion1 skipMaskBinVersion = iota + 1 ) -var availableVersions = map[docsFilterBinVersion]struct{}{ - docsFilterBinVersion1: {}, +var availableVersions = map[skipMaskBinVersion]struct{}{ + skipMaskBinVersion1: {}, } type lidsCodec byte @@ -86,8 +86,8 @@ func (h *lidsBlockHeader) unmarshal(src []byte) ([]byte, error) { return src, nil } -func marshalDocsFilter(dst []byte, in *DocsFilterBinIn) []byte { - dst = append(dst, uint8(docsFilterBinVersion1)) +func marshalSkipMask(dst []byte, in *SkipMaskBinIn) []byte { + dst = append(dst, uint8(skipMaskBinVersion1)) dst = marshalLIDsBlocks(dst, in.LIDs) return dst } @@ -168,17 +168,17 @@ func marshalLIDsBlock(dst []byte, in []seq.LID) ([]byte, lidsCodec) { return dst, lidsCodecDeltaZstd } -const minLIDsFIlterBytesLen = 10 // 1 byte lidsBinVersion + 8 byte number of LIDs + N (min 1) bytes varint + delta encoded LIDs +const minSkipMaskBytesLen = 10 // 1 byte skipMaskBinVersion + 8 byte number of LIDs + N (min 1) bytes varint + delta encoded LIDs -func unmarshalDocsFilter(dst *DocsFilterBinOut, src []byte) (_ []byte, err error) { - if len(src) < minLIDsFIlterBytesLen { - return nil, fmt.Errorf("invalid LIDs filter format; want %d bytes, got %d", minLIDsFIlterBytesLen, len(src)) +func unmarshalSkipMask(dst *SkipMaskBinOut, src []byte) (_ []byte, err error) { + if len(src) < minSkipMaskBytesLen { + return nil, fmt.Errorf("invalid skip mask format; want %d bytes, got %d", minSkipMaskBytesLen, len(src)) } - version := docsFilterBinVersion(src[0]) + version := skipMaskBinVersion(src[0]) src = src[1:] if _, ok := availableVersions[version]; !ok { - return nil, fmt.Errorf("invalid LIDs binary version: %d", version) + return nil, fmt.Errorf("invalid skip mask binary version: %d", version) } dst.LIDs, src, err = unmarshalLIDsBlocks(dst.LIDs, src) diff --git a/filtermanager/encoding_test.go b/skipmaskmanager/encoding_test.go similarity index 58% rename from filtermanager/encoding_test.go rename to skipmaskmanager/encoding_test.go index 8af0b7a3..38647571 100644 --- a/filtermanager/encoding_test.go +++ b/skipmaskmanager/encoding_test.go @@ -1,4 +1,4 @@ -package filtermanager +package skipmaskmanager import ( "testing" @@ -9,28 +9,28 @@ import ( "github.com/ozontech/seq-db/seq" ) -func TestMarshalUnmarshalLIDsFilter(t *testing.T) { - test := func(df DocsFilterBinIn) { +func TestMarshalUnmarshalSkipMask(t *testing.T) { + test := func(df SkipMaskBinIn) { t.Helper() - rawDocsFilter := marshalDocsFilter(nil, &df) - var out DocsFilterBinOut - tail, err := unmarshalDocsFilter(&out, rawDocsFilter) + rawSkipMask := marshalSkipMask(nil, &df) + var out SkipMaskBinOut + tail, err := unmarshalSkipMask(&out, rawSkipMask) require.NoError(t, err) require.Equal(t, 0, len(tail)) assert.Equal(t, lidsToUint32s(df.LIDs), out.LIDs) } - test(DocsFilterBinIn{LIDs: []seq.LID{0, 1, 2, 3}}) - test(DocsFilterBinIn{LIDs: []seq.LID{10, 15, 22, 18, 105, 1010}}) - test(DocsFilterBinIn{LIDs: []seq.LID{11}}) + test(SkipMaskBinIn{LIDs: []seq.LID{0, 1, 2, 3}}) + test(SkipMaskBinIn{LIDs: []seq.LID{10, 15, 22, 18, 105, 1010}}) + test(SkipMaskBinIn{LIDs: []seq.LID{11}}) multipleBlocksSize := maxLIDsBlockLen*3 + 15 multipleBlocksLIDs := make([]seq.LID, 0, multipleBlocksSize) for i := range multipleBlocksSize { multipleBlocksLIDs = append(multipleBlocksLIDs, seq.LID(i)) } - test(DocsFilterBinIn{LIDs: multipleBlocksLIDs}) + test(SkipMaskBinIn{LIDs: multipleBlocksLIDs}) } func lidsToUint32s(in []seq.LID) []uint32 { diff --git a/filtermanager/iterator.go b/skipmaskmanager/iterator.go similarity index 97% rename from filtermanager/iterator.go rename to skipmaskmanager/iterator.go index eff9fbb6..b8a4de12 100644 --- a/filtermanager/iterator.go +++ b/skipmaskmanager/iterator.go @@ -1,4 +1,4 @@ -package filtermanager +package skipmaskmanager import "sort" diff --git a/filtermanager/iterator_asc.go b/skipmaskmanager/iterator_asc.go similarity index 93% rename from filtermanager/iterator_asc.go rename to skipmaskmanager/iterator_asc.go index db0e0c15..a50b6dcb 100644 --- a/filtermanager/iterator_asc.go +++ b/skipmaskmanager/iterator_asc.go @@ -1,4 +1,4 @@ -package filtermanager +package skipmaskmanager import ( "go.uber.org/zap" @@ -17,7 +17,7 @@ func (it *IteratorAsc) Next() node.LID { if it.loader.headers == nil { headers, err := it.loader.getHeaders() if err != nil { - logger.Panic("can't load filter file headers", zap.Error(err)) + logger.Panic("can't load skip mask file headers", zap.Error(err)) } it.loader.headers = headers it.blockIndex = len(it.loader.headers) - 1 diff --git a/filtermanager/iterator_asc_test.go b/skipmaskmanager/iterator_asc_test.go similarity index 88% rename from filtermanager/iterator_asc_test.go rename to skipmaskmanager/iterator_asc_test.go index b9e6ce93..72ab47ff 100644 --- a/filtermanager/iterator_asc_test.go +++ b/skipmaskmanager/iterator_asc_test.go @@ -1,4 +1,4 @@ -package filtermanager +package skipmaskmanager import ( "math" @@ -53,9 +53,9 @@ func TestIteratorAsc(t *testing.T) { for _, tc := range tests { t.Run(tc.title, func(t *testing.T) { - rawDocsFilter := marshalDocsFilter(nil, &DocsFilterBinIn{LIDs: multipleBlocksLIDs}) - filePath := filepath.Join(t.TempDir(), "some.filter") - err := os.WriteFile(filePath, rawDocsFilter, 0o644) + rawSkipMask := marshalSkipMask(nil, &SkipMaskBinIn{LIDs: multipleBlocksLIDs}) + filePath := filepath.Join(t.TempDir(), "some.skipmask") + err := os.WriteFile(filePath, rawSkipMask, 0o644) require.NoError(t, err) loader, err := newLoader(filePath, cache.NewCache[[]lidsBlockHeader](nil, nil)) diff --git a/filtermanager/iterator_desc.go b/skipmaskmanager/iterator_desc.go similarity index 93% rename from filtermanager/iterator_desc.go rename to skipmaskmanager/iterator_desc.go index e5951948..c207d671 100644 --- a/filtermanager/iterator_desc.go +++ b/skipmaskmanager/iterator_desc.go @@ -1,4 +1,4 @@ -package filtermanager +package skipmaskmanager import ( "go.uber.org/zap" @@ -17,7 +17,7 @@ func (it *IteratorDesc) Next() node.LID { if it.loader.headers == nil { headers, err := it.loader.getHeaders() if err != nil { - logger.Panic("can't load filter file headers", zap.Error(err)) + logger.Panic("can't load skip mask file headers", zap.Error(err)) } it.loader.headers = headers } diff --git a/filtermanager/iterator_desc_test.go b/skipmaskmanager/iterator_desc_test.go similarity index 87% rename from filtermanager/iterator_desc_test.go rename to skipmaskmanager/iterator_desc_test.go index e2736160..749ced7d 100644 --- a/filtermanager/iterator_desc_test.go +++ b/skipmaskmanager/iterator_desc_test.go @@ -1,4 +1,4 @@ -package filtermanager +package skipmaskmanager import ( "math" @@ -48,9 +48,9 @@ func TestIteratorDesc(t *testing.T) { for _, tc := range tests { t.Run(tc.title, func(t *testing.T) { - rawDocsFilter := marshalDocsFilter(nil, &DocsFilterBinIn{LIDs: multipleBlocksLIDs}) - filePath := filepath.Join(t.TempDir(), "some.filter") - err := os.WriteFile(filePath, rawDocsFilter, 0o644) + rawSkipMask := marshalSkipMask(nil, &SkipMaskBinIn{LIDs: multipleBlocksLIDs}) + filePath := filepath.Join(t.TempDir(), "some.skipmask") + err := os.WriteFile(filePath, rawSkipMask, 0o644) require.NoError(t, err) loader, err := newLoader(filePath, cache.NewCache[[]lidsBlockHeader](nil, nil)) diff --git a/filtermanager/loader.go b/skipmaskmanager/loader.go similarity index 96% rename from filtermanager/loader.go rename to skipmaskmanager/loader.go index 0c795dd9..0a0717b3 100644 --- a/filtermanager/loader.go +++ b/skipmaskmanager/loader.go @@ -1,4 +1,4 @@ -package filtermanager +package skipmaskmanager import ( "encoding/binary" @@ -65,7 +65,7 @@ func (l *loader) loadHeaders() ([]lidsBlockHeader, error) { return nil, fmt.Errorf("can't read headers from disk: n=0") } - version := docsFilterBinVersion(numBuf[0]) + version := skipMaskBinVersion(numBuf[0]) if _, ok := availableVersions[version]; !ok { return nil, fmt.Errorf("invalid LIDs binary version: %d", version) } @@ -144,7 +144,7 @@ func (l *loader) loadBlock(index int) ([]uint32, error) { func (l *loader) release() error { if l.file != nil { if err := l.file.Close(); err != nil { - logger.Error("can't close filter file", zap.Error(err)) + logger.Error("can't close skip mask file", zap.Error(err)) return err } l.file = nil diff --git a/filtermanager/loader_test.go b/skipmaskmanager/loader_test.go similarity index 79% rename from filtermanager/loader_test.go rename to skipmaskmanager/loader_test.go index 3a13464f..eb49a472 100644 --- a/filtermanager/loader_test.go +++ b/skipmaskmanager/loader_test.go @@ -1,4 +1,4 @@ -package filtermanager +package skipmaskmanager import ( "os" @@ -18,9 +18,9 @@ func TestLoader(t *testing.T) { multipleBlocksLIDs = append(multipleBlocksLIDs, seq.LID(i)) } - rawDocsFilter := marshalDocsFilter(nil, &DocsFilterBinIn{LIDs: multipleBlocksLIDs}) - filePath := filepath.Join(t.TempDir(), "some.filter") - err := os.WriteFile(filePath, rawDocsFilter, 0o644) + rawSkipMask := marshalSkipMask(nil, &SkipMaskBinIn{LIDs: multipleBlocksLIDs}) + filePath := filepath.Join(t.TempDir(), "some.skipmask") + err := os.WriteFile(filePath, rawSkipMask, 0o644) require.NoError(t, err) loader, err := newLoader(filePath, cache.NewCache[[]lidsBlockHeader](nil, nil)) diff --git a/filtermanager/merged_iterator.go b/skipmaskmanager/merged_iterator.go similarity index 96% rename from filtermanager/merged_iterator.go rename to skipmaskmanager/merged_iterator.go index 7affabf5..e949c12f 100644 --- a/filtermanager/merged_iterator.go +++ b/skipmaskmanager/merged_iterator.go @@ -1,4 +1,4 @@ -package filtermanager +package skipmaskmanager import "github.com/ozontech/seq-db/node" diff --git a/filtermanager/merged_iterator_test.go b/skipmaskmanager/merged_iterator_test.go similarity index 98% rename from filtermanager/merged_iterator_test.go rename to skipmaskmanager/merged_iterator_test.go index c70d178c..8f448efd 100644 --- a/filtermanager/merged_iterator_test.go +++ b/skipmaskmanager/merged_iterator_test.go @@ -1,4 +1,4 @@ -package filtermanager +package skipmaskmanager import ( "fmt" diff --git a/filtermanager/metrics.go b/skipmaskmanager/metrics.go similarity index 50% rename from filtermanager/metrics.go rename to skipmaskmanager/metrics.go index ed47171e..be39505b 100644 --- a/filtermanager/metrics.go +++ b/skipmaskmanager/metrics.go @@ -1,4 +1,4 @@ -package filtermanager +package skipmaskmanager import ( "github.com/prometheus/client_golang/prometheus" @@ -6,22 +6,22 @@ import ( ) var ( - inProgressFilters = promauto.NewGauge(prometheus.GaugeOpts{ + inProgress = promauto.NewGauge(prometheus.GaugeOpts{ Namespace: "seq_db_store", - Subsystem: "filters", + Subsystem: "skip_masks", Name: "in_progress", - Help: "Number of doc filters in progress", + Help: "Number of skip masks in progress", }) diskUsage = promauto.NewGauge(prometheus.GaugeOpts{ Namespace: "seq_db_store", - Subsystem: "filters", + Subsystem: "skip_masks", Name: "disk_usage_bytes", - Help: "Disk space used by filter files in bytes", + Help: "Disk space used by skip mask files in bytes", }) - storedFilters = promauto.NewGauge(prometheus.GaugeOpts{ + stored = promauto.NewGauge(prometheus.GaugeOpts{ Namespace: "seq_db_store", - Subsystem: "filters", + Subsystem: "skip_masks", Name: "stored", - Help: "Number of active doc filters", + Help: "Number of active skip masks", }) ) diff --git a/filtermanager/filter.go b/skipmaskmanager/skip_mask.go similarity index 51% rename from filtermanager/filter.go rename to skipmaskmanager/skip_mask.go index f7d03566..1ebccdc2 100644 --- a/filtermanager/filter.go +++ b/skipmaskmanager/skip_mask.go @@ -1,4 +1,4 @@ -package filtermanager +package skipmaskmanager import ( "crypto/sha256" @@ -10,47 +10,49 @@ import ( "github.com/ozontech/seq-db/seq" ) -type FilterStatus byte +type SkipMaskStatus byte const ( - StatusCreated FilterStatus = iota + StatusCreated SkipMaskStatus = iota StatusInProgress StatusDone StatusError ) -type Params struct { +type SkipMaskParams struct { Query string From seq.MID To seq.MID } -type Filter struct { - params Params +type SkipMask struct { + params SkipMaskParams - status FilterStatus + status SkipMaskStatus ast parser.SeqQLQuery hash string dirPath string + mu *sync.RWMutex processWg *sync.WaitGroup } -func NewFilter(params Params) *Filter { - return &Filter{ +func NewSkipMask(params SkipMaskParams) *SkipMask { + return &SkipMask{ params: params, status: StatusCreated, + mu: &sync.RWMutex{}, processWg: &sync.WaitGroup{}, } } -func (f *Filter) String() string { +func (f *SkipMask) String() string { return fmt.Sprintf("%s_%d_%d", f.params.Query, f.params.From, f.params.To) } -func (f *Filter) Hash() string { +func (f *SkipMask) Hash() string { if f.hash == "" { h := sha256.New() h.Write([]byte(f.String())) @@ -60,6 +62,16 @@ func (f *Filter) Hash() string { return f.hash } -func (f *Filter) markAsDone() { - f.status = StatusDone +func (f *SkipMask) setStatus(status SkipMaskStatus) { + f.mu.Lock() + defer f.mu.Unlock() + + f.status = status +} + +func (f *SkipMask) getStatus() SkipMaskStatus { + f.mu.RLock() + defer f.mu.RUnlock() + + return f.status } diff --git a/skipmaskmanager/skip_mask_manager.go b/skipmaskmanager/skip_mask_manager.go new file mode 100644 index 00000000..4ee0f637 --- /dev/null +++ b/skipmaskmanager/skip_mask_manager.go @@ -0,0 +1,552 @@ +package skipmaskmanager + +import ( + "context" + "errors" + "fmt" + "hash/fnv" + "math" + "os" + "path" + "runtime" + "strings" + "sync" + "time" + + "go.uber.org/zap" + + "github.com/ozontech/seq-db/cache" + "github.com/ozontech/seq-db/frac" + "github.com/ozontech/seq-db/frac/processor" + "github.com/ozontech/seq-db/fracmanager" + "github.com/ozontech/seq-db/logger" + "github.com/ozontech/seq-db/node" + "github.com/ozontech/seq-db/parser" + "github.com/ozontech/seq-db/seq" + "github.com/ozontech/seq-db/util" +) + +const ( + fracInQueueExt = ".queue" + fracDoneExt = ".skipmask" + tmpExt = ".tmp" + + tmpDirSuffix = "_tmp" +) + +const ( + defaultMaintenanceInterval = 30 * time.Second + defaultCacheCleanInterval = 10 * time.Millisecond + defaultCacheGCDelay = 1 * time.Second +) + +type MappingProvider interface { + GetMapping() seq.Mapping +} + +type Config struct { + DataDir string + Workers int + CacheSizeLimit uint64 +} + +type SkipMaskManager struct { + ctx context.Context + ctxCancel context.CancelFunc + + config Config + skipMasks map[string]*SkipMask + + fracs map[string][]string + fracsMu *sync.RWMutex + + mp MappingProvider + + rateLimit chan struct{} + + bgWG *sync.WaitGroup + maintenanceInterval time.Duration + + cacheCleanInterval time.Duration + cacheGCDelay time.Duration + + headersCache *cache.Cache[[]lidsBlockHeader] + headersCacheCleaner *cache.Cleaner +} + +func New( + ctx context.Context, + cfg Config, + params []SkipMaskParams, + mp MappingProvider, +) *SkipMaskManager { + fmCtx, ctxCancel := context.WithCancel(ctx) + + workers := cfg.Workers + if workers <= 0 { + workers = runtime.GOMAXPROCS(0) + } + + skipMasksMap := make(map[string]*SkipMask, len(params)) + + for _, p := range params { + sm := NewSkipMask(p) + skipMasksMap[sm.Hash()] = sm + } + + cacheCleaner := cache.NewCleaner(cfg.CacheSizeLimit, nil) + + return &SkipMaskManager{ + ctx: fmCtx, + ctxCancel: ctxCancel, + config: cfg, + skipMasks: skipMasksMap, + fracs: make(map[string][]string), + fracsMu: &sync.RWMutex{}, + mp: mp, + rateLimit: make(chan struct{}, workers), + bgWG: &sync.WaitGroup{}, + maintenanceInterval: defaultMaintenanceInterval, + cacheCleanInterval: defaultCacheCleanInterval, + cacheGCDelay: defaultCacheGCDelay, + headersCache: cache.NewCache[[]lidsBlockHeader](cacheCleaner, nil), + headersCacheCleaner: cacheCleaner, + } +} + +func (smm *SkipMaskManager) Start(fracs fracmanager.List) { + smm.createDataDir() + + err := smm.loadSkipMasks() + if err != nil { + logger.Fatal("failed to load previous skip masks", zap.Error(err)) + } + + err = smm.buildQueue(fracs) + if err != nil { + logger.Fatal("failed to build skip mask manager queue", zap.Error(err)) + } + + smm.startMaintenance() + smm.cacheCleanLoop() + + mapping := smm.mp.GetMapping() + + smm.bgWG.Add(1) + go func() { + defer smm.bgWG.Done() + + for _, sm := range smm.skipMasks { + ast, err := parser.ParseSeqQL(sm.params.Query, mapping) + if err != nil { + panic(fmt.Errorf("BUG: search query must be valid: %s", err)) + } + sm.ast = ast + + smm.processSkipMask(sm, fracs.FilterInRange(sm.params.From, sm.params.To)) + } + }() +} + +func (smm *SkipMaskManager) Stop() { + smm.ctxCancel() + smm.bgWG.Wait() + logger.Info("skip mask manager stopped") +} + +func (smm *SkipMaskManager) GetIDsIteratorByFrac( + fracName string, + minLID, maxLID uint32, + reverse bool, +) (node.Node, bool, error) { + smm.fracsMu.RLock() + defer smm.fracsMu.RUnlock() + + fracFiles, has := smm.fracs[fracName] + if !has { + return &EmptyIterator{}, has, nil + } + + iterators := make([]node.Node, 0, len(fracFiles)) + for _, f := range fracFiles { + loader, err := newLoader(f, smm.headersCache) + if err != nil { + logger.Error("can't open skip mask file", zap.String("path", f), zap.Error(err)) + return nil, has, err + } + if reverse { + iterators = append(iterators, (*IteratorAsc)(NewIterator(loader, minLID, maxLID))) + } else { + iterators = append(iterators, (*IteratorDesc)(NewIterator(loader, minLID, maxLID))) + } + } + + return NewNMergedIterators(iterators), has, nil +} + +// RefreshFrac replaces frac's skip mask files with newly found results. Used after active frac is sealed. +func (smm *SkipMaskManager) RefreshFrac(fraction frac.Fraction) { + smm.fracsMu.Lock() + fracsFiles, has := smm.fracs[fraction.Info().Name()] + delete(smm.fracs, fraction.Info().Name()) + smm.fracsMu.Unlock() + + if !has { + return + } + + // mark skip masks as InProgress + for _, fileName := range fracsFiles { + smm.skipMasks[skipMaskNameFromPath(fileName)].setStatus(StatusInProgress) + } + + smm.bgWG.Add(1) + go func() { + defer smm.bgWG.Done() + + for _, fileName := range fracsFiles { + util.RemoveFile(fileName) + smm.headersCache.Evict(hashFilePath(fileName)) + + skipMask := smm.skipMasks[skipMaskNameFromPath(fileName)] + + queueFilePath := path.Join(skipMask.dirPath, makeFileName(fraction.Info().Name(), fracInQueueExt)) + util.MustWriteFileAtomic(queueFilePath, []byte{}, 0o666, tmpExt) + + select { + case <-smm.ctx.Done(): + // do not return because we have to create a .queue file for each of frac files to handle it on startup + continue + case smm.rateLimit <- struct{}{}: + go func() { + defer func() { <-smm.rateLimit }() + if err := smm.processFrac(fraction, skipMask); err != nil { + if errors.Is(err, context.Canceled) { + logger.Info("skip mask manager refresh frac context cancelled") + return + } + panic(fmt.Errorf("skip mask manager refresh frac err: %s", err)) + } + skipMask.setStatus(StatusDone) + }() + } + } + }() +} + +// RemoveFrac removes fraction's skip mask files. Used after frac is deleted +func (smm *SkipMaskManager) RemoveFrac(fracName string) { + // TODO: we might want to have some kind of GC on startup to clean up missed files + smm.bgWG.Go(func() { + smm.fracsMu.RLock() + fracsFiles, has := smm.fracs[fracName] + smm.fracsMu.RUnlock() + + if !has { + return + } + + smm.fracsMu.Lock() + delete(smm.fracs, fracName) + smm.fracsMu.Unlock() + + for _, fileName := range fracsFiles { + util.RemoveFile(fileName) + smm.headersCache.Evict(hashFilePath(fileName)) + } + }) +} + +func (smm *SkipMaskManager) IsDone() bool { + for _, sm := range smm.skipMasks { + if sm.getStatus() != StatusDone { + return false + } + } + return true +} + +func skipMaskNameFromPath(p string) string { + return path.Base(path.Dir(p)) +} + +func (smm *SkipMaskManager) addDoneFrac(fracName, fracPath string) { + smm.fracsMu.Lock() + defer smm.fracsMu.Unlock() + + smm.fracs[fracName] = append(smm.fracs[fracName], fracPath) +} + +// loadSkipMasks loads existing skip masks +func (smm *SkipMaskManager) loadSkipMasks() error { + des, err := os.ReadDir(smm.config.DataDir) + if err != nil { + return err + } + + var anyRemove bool + + for _, de := range des { + if !de.IsDir() { + continue + } + + if _, ok := smm.skipMasks[de.Name()]; !ok { + logger.Info("there is skip mask folder on disk, but not in config. need to delete it.") + err := os.RemoveAll(path.Join(smm.config.DataDir, de.Name())) + if err != nil && !os.IsNotExist(err) { + return err + } + anyRemove = true + continue + } + + sm := smm.skipMasks[de.Name()] + sm.setStatus(StatusInProgress) + sm.dirPath = path.Join(smm.config.DataDir, de.Name()) + + skipMaskDes, err := os.ReadDir(sm.dirPath) + if err != nil { + return fmt.Errorf("reading directory: %s", err) + } + + var hasFracsInQueue bool + + for _, smde := range skipMaskDes { + if smde.IsDir() { + continue + } + name := smde.Name() + + switch path.Ext(name) { + case fracInQueueExt: + hasFracsInQueue = true + case fracDoneExt: + smm.addDoneFrac(fracNameFromFilePath(name), path.Join(sm.dirPath, name)) + } + } + + if !hasFracsInQueue { + sm.setStatus(StatusDone) + } + } + + if anyRemove { + util.MustFsyncFile(smm.config.DataDir) + } + + return nil +} + +// buildQueue creates a directory for each of unprocessed skip masks and creates .queue files +func (smm *SkipMaskManager) buildQueue(fracs fracmanager.List) error { + for _, skipMask := range smm.skipMasks { + if skipMask.getStatus() != StatusCreated { + continue + } + + // create tmp dir + tmpDir := path.Join(smm.config.DataDir, fmt.Sprintf("%s%s", skipMask.Hash(), tmpDirSuffix)) + util.MustCreateDir(tmpDir) + + skipMaskFracs := fracs.FilterInRange(skipMask.params.From, skipMask.params.To) + for _, f := range skipMaskFracs { + queueFilePath := path.Join(tmpDir, makeFileName(f.Info().Name(), fracInQueueExt)) + util.MustWriteFileAtomic(queueFilePath, []byte{}, 0o666, tmpExt) + } + + // rename tmp dir + dir := path.Join(smm.config.DataDir, skipMask.Hash()) + if err := os.Rename(tmpDir, dir); err != nil { + return err + } + util.MustFsyncFile(smm.config.DataDir) + skipMask.dirPath = dir + } + + return nil +} + +// processSkipMask finds docs and writes to fs +func (smm *SkipMaskManager) processSkipMask(skipMask *SkipMask, fracs fracmanager.List) { + if len(fracs) == 0 { + skipMask.setStatus(StatusDone) + return + } + + fracsByName := make(map[string]frac.Fraction) + for _, f := range fracs { + fracsByName[f.Info().Name()] = f + } + + skipMaskDes, err := os.ReadDir(skipMask.dirPath) + if err != nil { + panic(fmt.Errorf("BUG: reading directory must be successful: %s", err)) + } + + inProgress.Add(1) + + processFracInQueue := func(name string) error { + f, ok := fracsByName[fracNameFromFilePath(name)] + if !ok { // skip missing fracs + return nil + } + + select { + case <-smm.ctx.Done(): + return nil + case smm.rateLimit <- struct{}{}: + skipMask.processWg.Add(1) + go func() { + defer skipMask.processWg.Done() + defer func() { <-smm.rateLimit }() + if err := smm.processFrac(f, skipMask); err != nil { + if errors.Is(err, context.Canceled) { + logger.Info("skip mask manager refresh frac context cancelled") + return + } + panic(fmt.Errorf("skip mask manager process frac err: %s", err)) + } + }() + } + return nil + } + _ = util.VisitFilesWithExt(skipMaskDes, fracInQueueExt, processFracInQueue) + + go func() { + skipMask.processWg.Wait() + skipMask.setStatus(StatusDone) + inProgress.Add(-1) + }() +} + +func (smm *SkipMaskManager) processFrac(f frac.Fraction, skipMask *SkipMask) error { + qpr, err := f.Search(smm.ctx, processor.SearchParams{ + AST: skipMask.ast.Root, + From: skipMask.params.From, + To: skipMask.params.To, + Limit: math.MaxInt64, + }) + if err != nil { + return err + } + + queueFilePath := path.Join(skipMask.dirPath, makeFileName(f.Info().Name(), fracInQueueExt)) + doneFilePath := path.Join(skipMask.dirPath, makeFileName(f.Info().Name(), fracDoneExt)) + + if len(qpr.IDs) == 0 { + util.RemoveFile(queueFilePath) + return nil + } + + // TODO: here we doing part of the work twice: + // first time we find LIDs inside f.Search() and then find IDs by these LIDs. + // Then we again find LIDs by earlier found IDs in f.FindLIDs(). + // We did it like this because otherwise we had to do serious f.Search() rewrite. + // For now we're ok with some performance penalty. + lids, err := f.FindLIDs(smm.ctx, qpr.IDs.IDs()) + if err != nil { + return err + } + + skipMaskBin := SkipMaskBinIn{LIDs: lids} + if err := writeSkipMask(&skipMaskBin, queueFilePath, doneFilePath); err != nil { + return err + } + + smm.addDoneFrac(f.Info().Name(), doneFilePath) + + return nil +} + +func (smm *SkipMaskManager) startMaintenance() { + smm.bgWG.Go(func() { + logger.Info("start skip mask manager maintenance") + util.RunEvery(smm.ctx.Done(), smm.maintenanceInterval, func() { + logger.Info("skip mask manager maintenance iteration") + smm.checkDiskUsage() + }) + }) +} + +func (smm *SkipMaskManager) cacheCleanLoop() { + smm.bgWG.Go(func() { + runs := 0 + gcRunsCount := int(smm.cacheGCDelay / smm.cacheCleanInterval) + + util.RunEvery(smm.ctx.Done(), smm.cacheCleanInterval, func() { + runs++ + smm.headersCacheCleaner.Cleanup(&cache.CleanStat{}) + smm.headersCacheCleaner.Rotate() + + if runs >= gcRunsCount { + runs = 0 + smm.headersCacheCleaner.CleanEmptyGenerations() + smm.headersCacheCleaner.ReleaseBuckets() + } + }) + }) +} + +func (smm *SkipMaskManager) checkDiskUsage() { + du := int64(0) + + for _, sm := range smm.skipMasks { + des, err := os.ReadDir(sm.dirPath) + if err != nil { + logger.Error("skip mask manager: can't read skip mask's dir", + zap.String("skip mask", sm.String()), zap.Error(err)) + return + } + + for _, smde := range des { + if smde.IsDir() { + continue + } + info, err := smde.Info() + if err != nil { + logger.Error("skip mask manager: can't read skip mask file info", + zap.String("skip mask", sm.String()), zap.Error(err)) + return + } + du += info.Size() + } + } + + diskUsage.Set(float64(du)) + stored.Set(float64(len(smm.skipMasks))) +} + +func makeFileName(name, ext string) string { + return name + ext +} + +func fracNameFromFilePath(skipMaskFilePath string) string { + return strings.Split(path.Base(skipMaskFilePath), ".")[0] +} + +func hashFilePath(filePath string) uint32 { + hash := fnv.New32a() + hash.Write([]byte(skipMaskNameFromPath(filePath) + fracNameFromFilePath(filePath))) + return hash.Sum32() +} + +var marshalBufferPool util.BufferPool + +func writeSkipMask(df *SkipMaskBinIn, queueFilePath, doneFilePath string) error { + rawSkipMask := marshalBufferPool.Get() + defer marshalBufferPool.Put(rawSkipMask) + + rawSkipMask.B = marshalSkipMask(rawSkipMask.B, df) + util.MustWriteFileAtomic(doneFilePath, rawSkipMask.B, 0o666, tmpExt) + util.RemoveFile(queueFilePath) + + return nil +} + +// createDataDir creates data dir. +func (smm *SkipMaskManager) createDataDir() { + if err := os.MkdirAll(smm.config.DataDir, 0o777); err != nil { + panic(err) + } +} diff --git a/storeapi/grpc_v1_test.go b/storeapi/grpc_v1_test.go index 29b68f8d..2dba2fc2 100644 --- a/storeapi/grpc_v1_test.go +++ b/storeapi/grpc_v1_test.go @@ -12,12 +12,12 @@ import ( "github.com/ozontech/seq-db/asyncsearcher" "github.com/ozontech/seq-db/consts" - "github.com/ozontech/seq-db/filtermanager" "github.com/ozontech/seq-db/fracmanager" "github.com/ozontech/seq-db/indexer" "github.com/ozontech/seq-db/mappingprovider" "github.com/ozontech/seq-db/pkg/storeapi" "github.com/ozontech/seq-db/seq" + "github.com/ozontech/seq-db/skipmaskmanager" "github.com/ozontech/seq-db/tests/common" ) @@ -71,13 +71,13 @@ func getTestGrpc(t *testing.T) (*GrpcV1, func(), func()) { mappingProvider, err := mappingprovider.New("", mappingprovider.WithMapping(seq.TestMapping)) assert.NoError(t, err) - filterManager := filtermanager.New(t.Context(), filtermanager.Config{}, nil, mappingProvider) + skipMaskManager := skipmaskmanager.New(t.Context(), skipmaskmanager.Config{}, nil, mappingProvider) fm, stop, err := fracmanager.New(t.Context(), &fracmanager.Config{ FracSize: 500, TotalSize: 5000, DataDir: dataDir, - }, nil, filterManager) + }, nil, skipMaskManager) assert.NoError(t, err) config := APIConfig{ diff --git a/storeapi/store.go b/storeapi/store.go index b6ac70e6..5ea0c5a7 100644 --- a/storeapi/store.go +++ b/storeapi/store.go @@ -9,10 +9,10 @@ import ( "go.uber.org/atomic" "github.com/ozontech/seq-db/consts" - "github.com/ozontech/seq-db/filtermanager" "github.com/ozontech/seq-db/fracmanager" "github.com/ozontech/seq-db/logger" "github.com/ozontech/seq-db/metric" + "github.com/ozontech/seq-db/skipmaskmanager" "github.com/ozontech/seq-db/storage/s3" ) @@ -30,15 +30,15 @@ type Store struct { FracManager *fracmanager.FracManager fracManagerStop func() - filterManagerStop func() + SkipMaskManager *skipmaskmanager.SkipMaskManager isStopped atomic.Bool } type StoreConfig struct { - API APIConfig - FracManager fracmanager.Config - Filters filtermanager.Config + API APIConfig + FracManager fracmanager.Config + SkipMaskManagerConfig skipmaskmanager.Config } func (c *StoreConfig) setDefaults() error { @@ -48,8 +48,8 @@ func (c *StoreConfig) setDefaults() error { if c.API.Search.Async.DataDir == "" { c.API.Search.Async.DataDir = path.Join(c.FracManager.DataDir, "async_searches") } - if c.Filters.DataDir == "" { - c.Filters.DataDir = path.Join(c.FracManager.DataDir, "filters") + if c.SkipMaskManagerConfig.DataDir == "" { + c.SkipMaskManagerConfig.DataDir = path.Join(c.FracManager.DataDir, "skipmasks") } return nil } @@ -59,30 +59,30 @@ func NewStore( c StoreConfig, s3cli *s3.Client, mappingProvider MappingProvider, - docFilterParams []filtermanager.Params, + skipMaskParams []skipmaskmanager.SkipMaskParams, ) (*Store, error) { if err := c.setDefaults(); err != nil { return nil, err } - filterManager := filtermanager.New(ctx, c.Filters, docFilterParams, mappingProvider) + skipMaskManager := skipmaskmanager.New(ctx, c.SkipMaskManagerConfig, skipMaskParams, mappingProvider) - fracManager, stop, err := fracmanager.New(ctx, &c.FracManager, s3cli, filterManager) + fracManager, stop, err := fracmanager.New(ctx, &c.FracManager, s3cli, skipMaskManager) if err != nil { return nil, fmt.Errorf("loading fractions error: %w", err) } - filterManager.Start(fracManager.Fractions()) + skipMaskManager.Start(fracManager.Fractions()) return &Store{ Config: c, // We will set grpcAddr later in Start() - grpcAddr: "", - grpcServer: newGRPCServer(c.API, fracManager, mappingProvider), - FracManager: fracManager, - fracManagerStop: stop, - filterManagerStop: filterManager.Stop, - isStopped: atomic.Bool{}, + grpcAddr: "", + grpcServer: newGRPCServer(c.API, fracManager, mappingProvider), + FracManager: fracManager, + fracManagerStop: stop, + SkipMaskManager: skipMaskManager, + isStopped: atomic.Bool{}, }, nil } @@ -106,7 +106,7 @@ func (s *Store) Stop() { s.grpcServer.Stop(ctx) s.fracManagerStop() - s.filterManagerStop() + s.SkipMaskManager.Stop() logger.Info("store stopped") } diff --git a/tests/integration_tests/integration_test.go b/tests/integration_tests/integration_test.go index 5f7b1127..d2abf818 100644 --- a/tests/integration_tests/integration_test.go +++ b/tests/integration_tests/integration_test.go @@ -27,11 +27,11 @@ import ( "github.com/ozontech/seq-db/asyncsearcher" "github.com/ozontech/seq-db/consts" - "github.com/ozontech/seq-db/filtermanager" "github.com/ozontech/seq-db/pkg/seqproxyapi/v1" "github.com/ozontech/seq-db/pkg/storeapi" "github.com/ozontech/seq-db/proxy/search" "github.com/ozontech/seq-db/seq" + "github.com/ozontech/seq-db/skipmaskmanager" "github.com/ozontech/seq-db/tests/common" "github.com/ozontech/seq-db/tests/setup" "github.com/ozontech/seq-db/tests/suites" @@ -1753,7 +1753,7 @@ func (s *IntegrationTestSuite) TestPaginationWithOffsetId() { } } -func (s *IntegrationTestSuite) TestFilterManager() { +func (s *IntegrationTestSuite) TestSkipMaskManager() { t := s.T() r := require.New(t) @@ -1781,18 +1781,31 @@ func (s *IntegrationTestSuite) TestFilterManager() { env.WaitIdle() env.StopAll() - cfg.DocsFilters = []filtermanager.Params{ + cfg.SkipMaskParams = []skipmaskmanager.SkipMaskParams{ { Query: "service:hidden", From: 0, - To: seq.MID(time.Now().UnixNano()), + To: seq.TimeToMID(time.Now()), }, } env = setup.NewTestingEnv(&cfg) defer env.StopAll() - // we don't have a convenient way to wait for doc filters processing, so just wait enough time for now - time.Sleep(1 * time.Second) + var checkSkipMasksStatus = func(stores setup.Stores) bool { + for _, ss := range stores { + for _, s := range ss { + if !s.SkipMaskManager.IsDone() { + return false + } + } + } + return true + } + + // wait for skip masks processing + r.Eventually(func() bool { + return checkSkipMasksStatus(env.HotStores) && checkSkipMasksStatus(env.ColdStores) + }, 5*time.Second, 100*time.Millisecond) // test search @@ -1817,7 +1830,11 @@ func (s *IntegrationTestSuite) TestFilterManager() { env.WaitIdle() env.SealAll() - time.Sleep(1 * time.Second) + + // wait for skip masks processing + r.Eventually(func() bool { + return checkSkipMasksStatus(env.HotStores) && checkSkipMasksStatus(env.ColdStores) + }, 5*time.Second, 100*time.Millisecond) qpr, _, _, err = env.Search(`service:hidden`, 10, setup.WithTotal(true)) r.NoError(err) diff --git a/tests/setup/env.go b/tests/setup/env.go index 3d003c25..a42ba006 100644 --- a/tests/setup/env.go +++ b/tests/setup/env.go @@ -22,7 +22,6 @@ import ( "github.com/ozontech/seq-db/buildinfo" "github.com/ozontech/seq-db/consts" - "github.com/ozontech/seq-db/filtermanager" "github.com/ozontech/seq-db/frac/common" "github.com/ozontech/seq-db/fracmanager" "github.com/ozontech/seq-db/logger" @@ -33,6 +32,7 @@ import ( "github.com/ozontech/seq-db/proxy/stores" "github.com/ozontech/seq-db/proxyapi" "github.com/ozontech/seq-db/seq" + "github.com/ozontech/seq-db/skipmaskmanager" seqs3 "github.com/ozontech/seq-db/storage/s3" "github.com/ozontech/seq-db/storeapi" testscommon "github.com/ozontech/seq-db/tests/common" @@ -49,7 +49,7 @@ type TestingEnvConfig struct { HotModeEnabled bool QueryRateLimit *float64 FracManagerConfig *fracmanager.Config - DocsFilters []filtermanager.Params + SkipMaskParams []skipmaskmanager.SkipMaskParams Mapping seq.Mapping IndexAllFields bool @@ -124,8 +124,8 @@ func (cfg *TestingEnvConfig) GetStoreConfig(replicaID string, cold bool) storeap LogThreshold: 0, }, }, - Filters: filtermanager.Config{ - DataDir: filepath.Join(cfg.DataDir, replicaID, "filters"), + SkipMaskManagerConfig: skipmaskmanager.Config{ + DataDir: filepath.Join(cfg.DataDir, replicaID, "skipmasks"), }, } } @@ -280,7 +280,7 @@ func (cfg *TestingEnvConfig) MakeStores( logger.Fatal("can't create mapping", zap.Error(err)) } - store, err := storeapi.NewStore(context.Background(), confs[i], s3cli, mappingProvider, cfg.DocsFilters) + store, err := storeapi.NewStore(context.Background(), confs[i], s3cli, mappingProvider, cfg.SkipMaskParams) if err != nil { panic(err) } From 27e61d1e12b1084d21030a3ef63d70f11d4afa03 Mon Sep 17 00:00:00 2001 From: Daniil Forshev Date: Fri, 10 Apr 2026 17:33:32 +0500 Subject: [PATCH 10/10] chore(skipmaskmanager): add doc comments --- skipmaskmanager/encoding.go | 56 +++++++++++--- skipmaskmanager/merged_iterator_test.go | 4 +- skipmaskmanager/skip_mask_manager.go | 97 ++++++++++++++++++++++--- 3 files changed, 135 insertions(+), 22 deletions(-) diff --git a/skipmaskmanager/encoding.go b/skipmaskmanager/encoding.go index 69b6e5a1..c0bd9c81 100644 --- a/skipmaskmanager/encoding.go +++ b/skipmaskmanager/encoding.go @@ -12,10 +12,15 @@ import ( "github.com/ozontech/seq-db/zstd" ) +// SkipMaskBinIn is the input structure for serializing a skip mask. +// It contains a slice of Local IDs (LIDs) that correspond to documents +// matching the skip mask query criteria. type SkipMaskBinIn struct { LIDs []seq.LID } +// SkipMaskBinOut is the output structure for deserialized skip mask data. +// After unmarshaling, LIDs are converted to uint32 array. type SkipMaskBinOut struct { LIDs []uint32 } @@ -30,22 +35,28 @@ var availableVersions = map[skipMaskBinVersion]struct{}{ skipMaskBinVersion1: {}, } +// lidsCodec represents the compression codec used for LIDs block encoding. type lidsCodec byte const ( - lidsCodecDelta = 1 - lidsCodecDeltaZstd = 2 + lidsCodecDelta = 1 // Delta-encoded varints without compression + lidsCodecDeltaZstd = 2 // Delta-encoded varints with zstd compression ) +// lidsBlockHeader contains metadata for a block of LIDs. +// Each block stores a subset of LIDs (up to maxLIDsBlockLen) along with +// information needed to decode and locate the block data. type lidsBlockHeader struct { - Codec lidsCodec - Length uint32 // Number of LIDs in block - MinLID uint32 - MaxLID uint32 - Size uint32 // Size of ids block in bytes. - Offset uint64 // block's offset in file + Codec lidsCodec // Compression codec used for this block (delta or delta+zstd) + Length uint32 // Number of LIDs in this block + MinLID uint32 // Minimum LID value in the block + MaxLID uint32 // Maximum LID value in the block + Size uint32 // Size of the compressed block data in bytes + Offset uint64 // Offset of the block data in the file } +// marshal serializes the block header into the provided byte slice. +// The format is: Codec (1 byte) + Length (4 bytes) + MinLID (4 bytes) + MaxLID (4 bytes) + Size (4 bytes) + Offset (8 bytes) = 25 bytes. func (h *lidsBlockHeader) marshal(dst []byte) { if len(dst) < int(lidsBlockHeaderSizeBytes) { panic("BUG: marshal lidsBlockHeader: len(dst) is less than header size") @@ -65,6 +76,8 @@ func (h *lidsBlockHeader) marshal(dst []byte) { dst = dst[sizeOfUint64:] } +// unmarshal deserializes a block header from the provided byte slice. +// Returns the remaining unconsumed bytes and any error encountered. func (h *lidsBlockHeader) unmarshal(src []byte) ([]byte, error) { if len(src) < int(lidsBlockHeaderSizeBytes) { return src, errors.New("too few bytes") @@ -86,6 +99,8 @@ func (h *lidsBlockHeader) unmarshal(src []byte) ([]byte, error) { return src, nil } +// marshalSkipMask serializes a skip mask into binary format. +// Returns the serialized data with the version byte prepended. func marshalSkipMask(dst []byte, in *SkipMaskBinIn) []byte { dst = append(dst, uint8(skipMaskBinVersion1)) dst = marshalLIDsBlocks(dst, in.LIDs) @@ -93,17 +108,22 @@ func marshalSkipMask(dst []byte, in *SkipMaskBinIn) []byte { } const ( - sizeOfUint32 = unsafe.Sizeof(uint32(0)) - sizeOfUint64 = unsafe.Sizeof(uint64(0)) + sizeOfUint32 = unsafe.Sizeof(uint32(0)) // 4 bytes + sizeOfUint64 = unsafe.Sizeof(uint64(0)) // 8 bytes ) const ( + // lidsBlockHeaderSizeBytes is the size of a single block header in bytes: 1 (Codec) + 4*4 (Length, MinLID, MaxLID, Size) + 8 (Offset) = 25 lidsBlockHeaderSizeBytes = 1 + (4 * sizeOfUint32) + sizeOfUint64 - maxLIDsBlockLen = 1024 + // maxLIDsBlockLen is the maximum number of LIDs stored in a single block + maxLIDsBlockLen = 1024 ) var lidsBlockBufPool util.BufferPool +// marshalLIDsBlocks splits the input LIDs into blocks and serializes them. +// Each block contains up to maxLIDsBlockLen LIDs. The output format is: +// [number of blocks: 4 bytes] [block 1 header] [block 2 header] ... [block 1 data] [block 2 data] ... func marshalLIDsBlocks(dst []byte, in []seq.LID) []byte { b := lidsBlockBufPool.Get() defer lidsBlockBufPool.Put(b) @@ -144,6 +164,10 @@ func marshalLIDsBlocks(dst []byte, in []seq.LID) []byte { return dst } +// marshalLIDsBlock encodes a slice of LIDs using delta compression. +// It first computes delta-encoded varints, then attempts zstd compression. +// If zstd provides at least 5% compression, it uses zstd; otherwise, it stores +// the raw delta-encoded data. Returns the encoded data and the codec used. func marshalLIDsBlock(dst []byte, in []seq.LID) ([]byte, lidsCodec) { b := lidsBlockBufPool.Get() defer lidsBlockBufPool.Put(b) @@ -170,6 +194,8 @@ func marshalLIDsBlock(dst []byte, in []seq.LID) ([]byte, lidsCodec) { const minSkipMaskBytesLen = 10 // 1 byte skipMaskBinVersion + 8 byte number of LIDs + N (min 1) bytes varint + delta encoded LIDs +// unmarshalSkipMask deserializes a skip mask from binary format. +// Validates the version and delegates to unmarshalLIDsBlocks for block processing. func unmarshalSkipMask(dst *SkipMaskBinOut, src []byte) (_ []byte, err error) { if len(src) < minSkipMaskBytesLen { return nil, fmt.Errorf("invalid skip mask format; want %d bytes, got %d", minSkipMaskBytesLen, len(src)) @@ -189,6 +215,9 @@ func unmarshalSkipMask(dst *SkipMaskBinOut, src []byte) (_ []byte, err error) { return src, nil } +// unmarshalLIDsBlocks reads all LIDs blocks from the source data. +// First reads the number of blocks, then parses each block header, +// and finally decodes each block's data. func unmarshalLIDsBlocks(dst []uint32, src []byte) ([]uint32, []byte, error) { numberOfBlocks := binary.LittleEndian.Uint32(src) src = src[sizeOfUint32:] @@ -219,6 +248,8 @@ func unmarshalLIDsBlocks(dst []uint32, src []byte) ([]uint32, []byte, error) { return dst, src, nil } +// unmarshalLIDsBlock decodes a single LIDs block based on its header. +// Handles both compressed (zstd) and uncompressed codec types. func unmarshalLIDsBlock(dst []uint32, src []byte, header lidsBlockHeader) ([]uint32, []byte, error) { if len(src) == 0 { return dst, src, fmt.Errorf("empty LIDs block") @@ -274,6 +305,9 @@ func unmarshalLIDsDelta(dst []uint32, block []byte, header lidsBlockHeader) ([]u return dst, nil } +// getCompressLevel returns the appropriate zstd compression level based on data size. +// Higher compression levels are used for larger data to achieve better ratios. +// Returns: 1 for <=512 bytes, 2 for <=4KB, 3 for larger data. func getCompressLevel(size int) int { level := 3 if size <= 512 { diff --git a/skipmaskmanager/merged_iterator_test.go b/skipmaskmanager/merged_iterator_test.go index 8f448efd..fe0af2af 100644 --- a/skipmaskmanager/merged_iterator_test.go +++ b/skipmaskmanager/merged_iterator_test.go @@ -69,7 +69,7 @@ func (it *testIteratorDesc) Next() node.LID { } func (it *testIteratorDesc) NextGeq(nextID node.LID) node.LID { - return node.NullLID() // TODO: ??? + return node.NullLID() } type testIteratorAsc struct { @@ -87,7 +87,7 @@ func (it *testIteratorAsc) Next() node.LID { lid := it.lids[0] it.lids = it.lids[1:] - return node.NewAscLID(lid) // TODO: ??? + return node.NewAscLID(lid) } func (it *testIteratorAsc) NextGeq(nextID node.LID) node.LID { diff --git a/skipmaskmanager/skip_mask_manager.go b/skipmaskmanager/skip_mask_manager.go index 4ee0f637..055cca33 100644 --- a/skipmaskmanager/skip_mask_manager.go +++ b/skipmaskmanager/skip_mask_manager.go @@ -44,12 +44,24 @@ type MappingProvider interface { GetMapping() seq.Mapping } +// Config holds configuration parameters for SkipMaskManager. type Config struct { - DataDir string - Workers int - CacheSizeLimit uint64 + DataDir string // Directory to store skip mask files + Workers int // Number of concurrent workers for processing + CacheSizeLimit uint64 // Maximum size of the headers cache in bytes } +// SkipMaskManager manages the lifecycle of skip masks across all fractions. +// It processes skip mask queries, stores results to disk, and provides +// iteration over matching document IDs. +// +// Skip masks are organized by query parameters (query string, from/to MID range). +// Each skip mask maintains a directory containing files for each fraction: +// - .queue file: fraction is currently being processed +// - .skipmask file: processing complete, contains matching document LIDs +// +// The manager runs background maintenance tasks for disk usage monitoring +// and cache cleanup. type SkipMaskManager struct { ctx context.Context ctxCancel context.CancelFunc @@ -74,6 +86,8 @@ type SkipMaskManager struct { headersCacheCleaner *cache.Cleaner } +// New creates a new SkipMaskManager with the given configuration. +// If Workers is not set (0), it defaults to GOMAXPROCS. func New( ctx context.Context, cfg Config, @@ -114,6 +128,15 @@ func New( } } +// Start initializes and starts the skip mask manager. +// It performs the following steps: +// - Creates the data directory if it doesn't exist +// - Loads any existing skip masks from previous sessions +// - Builds the processing queue for all fractions +// - Starts background maintenance and cache cleanup loops +// - Begins asynchronous processing of all skip mask queries +// +// This method must be called before using the manager. func (smm *SkipMaskManager) Start(fracs fracmanager.List) { smm.createDataDir() @@ -148,12 +171,28 @@ func (smm *SkipMaskManager) Start(fracs fracmanager.List) { }() } +// Stop gracefully stops the skip mask manager. +// It cancels the context, waits for all background goroutines to complete, +// and logs a message when fully stopped. func (smm *SkipMaskManager) Stop() { smm.ctxCancel() smm.bgWG.Wait() logger.Info("skip mask manager stopped") } +// GetIDsIteratorByFrac returns an iterator over document IDs that match +// the skip mask queries for a specific fraction, within the given LID range. +// +// Parameters: +// - fracName: the name of the fraction to query +// - minLID: minimum local ID (inclusive) +// - maxLID: maximum local ID (inclusive) +// - reverse: if true, iterates IDs in descending order +// +// Returns: +// - node.Node: iterator over matching document IDs +// - bool: true if the fraction has any skip mask files, false otherwise +// - error: any error encountered while opening the skip mask files func (smm *SkipMaskManager) GetIDsIteratorByFrac( fracName string, minLID, maxLID uint32, @@ -184,7 +223,13 @@ func (smm *SkipMaskManager) GetIDsIteratorByFrac( return NewNMergedIterators(iterators), has, nil } -// RefreshFrac replaces frac's skip mask files with newly found results. Used after active frac is sealed. +// RefreshFrac recomputes skip mask files for a fraction after it has been sealed. +// This is called when an active fraction becomes sealed. +// The method: +// - Removes existing skip mask files for the fraction +// - Marks relevant skip masks as in-progress +// - Queues the fraction for reprocessing +// - Asynchronously processes the fraction through all matching skip masks func (smm *SkipMaskManager) RefreshFrac(fraction frac.Fraction) { smm.fracsMu.Lock() fracsFiles, has := smm.fracs[fraction.Info().Name()] @@ -234,7 +279,9 @@ func (smm *SkipMaskManager) RefreshFrac(fraction frac.Fraction) { }() } -// RemoveFrac removes fraction's skip mask files. Used after frac is deleted +// RemoveFrac removes all skip mask files associated with a fraction. +// This should be called when a fraction is deleted from the system. +// The removal is performed asynchronously in the background. func (smm *SkipMaskManager) RemoveFrac(fracName string) { // TODO: we might want to have some kind of GC on startup to clean up missed files smm.bgWG.Go(func() { @@ -257,6 +304,8 @@ func (smm *SkipMaskManager) RemoveFrac(fracName string) { }) } +// IsDone returns true if all skip masks have been processed and are in StatusDone state. +// This is useful for determining when initial skip mask computation is complete. func (smm *SkipMaskManager) IsDone() bool { for _, sm := range smm.skipMasks { if sm.getStatus() != StatusDone { @@ -270,6 +319,8 @@ func skipMaskNameFromPath(p string) string { return path.Base(path.Dir(p)) } +// addDoneFrac registers a completed fraction's skip mask file path. +// Called when a fraction's skip mask processing finishes successfully. func (smm *SkipMaskManager) addDoneFrac(fracName, fracPath string) { smm.fracsMu.Lock() defer smm.fracsMu.Unlock() @@ -277,7 +328,13 @@ func (smm *SkipMaskManager) addDoneFrac(fracName, fracPath string) { smm.fracs[fracName] = append(smm.fracs[fracName], fracPath) } -// loadSkipMasks loads existing skip masks +// loadSkipMasks loads skip masks from a previous session. +// It scans the data directory for existing skip mask files and: +// - Removes directories that are not in the current configuration +// - Marks in-progress skip masks based on .queue files +// - Registers completed skip masks (.skipmask files) +// +// This allows the manager to resume processing after a restart. func (smm *SkipMaskManager) loadSkipMasks() error { des, err := os.ReadDir(smm.config.DataDir) if err != nil { @@ -338,7 +395,13 @@ func (smm *SkipMaskManager) loadSkipMasks() error { return nil } -// buildQueue creates a directory for each of unprocessed skip masks and creates .queue files +// buildQueue initializes the processing queue for newly created skip masks. +// For each skip mask in StatusCreated state, it: +// - Creates a temporary directory +// - Generates .queue files for all fractions in the mask's range +// - Atomically renames the temp directory to the final name +// +// This sets up the files needed for parallel processing. func (smm *SkipMaskManager) buildQueue(fracs fracmanager.List) error { for _, skipMask := range smm.skipMasks { if skipMask.getStatus() != StatusCreated { @@ -367,7 +430,10 @@ func (smm *SkipMaskManager) buildQueue(fracs fracmanager.List) error { return nil } -// processSkipMask finds docs and writes to fs +// processSkipMask executes the skip mask query against all fractions in range. +// It processes each fraction with a .queue file, running search queries in parallel +// (limited by the rate limiter). Each successful search writes results to a .skipmask +// file. The skip mask status is set to Done when all fractions are processed. func (smm *SkipMaskManager) processSkipMask(skipMask *SkipMask, fracs fracmanager.List) { if len(fracs) == 0 { skipMask.setStatus(StatusDone) @@ -420,6 +486,10 @@ func (smm *SkipMaskManager) processSkipMask(skipMask *SkipMask, fracs fracmanage }() } +// processFrac executes the skip mask query against a single fraction. +// It performs a search to find matching document IDs, then converts them +// to local IDs (LIDs), and writes the serialized skip mask to disk. +// The .queue file is replaced with a .skipmask file upon completion. func (smm *SkipMaskManager) processFrac(f frac.Fraction, skipMask *SkipMask) error { qpr, err := f.Search(smm.ctx, processor.SearchParams{ AST: skipMask.ast.Root, @@ -459,6 +529,8 @@ func (smm *SkipMaskManager) processFrac(f frac.Fraction, skipMask *SkipMask) err return nil } +// startMaintenance runs a background goroutine that periodically checks +// disk usage metrics. It logs the total size of all skip mask files. func (smm *SkipMaskManager) startMaintenance() { smm.bgWG.Go(func() { logger.Info("start skip mask manager maintenance") @@ -469,6 +541,9 @@ func (smm *SkipMaskManager) startMaintenance() { }) } +// cacheCleanLoop runs a background goroutine that periodically cleans up +// the headers cache. It performs incremental cleanup on each tick and +// full GC periodically (based on cacheGCDelay). func (smm *SkipMaskManager) cacheCleanLoop() { smm.bgWG.Go(func() { runs := 0 @@ -488,6 +563,8 @@ func (smm *SkipMaskManager) cacheCleanLoop() { }) } +// checkDiskUsage calculates and reports the total disk space used by all +// skip mask files. Updates the diskUsage and stored metrics. func (smm *SkipMaskManager) checkDiskUsage() { du := int64(0) @@ -533,6 +610,8 @@ func hashFilePath(filePath string) uint32 { var marshalBufferPool util.BufferPool +// writeSkipMask serializes the skip mask data and atomically writes it to disk. +// It removes the .queue file and creates the .skipmask file. func writeSkipMask(df *SkipMaskBinIn, queueFilePath, doneFilePath string) error { rawSkipMask := marshalBufferPool.Get() defer marshalBufferPool.Put(rawSkipMask) @@ -544,7 +623,7 @@ func writeSkipMask(df *SkipMaskBinIn, queueFilePath, doneFilePath string) error return nil } -// createDataDir creates data dir. +// createDataDir ensures the data directory exists, creating it if necessary. func (smm *SkipMaskManager) createDataDir() { if err := os.MkdirAll(smm.config.DataDir, 0o777); err != nil { panic(err)