Skip to content

Adding a default range to events that don't have the range field on searching/filtering/slicing operations #25577

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
janheise wants to merge 13 commits into
base: master
Choose a base branch
from
Draft

Adding a default range to events that don't have the range field on searching/filtering/slicing operations #25577

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
cd06778
adding a default range to documents that don't have the field
janheise Apr 8, 2026
2887ed9
fixing 0 case
janheise Apr 8, 2026
702e5f5
fixing 0 case
janheise Apr 9, 2026
5b3c085
Merge branch 'master' into events/fix-range-filter
janheise Apr 9, 2026
83e793e
adding should clause to query
janheise Apr 9, 2026
02035af
Merge remote-tracking branch 'origin/events/fix-range-filter' into ev…
janheise Apr 9, 2026
864387c
adding query to all adapters
janheise Apr 9, 2026
329cb71
Merge branch 'master' into events/fix-range-filter
janheise Apr 9, 2026
22329bc
Merge remote-tracking branch 'origin/master' into events/fix-range-fi…
janheise Apr 13, 2026
7c4754d
fixing missing first/last
janheise Apr 13, 2026
a9f8a94
add better check
janheise Apr 13, 2026
5e7308c
renamed per suggestion
janheise Apr 14, 2026
4fb7bb9
restructuring
janheise Apr 14, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 33 additions & 12 deletions ...elasticsearch7/src/main/java/org/graylog/storage/elasticsearch7/MoreSearchAdapterES7.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,6 @@
import org.graylog.shaded.elasticsearch7.org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval;
import org.graylog.shaded.elasticsearch7.org.elasticsearch.search.aggregations.bucket.histogram.ExtendedBounds;
import org.graylog.shaded.elasticsearch7.org.elasticsearch.search.aggregations.bucket.histogram.ParsedDateHistogram;
import org.graylog.shaded.elasticsearch7.org.elasticsearch.search.aggregations.bucket.range.ParsedRange;
import org.graylog.shaded.elasticsearch7.org.elasticsearch.search.aggregations.bucket.range.RangeAggregationBuilder;
import org.graylog.shaded.elasticsearch7.org.elasticsearch.search.aggregations.bucket.terms.ParsedTerms;
import org.graylog.shaded.elasticsearch7.org.elasticsearch.search.builder.SearchSourceBuilder;
Expand All @@ -67,18 +66,21 @@
import java.time.ZoneId;
import java.time.ZonedDateTime;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicReference;
import java.util.stream.Collectors;

import static com.google.common.base.Strings.isNullOrEmpty;
import static java.util.Objects.requireNonNull;
import static org.graylog.shaded.elasticsearch7.org.elasticsearch.index.query.QueryBuilders.boolQuery;
import static org.graylog.shaded.elasticsearch7.org.elasticsearch.index.query.QueryBuilders.existsQuery;
import static org.graylog.shaded.elasticsearch7.org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
import static org.graylog.shaded.elasticsearch7.org.elasticsearch.index.query.QueryBuilders.queryStringQuery;
import static org.graylog.shaded.elasticsearch7.org.elasticsearch.index.query.QueryBuilders.termsQuery;
Expand Down Expand Up @@ -109,7 +111,7 @@ public MoreSearchAdapterES7(ES7ResultMessageFactory resultMessageFactory,
public MoreSearch.Result eventSearch(String queryString, TimeRange timerange, Set<String> affectedIndices,
Sorting sorting, int page, int perPage, Set<String> eventStreams,
String filterString, Set<String> forbiddenSourceStreams, Map<String, Set<String>> extraFilters) {
final var filter = createQuery(queryString, timerange, eventStreams, filterString, forbiddenSourceStreams, extraFilters);
final var filter = checkAndAddRangeFilterFixWhenFieldIsMissing(createQuery(queryString, timerange, eventStreams, filterString, forbiddenSourceStreams, extraFilters), extraFilters);

final SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder()
.query(filter)
Expand Down Expand Up @@ -151,7 +153,7 @@ public MoreSearch.Result eventSearch(String queryString, TimeRange timerange, Se
public MoreSearch.Histogram eventHistogram(String queryString, AbsoluteRange timerange, Set<String> affectedIndices,
Set<String> eventStreams, String filterString, Set<String> forbiddenSourceStreams, ZoneId timeZone,
Map<String, Set<String>> extraFilters) {
final var filter = createQuery(queryString, timerange, eventStreams, filterString, forbiddenSourceStreams, extraFilters);
final var filter = checkAndAddRangeFilterFixWhenFieldIsMissing(createQuery(queryString, timerange, eventStreams, filterString, forbiddenSourceStreams, extraFilters), extraFilters);

final SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder()
.query(filter)
Expand Down Expand Up @@ -210,8 +212,25 @@ public MoreSearch.Histogram eventHistogram(String queryString, AbsoluteRange tim
return new MoreSearch.Histogram(new MoreSearch.Histogram.EventsBuckets(events, alerts));
}

private QueryBuilder createQuery(String queryString, TimeRange timerange, Set<String> eventStreams, String filterString,
Set<String> forbiddenSourceStreams, Map<String, Set<String>> extraFilters) {
private QueryBuilder addFieldIsMissing(QueryBuilder filter) {
return boolQuery().should(filter).should(boolQuery().mustNot(existsQuery(EventDto.FIELD_SCORES_NORMALIZED_RISK))).minimumShouldMatch("0");
}

private QueryBuilder checkAndAddRangeFilterFixWhenFieldIsMissing(QueryBuilder filter, Map<String, Set<String>> extraFilters) {
if(extraFilters.values().stream().flatMap(Collection::stream).anyMatch(MoreSearchAdapter::isLowerBoundZeroRangeFilter)) {
return addFieldIsMissing(filter);
} else {
return filter;
}
}

private QueryBuilder createQuery(String queryString,
TimeRange timerange,
Set<String> eventStreams,
String filterString,
Set<String> forbiddenSourceStreams,
Map<String, Set<String>> extraFilters) {

final QueryBuilder query = QueryStringUtils.isEmptyOrMatchAllQueryString(queryString)
? matchAllQuery()
: queryStringQuery(queryString).allowLeadingWildcard(allowLeadingWildcard);
Expand Down Expand Up @@ -285,11 +304,7 @@ private List<FieldSortBuilder> createSorting(Sorting sorting) {
.toList();
}

public List<Slice> aggregateSlices(String queryString, TimeRange timerange, Set<String> affectedIndices,
Set<String> eventStreams, String filterString, Set<String> forbiddenSourceStreams,
Map<String, Set<String>> extraFilters, Map<String, Object> meta, AggregationBuilder aggregationBuilder) {
final var filter = createQuery(queryString, timerange, eventStreams, filterString, forbiddenSourceStreams, extraFilters);

public List<Slice> aggregateSlices(QueryBuilder filter, Set<String> affectedIndices, Map<String, Object> meta, AggregationBuilder aggregationBuilder) {
final SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder()
.query(filter)
.aggregation(aggregationBuilder)
Expand All @@ -315,17 +330,21 @@ public List<Slice> aggregateSlices(String queryString, TimeRange timerange, Set<
public List<Slice> aggregateSlicesForColumn(String queryString, TimeRange timerange, Set<String> affectedIndices,
Set<String> eventStreams, String filterString, Set<String> forbiddenSourceStreams,
Map<String, Set<String>> extraFilters, String slicingColumn, Map<String, Object> meta, int maxBuckets) {
final var filter = checkAndAddRangeFilterFixWhenFieldIsMissing(createQuery(queryString, timerange, eventStreams, filterString, forbiddenSourceStreams, extraFilters), extraFilters);

final var builder = AggregationBuilders.terms(slicesAggregationName)
.field(slicingColumn)
.size(maxBuckets);

return aggregateSlices(queryString, timerange, affectedIndices, eventStreams, filterString, forbiddenSourceStreams, extraFilters, meta, builder);
return aggregateSlices(filter, affectedIndices, meta, builder);
}

@Override
public List<Slice> aggregateSlicesForRangeQuery(String queryString, TimeRange timerange, Set<String> affectedIndices,
Set<String> eventStreams, String filterString, Set<String> forbiddenSourceStreams,
Map<String, Set<String>> extraFilters, String slicingColumn, Map<String, Object> meta, List<NumberRange> ranges) {
QueryBuilder filter = createQuery(queryString, timerange, eventStreams, filterString, forbiddenSourceStreams, extraFilters);

final RangeAggregationBuilder builder = AggregationBuilders.range(slicesAggregationName).field(slicingColumn);
ranges.forEach(r -> {
final Double from = r.from();
Expand All @@ -338,7 +357,9 @@ public List<Slice> aggregateSlicesForRangeQuery(String queryString, TimeRange ti
builder.addUnboundedFrom(from);
}
});
return aggregateSlices(queryString, timerange, affectedIndices, eventStreams, filterString, forbiddenSourceStreams, extraFilters, meta, builder);

filter = ranges.stream().anyMatch(r -> r.from() == null || r.from() == 0d) ? addFieldIsMissing(filter) : filter;
return aggregateSlices(filter, affectedIndices, meta, builder);
}

@Override
Expand Down
43 changes: 31 additions & 12 deletions ...orage-opensearch2/src/main/java/org/graylog/storage/opensearch2/MoreSearchAdapterOS2.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,6 @@
import org.graylog.shaded.opensearch2.org.opensearch.search.aggregations.bucket.histogram.DateHistogramInterval;
import org.graylog.shaded.opensearch2.org.opensearch.search.aggregations.bucket.histogram.LongBounds;
import org.graylog.shaded.opensearch2.org.opensearch.search.aggregations.bucket.histogram.ParsedDateHistogram;
import org.graylog.shaded.opensearch2.org.opensearch.search.aggregations.bucket.range.ParsedRange;
import org.graylog.shaded.opensearch2.org.opensearch.search.aggregations.bucket.range.RangeAggregationBuilder;
import org.graylog.shaded.opensearch2.org.opensearch.search.aggregations.bucket.terms.ParsedTerms;
import org.graylog.shaded.opensearch2.org.opensearch.search.builder.SearchSourceBuilder;
Expand All @@ -67,18 +66,21 @@
import java.time.ZoneId;
import java.time.ZonedDateTime;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicReference;
import java.util.stream.Collectors;

import static com.google.common.base.Strings.isNullOrEmpty;
import static java.util.Objects.requireNonNull;
import static org.graylog.shaded.opensearch2.org.opensearch.index.query.QueryBuilders.boolQuery;
import static org.graylog.shaded.opensearch2.org.opensearch.index.query.QueryBuilders.existsQuery;
import static org.graylog.shaded.opensearch2.org.opensearch.index.query.QueryBuilders.matchAllQuery;
import static org.graylog.shaded.opensearch2.org.opensearch.index.query.QueryBuilders.queryStringQuery;
import static org.graylog.shaded.opensearch2.org.opensearch.index.query.QueryBuilders.termsQuery;
Expand Down Expand Up @@ -110,7 +112,7 @@ public MoreSearchAdapterOS2(OpenSearchClient client,
public MoreSearch.Result eventSearch(String queryString, TimeRange timerange, Set<String> affectedIndices,
Sorting sorting, int page, int perPage, Set<String> eventStreams,
String filterString, Set<String> forbiddenSourceStreams, Map<String, Set<String>> extraFilters) {
final var filter = createQuery(queryString, timerange, eventStreams, filterString, forbiddenSourceStreams, extraFilters);
final var filter = checkAndAddRangeFilterFixWhenFieldIsMissing(createQuery(queryString, timerange, eventStreams, filterString, forbiddenSourceStreams, extraFilters), extraFilters);

final SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder()
.query(filter)
Expand Down Expand Up @@ -151,7 +153,7 @@ public MoreSearch.Result eventSearch(String queryString, TimeRange timerange, Se
public MoreSearch.Histogram eventHistogram(String queryString, AbsoluteRange timerange, Set<String> affectedIndices,
Set<String> eventStreams, String filterString, Set<String> forbiddenSourceStreams,
ZoneId timeZone, Map<String, Set<String>> extraFilters) {
final var filter = createQuery(queryString, timerange, eventStreams, filterString, forbiddenSourceStreams, extraFilters);
final var filter = checkAndAddRangeFilterFixWhenFieldIsMissing(createQuery(queryString, timerange, eventStreams, filterString, forbiddenSourceStreams, extraFilters), extraFilters);

final SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder()
.query(filter)
Expand Down Expand Up @@ -211,7 +213,25 @@ public MoreSearch.Histogram eventHistogram(String queryString, AbsoluteRange tim
return new MoreSearch.Histogram(new MoreSearch.Histogram.EventsBuckets(events, alerts));
}

private QueryBuilder createQuery(String queryString, TimeRange timerange, Set<String> eventStreams, String filterString, Set<String> forbiddenSourceStreams, Map<String, Set<String>> extraFilters) {
private QueryBuilder addFieldIsMissing(QueryBuilder filter) {
return boolQuery().should(filter).should(boolQuery().mustNot(existsQuery(EventDto.FIELD_SCORES_NORMALIZED_RISK))).minimumShouldMatch("0");
}

private QueryBuilder checkAndAddRangeFilterFixWhenFieldIsMissing(QueryBuilder filter, Map<String, Set<String>> extraFilters) {
if(extraFilters.values().stream().flatMap(Collection::stream).anyMatch(MoreSearchAdapter::isLowerBoundZeroRangeFilter)) {
return addFieldIsMissing(filter);
} else {
return filter;
}
}

private QueryBuilder createQuery(String queryString,
TimeRange timerange,
Set<String> eventStreams,
String filterString,
Set<String> forbiddenSourceStreams,
Map<String, Set<String>> extraFilters) {

final QueryBuilder query = QueryStringUtils.isEmptyOrMatchAllQueryString(queryString)
? matchAllQuery()
: queryStringQuery(queryString).allowLeadingWildcard(allowLeadingWildcard);
Expand Down Expand Up @@ -246,6 +266,7 @@ private QueryBuilder createQuery(String queryString, TimeRange timerange, Set<St
// the event must not be in the search result.
filter.filter(boolQuery().mustNot(termsQuery(EventDto.FIELD_SOURCE_STREAMS, forbiddenSourceStreams)));
}

return filter;
}

Expand Down Expand Up @@ -284,12 +305,7 @@ private List<FieldSortBuilder> createSorting(Sorting sorting) {
.toList();
}

private List<Slice> aggregateSlices(String queryString, TimeRange timerange, Set<String> affectedIndices,
Set<String> eventStreams, String filterString, Set<String> forbiddenSourceStreams,
Map<String, Set<String>> extraFilters, Map<String, Object> meta, AggregationBuilder aggregation) {

final var filter = createQuery(queryString, timerange, eventStreams, filterString, forbiddenSourceStreams, extraFilters);

private List<Slice> aggregateSlices(QueryBuilder filter, Set<String> affectedIndices, Map<String, Object> meta, AggregationBuilder aggregation) {
final SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder()
.query(filter)
.aggregation(aggregation)
Expand Down Expand Up @@ -319,13 +335,15 @@ public List<Slice> aggregateSlicesForColumn(String queryString, TimeRange timera
.field(slicingColumn)
.size(maxBuckets);

return aggregateSlices(queryString, timerange, affectedIndices, eventStreams, filterString, forbiddenSourceStreams, extraFilters, meta, builder);
final var filter = checkAndAddRangeFilterFixWhenFieldIsMissing(createQuery(queryString, timerange, eventStreams, filterString, forbiddenSourceStreams, extraFilters), extraFilters);
return aggregateSlices(filter, affectedIndices, meta, builder);
}

@Override
public List<Slice> aggregateSlicesForRangeQuery(String queryString, TimeRange timerange, Set<String> affectedIndices,
Set<String> eventStreams, String filterString, Set<String> forbiddenSourceStreams,
Map<String, Set<String>> extraFilters, String slicingColumn, Map<String, Object> meta, List<NumberRange> ranges) {
QueryBuilder filter = createQuery(queryString, timerange, eventStreams, filterString, forbiddenSourceStreams, extraFilters);

final RangeAggregationBuilder builder = AggregationBuilders.range(slicesAggregationName).field(slicingColumn);
ranges.forEach(r -> {
Expand All @@ -340,7 +358,8 @@ public List<Slice> aggregateSlicesForRangeQuery(String queryString, TimeRange ti
}
});

return aggregateSlices(queryString, timerange, affectedIndices, eventStreams, filterString, forbiddenSourceStreams, extraFilters, meta, builder);
filter = ranges.stream().anyMatch(r -> r.from() == null || r.from() == 0d) ? addFieldIsMissing(filter) : filter;
return aggregateSlices(filter, affectedIndices, meta, builder);
}

@Override
Expand Down
Loading
Loading