diff --git a/processing/src/main/java/org/apache/druid/segment/serde/NestedCommonFormatColumnPartSerde.java b/processing/src/main/java/org/apache/druid/segment/serde/NestedCommonFormatColumnPartSerde.java index 16d41cfccc0f..f65a126b4c5c 100644 --- a/processing/src/main/java/org/apache/druid/segment/serde/NestedCommonFormatColumnPartSerde.java +++ b/processing/src/main/java/org/apache/druid/segment/serde/NestedCommonFormatColumnPartSerde.java @@ -482,8 +482,8 @@ public FormatSpec( @JsonProperty("longColumnCompression")@Nullable CompressionStrategy longColumnCompression, @JsonProperty("doubleColumnCompression")@Nullable CompressionStrategy doubleColumnCompression, @JsonProperty("bitmapEncoding") @Nullable BitmapSerdeFactory bitmapEncoding, - @JsonProperty("longFieldBitmapIndex")@Nullable BitmapIndexType longFieldBitmapIndex, - @JsonProperty("doubleFieldBitmapIndex")@Nullable BitmapIndexType doubleFieldBitmapIndex + @JsonProperty("longFieldBitmapIndexType")@Nullable BitmapIndexType longFieldBitmapIndex, + @JsonProperty("doubleFieldBitmapIndexType")@Nullable BitmapIndexType doubleFieldBitmapIndex ) { super( diff --git a/processing/src/test/java/org/apache/druid/segment/nested/NestedCommonFormatColumnFormatSpecTest.java b/processing/src/test/java/org/apache/druid/segment/nested/NestedCommonFormatColumnFormatSpecTest.java index d14bc35b4287..2b6b2eb15651 100644 --- a/processing/src/test/java/org/apache/druid/segment/nested/NestedCommonFormatColumnFormatSpecTest.java +++ b/processing/src/test/java/org/apache/druid/segment/nested/NestedCommonFormatColumnFormatSpecTest.java @@ -30,6 +30,7 @@ import org.apache.druid.segment.data.ConciseBitmapSerdeFactory; import org.apache.druid.segment.data.FrontCodedIndexed; import org.apache.druid.segment.data.RoaringBitmapSerdeFactory; +import org.apache.druid.segment.serde.NestedCommonFormatColumnPartSerde; import org.junit.Assert; import org.junit.jupiter.api.Test; @@ -39,19 +40,18 @@ public class NestedCommonFormatColumnFormatSpecTest public void testSerde() throws JsonProcessingException { NestedCommonFormatColumnFormatSpec spec = - NestedCommonFormatColumnFormatSpec.builder() - .setObjectFieldsDictionaryEncoding( - new StringEncodingStrategy.FrontCoded(4, FrontCodedIndexed.V1) - ) - .setObjectStorageCompression(CompressionStrategy.ZSTD) - .setStringDictionaryEncoding( - new StringEncodingStrategy.FrontCoded(16, FrontCodedIndexed.V1) - ) - .setLongFieldBitmapIndexType(BitmapIndexType.NullValueIndex.INSTANCE) - .setDoubleFieldBitmapIndexType(BitmapIndexType.NullValueIndex.INSTANCE) - .build(); - Assert.assertEquals( - spec, + NestedCommonFormatColumnFormatSpec + .builder() + .setBitmapEncoding(new ConciseBitmapSerdeFactory()) + .setObjectFieldsDictionaryEncoding(new StringEncodingStrategy.FrontCoded(4, FrontCodedIndexed.V1)) + .setObjectStorageCompression(CompressionStrategy.ZSTD) + .setStringDictionaryEncoding(new StringEncodingStrategy.FrontCoded(16, FrontCodedIndexed.V1)) + .setLongFieldBitmapIndexType(BitmapIndexType.NullValueIndex.INSTANCE) + .setDoubleFieldBitmapIndexType(BitmapIndexType.NullValueIndex.INSTANCE) + .build(); + // NestedCommonFormatColumnFormatSpec does not support serde for BitmapEncoding, value would be ignored + Assert.assertEquals( + NestedCommonFormatColumnFormatSpec.builder(spec).setBitmapEncoding(null).build(), TestHelper.JSON_MAPPER.readValue( TestHelper.JSON_MAPPER.writeValueAsString(spec), NestedCommonFormatColumnFormatSpec.class @@ -59,6 +59,25 @@ public void testSerde() throws JsonProcessingException ); } + @Test + public void testSerdeFormatSpec() throws JsonProcessingException + { + NestedCommonFormatColumnFormatSpec spec = + NestedCommonFormatColumnFormatSpec + .builder() + .setObjectFieldsDictionaryEncoding(new StringEncodingStrategy.FrontCoded(4, FrontCodedIndexed.V1)) + .setObjectStorageCompression(CompressionStrategy.ZSTD) + .setStringDictionaryEncoding(new StringEncodingStrategy.FrontCoded(16, FrontCodedIndexed.V1)) + .setLongFieldBitmapIndexType(BitmapIndexType.NullValueIndex.INSTANCE) + .setDoubleFieldBitmapIndexType(BitmapIndexType.DictionaryEncodedValueIndex.INSTANCE) + .build(); + NestedCommonFormatColumnPartSerde.FormatSpec formatSpec = TestHelper.JSON_MAPPER.readValue( + TestHelper.JSON_MAPPER.writeValueAsString(spec), + NestedCommonFormatColumnPartSerde.FormatSpec.class + ); + Assert.assertEquals(spec.toString(), formatSpec.toString()); + } + @Test public void testGetEffectiveSpecDefaults() { @@ -67,18 +86,9 @@ public void testGetEffectiveSpecDefaults() IndexSpec.getDefault().getEffectiveSpec() ); - Assert.assertEquals( - StringEncodingStrategy.UTF8_STRATEGY, - defaults.getObjectFieldsDictionaryEncoding() - ); - Assert.assertEquals( - ObjectStorageEncoding.SMILE, - defaults.getObjectStorageEncoding() - ); - Assert.assertEquals( - CompressionStrategy.LZ4, - defaults.getObjectStorageCompression() - ); + Assert.assertEquals(StringEncodingStrategy.UTF8_STRATEGY, defaults.getObjectFieldsDictionaryEncoding()); + Assert.assertEquals(ObjectStorageEncoding.SMILE, defaults.getObjectStorageEncoding()); + Assert.assertEquals(CompressionStrategy.LZ4, defaults.getObjectStorageCompression()); Assert.assertEquals( IndexSpec.getDefault().getEffectiveSpec().getDimensionCompression(), defaults.getDictionaryEncodedColumnCompression() @@ -95,6 +105,8 @@ public void testGetEffectiveSpecDefaults() IndexSpec.getDefault().getEffectiveSpec().getMetricCompression(), defaults.getDoubleColumnCompression() ); + Assert.assertEquals(BitmapIndexType.DictionaryEncodedValueIndex.INSTANCE, defaults.getLongFieldBitmapIndexType()); + Assert.assertEquals(BitmapIndexType.DictionaryEncodedValueIndex.INSTANCE, defaults.getLongFieldBitmapIndexType()); } @Test @@ -108,6 +120,8 @@ public void testEffectiveSpecIndexSpecOverrides() NestedCommonFormatColumnFormatSpec.builder() .setObjectFieldsDictionaryEncoding(frontcoded) .setObjectStorageEncoding(ObjectStorageEncoding.NONE) + .setLongFieldBitmapIndexType(BitmapIndexType.NullValueIndex.INSTANCE) + .setDoubleFieldBitmapIndexType(BitmapIndexType.NullValueIndex.INSTANCE) .build() ) .withMetricCompression(CompressionStrategy.LZF) @@ -115,18 +129,9 @@ public void testEffectiveSpecIndexSpecOverrides() .getEffectiveSpec() ); - Assert.assertEquals( - frontcoded, - defaults.getObjectFieldsDictionaryEncoding() - ); - Assert.assertEquals( - ObjectStorageEncoding.NONE, - defaults.getObjectStorageEncoding() - ); - Assert.assertEquals( - CompressionStrategy.LZ4, - defaults.getObjectStorageCompression() - ); + Assert.assertEquals(frontcoded, defaults.getObjectFieldsDictionaryEncoding()); + Assert.assertEquals(ObjectStorageEncoding.NONE, defaults.getObjectStorageEncoding()); + Assert.assertEquals(CompressionStrategy.LZ4, defaults.getObjectStorageCompression()); Assert.assertEquals( IndexSpec.getDefault().getEffectiveSpec().getDimensionCompression(), defaults.getDictionaryEncodedColumnCompression() @@ -135,30 +140,26 @@ public void testEffectiveSpecIndexSpecOverrides() IndexSpec.getDefault().getEffectiveSpec().getStringDictionaryEncoding(), defaults.getStringDictionaryEncoding() ); - Assert.assertEquals( - CompressionStrategy.LZF, - defaults.getLongColumnCompression() - ); - Assert.assertEquals( - CompressionStrategy.LZF, - defaults.getDoubleColumnCompression() - ); + Assert.assertEquals(CompressionStrategy.LZF, defaults.getLongColumnCompression()); + Assert.assertEquals(CompressionStrategy.LZF, defaults.getDoubleColumnCompression()); + Assert.assertEquals(BitmapIndexType.NullValueIndex.INSTANCE, defaults.getLongFieldBitmapIndexType()); + Assert.assertEquals(BitmapIndexType.NullValueIndex.INSTANCE, defaults.getDoubleFieldBitmapIndexType()); } @Test public void testGetEffectiveSpecMerge() { NestedCommonFormatColumnFormatSpec merged = NestedCommonFormatColumnFormatSpec.getEffectiveFormatSpec( - NestedCommonFormatColumnFormatSpec.builder() - .setObjectFieldsDictionaryEncoding( - new StringEncodingStrategy.FrontCoded(4, FrontCodedIndexed.V1) - ) - .setObjectStorageCompression(CompressionStrategy.ZSTD) - .setStringDictionaryEncoding( - new StringEncodingStrategy.FrontCoded(4, FrontCodedIndexed.V1) - ) - .setDoubleColumnCompression(CompressionStrategy.ZSTD) - .build(), + NestedCommonFormatColumnFormatSpec + .builder() + .setObjectFieldsDictionaryEncoding(new StringEncodingStrategy.FrontCoded(4, FrontCodedIndexed.V1)) + .setObjectStorageCompression(CompressionStrategy.ZSTD) + .setObjectStorageEncoding(ObjectStorageEncoding.NONE) + .setStringDictionaryEncoding(new StringEncodingStrategy.FrontCoded(4, FrontCodedIndexed.V1)) + .setDoubleColumnCompression(CompressionStrategy.ZSTD) + .setDoubleFieldBitmapIndexType(BitmapIndexType.NullValueIndex.INSTANCE) + .setLongFieldBitmapIndexType(BitmapIndexType.NullValueIndex.INSTANCE) + .build(), IndexSpec.getDefault().getEffectiveSpec() ); @@ -170,26 +171,19 @@ public void testGetEffectiveSpecMerge() new StringEncodingStrategy.FrontCoded(4, FrontCodedIndexed.V1), merged.getStringDictionaryEncoding() ); - Assert.assertEquals( - ObjectStorageEncoding.SMILE, - merged.getObjectStorageEncoding() - ); + Assert.assertEquals(ObjectStorageEncoding.NONE, merged.getObjectStorageEncoding()); Assert.assertEquals( IndexSpec.getDefault().getEffectiveSpec().getDimensionCompression(), merged.getDictionaryEncodedColumnCompression() ); - Assert.assertEquals( - CompressionStrategy.ZSTD, - merged.getObjectStorageCompression() - ); + Assert.assertEquals(CompressionStrategy.ZSTD, merged.getObjectStorageCompression()); Assert.assertEquals( IndexSpec.getDefault().getEffectiveSpec().getMetricCompression(), merged.getLongColumnCompression() ); - Assert.assertEquals( - CompressionStrategy.ZSTD, - merged.getDoubleColumnCompression() - ); + Assert.assertEquals(CompressionStrategy.ZSTD, merged.getDoubleColumnCompression()); + Assert.assertEquals(BitmapIndexType.NullValueIndex.INSTANCE, merged.getLongFieldBitmapIndexType()); + Assert.assertEquals(BitmapIndexType.NullValueIndex.INSTANCE, merged.getDoubleFieldBitmapIndexType()); } @Test