From db5cfedae426eb9ec29bbb11c6218906d8b4ddcd Mon Sep 17 00:00:00 2001 From: R Vadai Date: Sat, 19 Jun 2021 08:14:46 +0100 Subject: [PATCH 1/5] AVRO-2123: Implementation of Avro duration logical type --- .../java/org/apache/avro/Conversions.java | 108 ++++++++++++++++++ .../java/org/apache/avro/LogicalTypes.java | 57 +++++++++ 2 files changed, 165 insertions(+) diff --git a/lang/java/avro/src/main/java/org/apache/avro/Conversions.java b/lang/java/avro/src/main/java/org/apache/avro/Conversions.java index 1c28c9adb81..0cc9ad7ef7d 100644 --- a/lang/java/avro/src/main/java/org/apache/avro/Conversions.java +++ b/lang/java/avro/src/main/java/org/apache/avro/Conversions.java @@ -31,6 +31,7 @@ import java.util.Collection; import java.util.Map; import java.util.UUID; +import java.time.Duration; public class Conversions { @@ -146,6 +147,113 @@ private static BigDecimal validate(final LogicalTypes.Decimal decimal, BigDecima } } + public static class DurationConversion extends Conversion { + + private static final int MONTH_DAYS = 30; + private static final int GRANULARITY_BYTES = 4; + private static final String DOC_URL = "http://avro.apache.org/docs/current/spec.html#Duration"; + private static final String DOC_STR = "For more information on the duration logical type, please refer to " + DOC_URL; + + private static final byte[] EMPTY_BYTE_ARRAY = new byte[] { + Byte.MIN_VALUE, Byte.MIN_VALUE, Byte.MIN_VALUE, Byte.MIN_VALUE, + Byte.MIN_VALUE, Byte.MIN_VALUE, Byte.MIN_VALUE, Byte.MIN_VALUE, + Byte.MIN_VALUE, Byte.MIN_VALUE, Byte.MIN_VALUE, Byte.MIN_VALUE + }; + + private static byte[] toBytes(int value){ + return ByteBuffer.allocate(GRANULARITY_BYTES).order(ByteOrder.LITTLE_ENDIAN).putInt(value).array(); + } + + private static int toInt(byte[] bytes) { + return ByteBuffer.wrap(bytes).getInt(); + } + + private static byte[] buildByteArray(int months, int days, int milliSeconds) { + byte[] monthBytes = toBytes(months); + byte[] dayBytes = toBytes(days); + byte[] milliSecondBytes = toBytes(milliSeconds); + + return ByteBuffer + .allocate(monthBytes.length + dayBytes.length + milliSecondBytes.length) + .put(monthBytes) + .put(dayBytes) + .put(milliSecondBytes) + .array(); + } + + @Override + public Class getConvertedType() { + return Duration.class; + } + + @Override + public String getLogicalTypeName() { + return "duration"; + } + + @Override + public Duration fromFixed(GenericFixed value, Schema schema, LogicalType type) { + byte[] payload = value.bytes(); + int payloadLength = payload.length; + + if (payloadLength != GRANULARITY_BYTES * 3) { + throw new IllegalArgumentException("Duration must be stored on a 12-byte long value, but " + payloadLength + " bytes given. " + DOC_STR); + } + + if (Arrays.equals(payload, EMPTY_BYTE_ARRAY)) { + return Duration.ZERO; + } + + byte[] monthBytes = new byte[]{ payload[0], payload[1], payload[2], payload[3] }; + byte[] dayBytes = new byte[]{ payload[4], payload[5], payload[6], payload[7] }; + byte[] milliSecondBytes = new byte[]{ payload[8], payload[9], payload[10], payload[11] }; + + int months = toInt(monthBytes); + int days = toInt(dayBytes); + int milliSeconds = toInt(milliSecondBytes); + + return Duration.ofDays((long) months * MONTH_DAYS + days).plusMillis(milliSeconds); + } + + @Override + public GenericFixed toFixed(Duration value, Schema schema, LogicalType type) { + + if (value.isZero()) { + return new GenericData.Fixed(schema, EMPTY_BYTE_ARRAY); + } + + long totalDays = value.toDays(); + + int months = 0; + int days = 0; + int milliSeconds = 0; + + try { + months = (int) (totalDays / MONTH_DAYS); + } catch (Throwable e) { + throw new IllegalArgumentException("The months part of a duration must fit a 4-byte int, longer duration given. " + DOC_STR, e); + } + + try { + days = (int) (totalDays % MONTH_DAYS); + } catch (Throwable e) { + throw new IllegalArgumentException("The days part of a duration must fit a 4-byte int, longer duration given. " + DOC_STR, e); + } + + try { + milliSeconds = (int) value.toMillis(); + } catch (Throwable e) { + throw new IllegalArgumentException("The milliseconds part of a duration must fit a 4-byte int, longer duration given. " + DOC_STR, e); + } + + // TODO adjust millis with nano + + byte[] result = buildByteArray(months, days, milliSeconds); + + return new GenericData.Fixed(schema, result); + } + } + /** * Convert a underlying representation of a logical type (such as a ByteBuffer) * to a higher level object (such as a BigDecimal). diff --git a/lang/java/avro/src/main/java/org/apache/avro/LogicalTypes.java b/lang/java/avro/src/main/java/org/apache/avro/LogicalTypes.java index ef9c69dc511..c95ceee0caa 100644 --- a/lang/java/avro/src/main/java/org/apache/avro/LogicalTypes.java +++ b/lang/java/avro/src/main/java/org/apache/avro/LogicalTypes.java @@ -102,6 +102,9 @@ private static LogicalType fromSchemaImpl(Schema schema, boolean throwErrors) { case LOCAL_TIMESTAMP_MILLIS: logicalType = LOCAL_TIMESTAMP_MILLIS_TYPE; break; + case DURATION: + logicalType = DURATION_TYPE; + break; default: final LogicalTypeFactory typeFactory = REGISTERED_TYPES.get(typeName); logicalType = (typeFactory == null) ? null : typeFactory.fromSchema(schema); @@ -134,6 +137,7 @@ private static LogicalType fromSchemaImpl(Schema schema, boolean throwErrors) { private static final String TIMESTAMP_MICROS = "timestamp-micros"; private static final String LOCAL_TIMESTAMP_MILLIS = "local-timestamp-millis"; private static final String LOCAL_TIMESTAMP_MICROS = "local-timestamp-micros"; + private static final String DURATION = "duration"; /** Create a Decimal LogicalType with the given precision and scale 0 */ public static Decimal decimal(int precision) { @@ -193,6 +197,13 @@ public static LocalTimestampMicros localTimestampMicros() { return LOCAL_TIMESTAMP_MICROS_TYPE; } + private static final Duration DURATION_TYPE = new Duration(); + + public static Duration duration() { + return DURATION_TYPE; + } + + /** Decimal represents arbitrary-precision fixed-scale decimal numbers */ public static class Decimal extends LogicalType { private static final String PRECISION_PROP = "precision"; @@ -410,4 +421,50 @@ public void validate(Schema schema) { } } + + /** TimeMicros represents a time in microseconds without a date */ + public static class TimeMicros extends LogicalType { + private TimeMicros() { + super(TIME_MICROS); + } + + @Override + public void validate(Schema schema) { + super.validate(schema); + if (schema.getType() != Schema.Type.LONG) { + throw new IllegalArgumentException("Time (micros) can only be used with an underlying long type"); + } + } + } + + /** TimestampMillis represents a date and time in milliseconds */ + public static class TimestampMillis extends LogicalType { + private TimestampMillis() { + super(TIMESTAMP_MILLIS); + } + + @Override + public void validate(Schema schema) { + super.validate(schema); + if (schema.getType() != Schema.Type.LONG) { + throw new IllegalArgumentException("Timestamp (millis) can only be used with an underlying long type"); + } + } + } + + /** Duration represents an amount of time defined by a number of months, days and milliseconds */ + public static class Duration extends LogicalType { + private Duration() { + super(DURATION); + } + + @Override + public void validate(Schema schema) { + super.validate(schema); + if (schema.getType() != Schema.Type.FIXED || schema.getSize() != 12) { + throw new IllegalArgumentException("Duration can only be used with an underlying fixed type of size 12"); + } + } + } + } From cf509fc5bb5b8bf525ecb3a5b0886e15512d6931 Mon Sep 17 00:00:00 2001 From: R Vadai Date: Sat, 19 Jun 2021 08:34:34 +0100 Subject: [PATCH 2/5] AVRO-2123: Added duration logical type tests --- .../java/org/apache/avro/Conversions.java | 1 + .../java/org/apache/avro/LogicalTypes.java | 33 +-- .../apache/avro/TestDurationConversion.java | 209 ++++++++++++++++++ .../java/org/apache/avro/TestLogicalType.java | 28 +++ 4 files changed, 239 insertions(+), 32 deletions(-) create mode 100644 lang/java/avro/src/test/java/org/apache/avro/TestDurationConversion.java diff --git a/lang/java/avro/src/main/java/org/apache/avro/Conversions.java b/lang/java/avro/src/main/java/org/apache/avro/Conversions.java index 0cc9ad7ef7d..b2b92c3add7 100644 --- a/lang/java/avro/src/main/java/org/apache/avro/Conversions.java +++ b/lang/java/avro/src/main/java/org/apache/avro/Conversions.java @@ -27,6 +27,7 @@ import java.math.BigDecimal; import java.math.BigInteger; import java.nio.ByteBuffer; +import java.nio.ByteOrder; import java.util.Arrays; import java.util.Collection; import java.util.Map; diff --git a/lang/java/avro/src/main/java/org/apache/avro/LogicalTypes.java b/lang/java/avro/src/main/java/org/apache/avro/LogicalTypes.java index c95ceee0caa..35fddf7ede2 100644 --- a/lang/java/avro/src/main/java/org/apache/avro/LogicalTypes.java +++ b/lang/java/avro/src/main/java/org/apache/avro/LogicalTypes.java @@ -421,37 +421,6 @@ public void validate(Schema schema) { } } - - /** TimeMicros represents a time in microseconds without a date */ - public static class TimeMicros extends LogicalType { - private TimeMicros() { - super(TIME_MICROS); - } - - @Override - public void validate(Schema schema) { - super.validate(schema); - if (schema.getType() != Schema.Type.LONG) { - throw new IllegalArgumentException("Time (micros) can only be used with an underlying long type"); - } - } - } - - /** TimestampMillis represents a date and time in milliseconds */ - public static class TimestampMillis extends LogicalType { - private TimestampMillis() { - super(TIMESTAMP_MILLIS); - } - - @Override - public void validate(Schema schema) { - super.validate(schema); - if (schema.getType() != Schema.Type.LONG) { - throw new IllegalArgumentException("Timestamp (millis) can only be used with an underlying long type"); - } - } - } - /** Duration represents an amount of time defined by a number of months, days and milliseconds */ public static class Duration extends LogicalType { private Duration() { @@ -461,7 +430,7 @@ private Duration() { @Override public void validate(Schema schema) { super.validate(schema); - if (schema.getType() != Schema.Type.FIXED || schema.getSize() != 12) { + if (schema.getType() != Schema.Type.FIXED || schema.getFixedSize() != 12) { throw new IllegalArgumentException("Duration can only be used with an underlying fixed type of size 12"); } } diff --git a/lang/java/avro/src/test/java/org/apache/avro/TestDurationConversion.java b/lang/java/avro/src/test/java/org/apache/avro/TestDurationConversion.java new file mode 100644 index 00000000000..3078cbd1ab3 --- /dev/null +++ b/lang/java/avro/src/test/java/org/apache/avro/TestDurationConversion.java @@ -0,0 +1,209 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.avro; + +import org.apache.avro.generic.GenericData; +import org.apache.avro.generic.GenericFixed; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +import java.math.BigDecimal; +import java.nio.ByteBuffer; + +import static java.math.RoundingMode.HALF_EVEN; +import static org.junit.Assert.*; + +public class TestDurationConversion { + + private static final Conversion CONVERSION = new Conversions.DecimalConversion(); + + @Rule + public ExpectedException expectedException = ExpectedException.none(); + + private Schema smallerSchema; + private LogicalType smallerLogicalType; + private Schema largerSchema; + private LogicalType largerLogicalType; + + @Before + public void setup() { + smallerSchema = Schema.createFixed("smallFixed", null, null, 3); + smallerSchema.addProp("logicalType", "decimal"); + smallerSchema.addProp("precision", 5); + smallerSchema.addProp("scale", 2); + smallerLogicalType = LogicalTypes.fromSchema(smallerSchema); + + largerSchema = Schema.createFixed("largeFixed", null, null, 12); + largerSchema.addProp("logicalType", "decimal"); + largerSchema.addProp("precision", 28); + largerSchema.addProp("scale", 15); + largerLogicalType = LogicalTypes.fromSchema(largerSchema); + } + + @Test + public void testToFromBytes() { + final BigDecimal value = BigDecimal.valueOf(10.99).setScale(15, HALF_EVEN); + final ByteBuffer byteBuffer = CONVERSION.toBytes(value, largerSchema, largerLogicalType); + final BigDecimal result = CONVERSION.fromBytes(byteBuffer, largerSchema, largerLogicalType); + assertEquals(value, result); + } + + @Test + public void testToFromBytesMaxPrecision() { + final BigDecimal value = new BigDecimal("4567335489766.99834").setScale(15, HALF_EVEN); + final ByteBuffer byteBuffer = CONVERSION.toBytes(value, largerSchema, largerLogicalType); + final BigDecimal result = CONVERSION.fromBytes(byteBuffer, largerSchema, largerLogicalType); + assertEquals(value, result); + } + + @Test + public void testToBytesPrecisionError() { + final BigDecimal value = new BigDecimal("1.07046455859736525E+18").setScale(15, HALF_EVEN); + expectedException.expect(AvroTypeException.class); + expectedException.expectMessage("Cannot encode decimal with precision 34 as max precision 28"); + CONVERSION.toBytes(value, largerSchema, largerLogicalType); + } + + @Test + public void testToBytesFixedSmallerScale() { + final BigDecimal value = new BigDecimal("99892.1234").setScale(10, HALF_EVEN); + final ByteBuffer byteBuffer = CONVERSION.toBytes(value, largerSchema, largerLogicalType); + final BigDecimal result = CONVERSION.fromBytes(byteBuffer, largerSchema, largerLogicalType); + assertEquals(new BigDecimal("99892.123400000000000"), result); + } + + @Test + public void testToBytesScaleError() { + final BigDecimal value = new BigDecimal("4567335489766.989989998435899453").setScale(16, HALF_EVEN); + expectedException.expect(AvroTypeException.class); + expectedException.expectMessage("Cannot encode decimal with scale 16 as scale 15 without rounding"); + CONVERSION.toBytes(value, largerSchema, largerLogicalType); + } + + @Test + public void testToFromFixed() { + final BigDecimal value = new BigDecimal("3").setScale(15, HALF_EVEN); + final GenericFixed fixed = CONVERSION.toFixed(value, largerSchema, largerLogicalType); + final BigDecimal result = CONVERSION.fromFixed(fixed, largerSchema, largerLogicalType); + assertEquals(value, result); + } + + @Test + public void testToFromFixedMaxPrecision() { + final BigDecimal value = new BigDecimal("4567335489766.99834").setScale(15, HALF_EVEN); + final GenericFixed fixed = CONVERSION.toFixed(value, largerSchema, largerLogicalType); + final BigDecimal result = CONVERSION.fromFixed(fixed, largerSchema, largerLogicalType); + assertEquals(value, result); + } + + @Test + public void testToFixedPrecisionError() { + final BigDecimal value = new BigDecimal("1.07046455859736525E+18").setScale(15, HALF_EVEN); + expectedException.expect(AvroTypeException.class); + expectedException.expectMessage("Cannot encode decimal with precision 34 as max precision 28"); + CONVERSION.toFixed(value, largerSchema, largerLogicalType); + } + + @Test + public void testToFromFixedSmallerScale() { + final BigDecimal value = new BigDecimal("99892.1234").setScale(10, HALF_EVEN); + final GenericFixed fixed = CONVERSION.toFixed(value, largerSchema, largerLogicalType); + final BigDecimal result = CONVERSION.fromFixed(fixed, largerSchema, largerLogicalType); + assertEquals(new BigDecimal("99892.123400000000000"), result); + } + + @Test + public void testToFixedScaleError() { + final BigDecimal value = new BigDecimal("4567335489766.3453453453453453453453").setScale(16, HALF_EVEN); + expectedException.expect(AvroTypeException.class); + expectedException.expectMessage("Cannot encode decimal with scale 16 as scale 15 without rounding"); + CONVERSION.toFixed(value, largerSchema, largerLogicalType); + } + + @Test + public void testToFromFixedMatchScaleAndPrecision() { + final BigDecimal value = new BigDecimal("123.45"); + final GenericFixed fixed = CONVERSION.toFixed(value, smallerSchema, smallerLogicalType); + final BigDecimal result = CONVERSION.fromFixed(fixed, smallerSchema, smallerLogicalType); + assertEquals(value, result); + } + + @Test + public void testToFromFixedRepresentedInLogicalTypeAllowRoundUnneccesary() { + final BigDecimal value = new BigDecimal("123.4500"); + final GenericFixed fixed = CONVERSION.toFixed(value, smallerSchema, smallerLogicalType); + final BigDecimal result = CONVERSION.fromFixed(fixed, smallerSchema, smallerLogicalType); + assertEquals(new BigDecimal("123.45"), result); + } + + @Test + public void testToFromFixedPrecisionErrorAfterAdjustingScale() { + final BigDecimal value = new BigDecimal("1234.560"); + expectedException.expect(AvroTypeException.class); + expectedException.expectMessage( + "Cannot encode decimal with precision 6 as max precision 5. This is after safely adjusting scale from 3 to required 2"); + CONVERSION.toFixed(value, smallerSchema, smallerLogicalType); + } + + @Test + public void testToFixedRepresentedInLogicalTypeErrorIfRoundingRequired() { + final BigDecimal value = new BigDecimal("123.456"); + expectedException.expect(AvroTypeException.class); + expectedException.expectMessage("Cannot encode decimal with scale 3 as scale 2 without rounding"); + CONVERSION.toFixed(value, smallerSchema, smallerLogicalType); + } + + @Test + public void testImportanceOfEnsuringCorrectScaleWhenConvertingFixed() { + LogicalTypes.Decimal decimal = (LogicalTypes.Decimal) smallerLogicalType; + + final BigDecimal bigDecimal = new BigDecimal("1234.5"); + assertEquals(decimal.getPrecision(), bigDecimal.precision()); + assertTrue(decimal.getScale() >= bigDecimal.scale()); + + final byte[] bytes = bigDecimal.unscaledValue().toByteArray(); + + final BigDecimal fromFixed = CONVERSION.fromFixed(new GenericData.Fixed(smallerSchema, bytes), smallerSchema, + decimal); + + assertNotEquals(0, bigDecimal.compareTo(fromFixed)); + assertNotEquals(bigDecimal, fromFixed); + + assertEquals(new BigDecimal("123.45"), fromFixed); + } + + @Test + public void testImportanceOfEnsuringCorrectScaleWhenConvertingBytes() { + LogicalTypes.Decimal decimal = (LogicalTypes.Decimal) smallerLogicalType; + + final BigDecimal bigDecimal = new BigDecimal("1234.5"); + assertEquals(decimal.getPrecision(), bigDecimal.precision()); + assertTrue(decimal.getScale() >= bigDecimal.scale()); + + final byte[] bytes = bigDecimal.unscaledValue().toByteArray(); + + final BigDecimal fromBytes = CONVERSION.fromBytes(ByteBuffer.wrap(bytes), smallerSchema, decimal); + + assertNotEquals(0, bigDecimal.compareTo(fromBytes)); + assertNotEquals(bigDecimal, fromBytes); + + assertEquals(new BigDecimal("123.45"), fromBytes); + } +} diff --git a/lang/java/avro/src/test/java/org/apache/avro/TestLogicalType.java b/lang/java/avro/src/test/java/org/apache/avro/TestLogicalType.java index d830db68775..22ff2045dde 100644 --- a/lang/java/avro/src/test/java/org/apache/avro/TestLogicalType.java +++ b/lang/java/avro/src/test/java/org/apache/avro/TestLogicalType.java @@ -226,6 +226,34 @@ public void testLogicalTypeInSchemaEquals() { assertEqualsFalse("Different logical type", schema1, schema3); } + @Test + public void testDurationFromSchema() { + Schema schema = Schema.createFixed("aFixed", null, null, 12); + schema.addProp("logicalType", "duration"); + LogicalType logicalType = LogicalTypes.fromSchemaIgnoreInvalid(schema); + + Assert.assertTrue("Should be a Duration", logicalType instanceof LogicalTypes.Duration); + } + + @Test + public void testDurationValidation() { + assertThrows("Should reject too short type", IllegalArgumentException.class, + "Duration can only be used with an underlying fixed type of size 12", () -> { + LogicalTypes.duration().addToSchema( + Schema.createFixed("aFixed", null, null, 11) + ); + return null; + }); + + assertThrows("Should reject too long type", IllegalArgumentException.class, + "Duration can only be used with an underlying fixed type of size 12", () -> { + LogicalTypes.duration().addToSchema( + Schema.createFixed("anotherFixed", null, null, 13) + ); + return null; + }); + } + public static void assertEqualsTrue(String message, Object o1, Object o2) { Assert.assertTrue("Should be equal (forward): " + message, o1.equals(o2)); Assert.assertTrue("Should be equal (reverse): " + message, o2.equals(o1)); From 96520d89918720fecfc8718b426869d2775b99e2 Mon Sep 17 00:00:00 2001 From: R Vadai Date: Sat, 19 Jun 2021 11:44:10 +0100 Subject: [PATCH 3/5] AVRO-2123: Working duration implementation tests --- .../java/org/apache/avro/Conversions.java | 11 +- .../apache/avro/TestDurationConversion.java | 212 +++++++----------- 2 files changed, 91 insertions(+), 132 deletions(-) diff --git a/lang/java/avro/src/main/java/org/apache/avro/Conversions.java b/lang/java/avro/src/main/java/org/apache/avro/Conversions.java index b2b92c3add7..0c744f44393 100644 --- a/lang/java/avro/src/main/java/org/apache/avro/Conversions.java +++ b/lang/java/avro/src/main/java/org/apache/avro/Conversions.java @@ -28,6 +28,7 @@ import java.math.BigInteger; import java.nio.ByteBuffer; import java.nio.ByteOrder; +import java.time.temporal.ChronoUnit; import java.util.Arrays; import java.util.Collection; import java.util.Map; @@ -156,9 +157,9 @@ public static class DurationConversion extends Conversion { private static final String DOC_STR = "For more information on the duration logical type, please refer to " + DOC_URL; private static final byte[] EMPTY_BYTE_ARRAY = new byte[] { - Byte.MIN_VALUE, Byte.MIN_VALUE, Byte.MIN_VALUE, Byte.MIN_VALUE, - Byte.MIN_VALUE, Byte.MIN_VALUE, Byte.MIN_VALUE, Byte.MIN_VALUE, - Byte.MIN_VALUE, Byte.MIN_VALUE, Byte.MIN_VALUE, Byte.MIN_VALUE + 0x0, 0x0, 0x0, 0x0, + 0x0, 0x0, 0x0, 0x0, + 0x0, 0x0, 0x0, 0x0 }; private static byte[] toBytes(int value){ @@ -166,7 +167,7 @@ private static byte[] toBytes(int value){ } private static int toInt(byte[] bytes) { - return ByteBuffer.wrap(bytes).getInt(); + return ByteBuffer.wrap(bytes).order(ByteOrder.LITTLE_ENDIAN).getInt(); } private static byte[] buildByteArray(int months, int days, int milliSeconds) { @@ -242,7 +243,7 @@ public GenericFixed toFixed(Duration value, Schema schema, LogicalType type) { } try { - milliSeconds = (int) value.toMillis(); + milliSeconds = (int) value.minus(months * MONTH_DAYS, ChronoUnit.DAYS).minus(days, ChronoUnit.DAYS).toMillis(); } catch (Throwable e) { throw new IllegalArgumentException("The milliseconds part of a duration must fit a 4-byte int, longer duration given. " + DOC_STR, e); } diff --git a/lang/java/avro/src/test/java/org/apache/avro/TestDurationConversion.java b/lang/java/avro/src/test/java/org/apache/avro/TestDurationConversion.java index 3078cbd1ab3..5ebad43ab3d 100644 --- a/lang/java/avro/src/test/java/org/apache/avro/TestDurationConversion.java +++ b/lang/java/avro/src/test/java/org/apache/avro/TestDurationConversion.java @@ -26,184 +26,142 @@ import java.math.BigDecimal; import java.nio.ByteBuffer; +import java.time.Duration; +import java.util.Arrays; import static java.math.RoundingMode.HALF_EVEN; import static org.junit.Assert.*; public class TestDurationConversion { - private static final Conversion CONVERSION = new Conversions.DecimalConversion(); + private static final Conversion CONVERSION = new Conversions.DurationConversion(); @Rule public ExpectedException expectedException = ExpectedException.none(); - private Schema smallerSchema; - private LogicalType smallerLogicalType; - private Schema largerSchema; - private LogicalType largerLogicalType; + private Schema schema; + private LogicalType logicalType; @Before public void setup() { - smallerSchema = Schema.createFixed("smallFixed", null, null, 3); - smallerSchema.addProp("logicalType", "decimal"); - smallerSchema.addProp("precision", 5); - smallerSchema.addProp("scale", 2); - smallerLogicalType = LogicalTypes.fromSchema(smallerSchema); - - largerSchema = Schema.createFixed("largeFixed", null, null, 12); - largerSchema.addProp("logicalType", "decimal"); - largerSchema.addProp("precision", 28); - largerSchema.addProp("scale", 15); - largerLogicalType = LogicalTypes.fromSchema(largerSchema); + schema = Schema.createFixed("aFixed", null, null, 12); + schema.addProp("logicalType", "duration"); + logicalType = LogicalTypes.fromSchema(schema); } @Test - public void testToFromBytes() { - final BigDecimal value = BigDecimal.valueOf(10.99).setScale(15, HALF_EVEN); - final ByteBuffer byteBuffer = CONVERSION.toBytes(value, largerSchema, largerLogicalType); - final BigDecimal result = CONVERSION.fromBytes(byteBuffer, largerSchema, largerLogicalType); + public void testToFromFixed() { + final Duration value = Duration.ofDays(10).plusMillis(100); + final GenericFixed fixed = CONVERSION.toFixed(value, schema, logicalType); + final Duration result = CONVERSION.fromFixed(fixed, schema, logicalType); assertEquals(value, result); } @Test - public void testToFromBytesMaxPrecision() { - final BigDecimal value = new BigDecimal("4567335489766.99834").setScale(15, HALF_EVEN); - final ByteBuffer byteBuffer = CONVERSION.toBytes(value, largerSchema, largerLogicalType); - final BigDecimal result = CONVERSION.fromBytes(byteBuffer, largerSchema, largerLogicalType); - assertEquals(value, result); - } + public void testConvertingMillisecondsFromBytes() { + LogicalTypes.Duration duration = (LogicalTypes.Duration) logicalType; - @Test - public void testToBytesPrecisionError() { - final BigDecimal value = new BigDecimal("1.07046455859736525E+18").setScale(15, HALF_EVEN); - expectedException.expect(AvroTypeException.class); - expectedException.expectMessage("Cannot encode decimal with precision 34 as max precision 28"); - CONVERSION.toBytes(value, largerSchema, largerLogicalType); - } + final Duration durationValue = Duration.ofMillis(200); - @Test - public void testToBytesFixedSmallerScale() { - final BigDecimal value = new BigDecimal("99892.1234").setScale(10, HALF_EVEN); - final ByteBuffer byteBuffer = CONVERSION.toBytes(value, largerSchema, largerLogicalType); - final BigDecimal result = CONVERSION.fromBytes(byteBuffer, largerSchema, largerLogicalType); - assertEquals(new BigDecimal("99892.123400000000000"), result); - } + final byte[] bytes = new byte[]{ + 0x0, 0x0, 0x0, 0x0, + 0x0, 0x0, 0x0, 0x0, + -0x38, 0x0, 0x0, 0x0 + }; - @Test - public void testToBytesScaleError() { - final BigDecimal value = new BigDecimal("4567335489766.989989998435899453").setScale(16, HALF_EVEN); - expectedException.expect(AvroTypeException.class); - expectedException.expectMessage("Cannot encode decimal with scale 16 as scale 15 without rounding"); - CONVERSION.toBytes(value, largerSchema, largerLogicalType); - } + final GenericFixed dd = new GenericData.Fixed(schema, bytes); - @Test - public void testToFromFixed() { - final BigDecimal value = new BigDecimal("3").setScale(15, HALF_EVEN); - final GenericFixed fixed = CONVERSION.toFixed(value, largerSchema, largerLogicalType); - final BigDecimal result = CONVERSION.fromFixed(fixed, largerSchema, largerLogicalType); - assertEquals(value, result); - } + final Duration fromBytes = CONVERSION.fromFixed(dd, schema, duration); - @Test - public void testToFromFixedMaxPrecision() { - final BigDecimal value = new BigDecimal("4567335489766.99834").setScale(15, HALF_EVEN); - final GenericFixed fixed = CONVERSION.toFixed(value, largerSchema, largerLogicalType); - final BigDecimal result = CONVERSION.fromFixed(fixed, largerSchema, largerLogicalType); - assertEquals(value, result); + assertEquals(durationValue, fromBytes); } @Test - public void testToFixedPrecisionError() { - final BigDecimal value = new BigDecimal("1.07046455859736525E+18").setScale(15, HALF_EVEN); - expectedException.expect(AvroTypeException.class); - expectedException.expectMessage("Cannot encode decimal with precision 34 as max precision 28"); - CONVERSION.toFixed(value, largerSchema, largerLogicalType); - } + public void testConvertingDaysFromBytes() { + LogicalTypes.Duration duration = (LogicalTypes.Duration) logicalType; - @Test - public void testToFromFixedSmallerScale() { - final BigDecimal value = new BigDecimal("99892.1234").setScale(10, HALF_EVEN); - final GenericFixed fixed = CONVERSION.toFixed(value, largerSchema, largerLogicalType); - final BigDecimal result = CONVERSION.fromFixed(fixed, largerSchema, largerLogicalType); - assertEquals(new BigDecimal("99892.123400000000000"), result); - } + final Duration durationValue = Duration.ofDays(29); - @Test - public void testToFixedScaleError() { - final BigDecimal value = new BigDecimal("4567335489766.3453453453453453453453").setScale(16, HALF_EVEN); - expectedException.expect(AvroTypeException.class); - expectedException.expectMessage("Cannot encode decimal with scale 16 as scale 15 without rounding"); - CONVERSION.toFixed(value, largerSchema, largerLogicalType); - } + final byte[] bytes = new byte[]{ + 0x0, 0x0, 0x0, 0x0, + 0x1D, 0x0, 0x0, 0x0, + 0x0, 0x0, 0x0, 0x0 + }; - @Test - public void testToFromFixedMatchScaleAndPrecision() { - final BigDecimal value = new BigDecimal("123.45"); - final GenericFixed fixed = CONVERSION.toFixed(value, smallerSchema, smallerLogicalType); - final BigDecimal result = CONVERSION.fromFixed(fixed, smallerSchema, smallerLogicalType); - assertEquals(value, result); - } + final GenericFixed dd = new GenericData.Fixed(schema, bytes); - @Test - public void testToFromFixedRepresentedInLogicalTypeAllowRoundUnneccesary() { - final BigDecimal value = new BigDecimal("123.4500"); - final GenericFixed fixed = CONVERSION.toFixed(value, smallerSchema, smallerLogicalType); - final BigDecimal result = CONVERSION.fromFixed(fixed, smallerSchema, smallerLogicalType); - assertEquals(new BigDecimal("123.45"), result); + final Duration fromBytes = CONVERSION.fromFixed(dd, schema, duration); + + assertEquals(durationValue, fromBytes); } @Test - public void testToFromFixedPrecisionErrorAfterAdjustingScale() { - final BigDecimal value = new BigDecimal("1234.560"); - expectedException.expect(AvroTypeException.class); - expectedException.expectMessage( - "Cannot encode decimal with precision 6 as max precision 5. This is after safely adjusting scale from 3 to required 2"); - CONVERSION.toFixed(value, smallerSchema, smallerLogicalType); + public void testConvertingMonthsFromBytes() { + LogicalTypes.Duration duration = (LogicalTypes.Duration) logicalType; + + final Duration durationValue = Duration.ofDays(60); + + final byte[] bytes = new byte[]{ + 0x02, 0x0, 0x0, 0x0, + 0x0, 0x0, 0x0, 0x0, + 0x0, 0x0, 0x0, 0x0 + }; + + final GenericFixed dd = new GenericData.Fixed(schema, bytes); + + final Duration fromBytes = CONVERSION.fromFixed(dd, schema, duration); + + assertEquals(durationValue, fromBytes); } @Test - public void testToFixedRepresentedInLogicalTypeErrorIfRoundingRequired() { - final BigDecimal value = new BigDecimal("123.456"); - expectedException.expect(AvroTypeException.class); - expectedException.expectMessage("Cannot encode decimal with scale 3 as scale 2 without rounding"); - CONVERSION.toFixed(value, smallerSchema, smallerLogicalType); + public void testConvertingMillisecondsToBytes() { + LogicalTypes.Duration duration = (LogicalTypes.Duration) logicalType; + + final Duration durationValue = Duration.ofMillis(200); + + final byte[] bytes = new byte[]{ + 0x0, 0x0, 0x0, 0x0, + 0x0, 0x0, 0x0, 0x0, + -0x38, 0x0, 0x0, 0x0 + }; + + final GenericFixed fixed = CONVERSION.toFixed(durationValue, schema, duration); + + assertArrayEquals(bytes, fixed.bytes()); } @Test - public void testImportanceOfEnsuringCorrectScaleWhenConvertingFixed() { - LogicalTypes.Decimal decimal = (LogicalTypes.Decimal) smallerLogicalType; + public void testConvertingDaysToBytes() { + LogicalTypes.Duration duration = (LogicalTypes.Duration) logicalType; - final BigDecimal bigDecimal = new BigDecimal("1234.5"); - assertEquals(decimal.getPrecision(), bigDecimal.precision()); - assertTrue(decimal.getScale() >= bigDecimal.scale()); + final Duration durationValue = Duration.ofDays(29); - final byte[] bytes = bigDecimal.unscaledValue().toByteArray(); + final byte[] bytes = new byte[]{ + 0x0, 0x0, 0x0, 0x0, + 0x1D, 0x0, 0x0, 0x0, + 0x0, 0x0, 0x0, 0x0 + }; - final BigDecimal fromFixed = CONVERSION.fromFixed(new GenericData.Fixed(smallerSchema, bytes), smallerSchema, - decimal); + final GenericFixed fixed = CONVERSION.toFixed(durationValue, schema, duration); - assertNotEquals(0, bigDecimal.compareTo(fromFixed)); - assertNotEquals(bigDecimal, fromFixed); - - assertEquals(new BigDecimal("123.45"), fromFixed); + assertArrayEquals(bytes, fixed.bytes()); } @Test - public void testImportanceOfEnsuringCorrectScaleWhenConvertingBytes() { - LogicalTypes.Decimal decimal = (LogicalTypes.Decimal) smallerLogicalType; - - final BigDecimal bigDecimal = new BigDecimal("1234.5"); - assertEquals(decimal.getPrecision(), bigDecimal.precision()); - assertTrue(decimal.getScale() >= bigDecimal.scale()); + public void testConvertingWeeksToBytes() { + LogicalTypes.Duration duration = (LogicalTypes.Duration) logicalType; - final byte[] bytes = bigDecimal.unscaledValue().toByteArray(); + final Duration durationValue = Duration.ofDays(60); - final BigDecimal fromBytes = CONVERSION.fromBytes(ByteBuffer.wrap(bytes), smallerSchema, decimal); + final byte[] bytes = new byte[]{ + 0x02, 0x0, 0x0, 0x0, + 0x0, 0x0, 0x0, 0x0, + 0x0, 0x0, 0x0, 0x0 + }; - assertNotEquals(0, bigDecimal.compareTo(fromBytes)); - assertNotEquals(bigDecimal, fromBytes); + final GenericFixed fixed = CONVERSION.toFixed(durationValue, schema, duration); - assertEquals(new BigDecimal("123.45"), fromBytes); + assertArrayEquals(bytes, fixed.bytes()); } } From fb60b268eb93cec636cae054cc03c81470e28b7e Mon Sep 17 00:00:00 2001 From: R Vadai Date: Sat, 19 Jun 2021 12:00:05 +0100 Subject: [PATCH 4/5] AVRO-2123: Adjusting for nanoseconds --- .../java/org/apache/avro/Conversions.java | 18 +++++++--------- .../apache/avro/TestDurationConversion.java | 21 +++++++++++++++++++ 2 files changed, 29 insertions(+), 10 deletions(-) diff --git a/lang/java/avro/src/main/java/org/apache/avro/Conversions.java b/lang/java/avro/src/main/java/org/apache/avro/Conversions.java index 0c744f44393..efd1fe248ad 100644 --- a/lang/java/avro/src/main/java/org/apache/avro/Conversions.java +++ b/lang/java/avro/src/main/java/org/apache/avro/Conversions.java @@ -156,11 +156,7 @@ public static class DurationConversion extends Conversion { private static final String DOC_URL = "http://avro.apache.org/docs/current/spec.html#Duration"; private static final String DOC_STR = "For more information on the duration logical type, please refer to " + DOC_URL; - private static final byte[] EMPTY_BYTE_ARRAY = new byte[] { - 0x0, 0x0, 0x0, 0x0, - 0x0, 0x0, 0x0, 0x0, - 0x0, 0x0, 0x0, 0x0 - }; + private static final byte[] EMPTY_BYTE_ARRAY = new byte[12]; private static byte[] toBytes(int value){ return ByteBuffer.allocate(GRANULARITY_BYTES).order(ByteOrder.LITTLE_ENDIAN).putInt(value).array(); @@ -226,9 +222,9 @@ public GenericFixed toFixed(Duration value, Schema schema, LogicalType type) { long totalDays = value.toDays(); - int months = 0; - int days = 0; - int milliSeconds = 0; + int months; + int days; + int milliSeconds; try { months = (int) (totalDays / MONTH_DAYS); @@ -243,12 +239,14 @@ public GenericFixed toFixed(Duration value, Schema schema, LogicalType type) { } try { - milliSeconds = (int) value.minus(months * MONTH_DAYS, ChronoUnit.DAYS).minus(days, ChronoUnit.DAYS).toMillis(); + milliSeconds = (int) value.minus(totalDays, ChronoUnit.DAYS).toMillis(); } catch (Throwable e) { throw new IllegalArgumentException("The milliseconds part of a duration must fit a 4-byte int, longer duration given. " + DOC_STR, e); } - // TODO adjust millis with nano + if (value.getNano() - milliSeconds * 1_000_000 > 0) { + milliSeconds += 1; + } byte[] result = buildByteArray(months, days, milliSeconds); diff --git a/lang/java/avro/src/test/java/org/apache/avro/TestDurationConversion.java b/lang/java/avro/src/test/java/org/apache/avro/TestDurationConversion.java index 5ebad43ab3d..1cd46d0118d 100644 --- a/lang/java/avro/src/test/java/org/apache/avro/TestDurationConversion.java +++ b/lang/java/avro/src/test/java/org/apache/avro/TestDurationConversion.java @@ -32,6 +32,10 @@ import static java.math.RoundingMode.HALF_EVEN; import static org.junit.Assert.*; +/* + * TODO test for negative duration + * TODO test for empty byte array (0 duration) + */ public class TestDurationConversion { private static final Conversion CONVERSION = new Conversions.DurationConversion(); @@ -131,6 +135,23 @@ public void testConvertingMillisecondsToBytes() { assertArrayEquals(bytes, fixed.bytes()); } + @Test + public void testConvertingMillisecondsWithNanosecondAdjustmentToBytes() { + LogicalTypes.Duration duration = (LogicalTypes.Duration) logicalType; + + final Duration durationValue = Duration.ofMillis(200).plusNanos(10); + + final byte[] bytes = new byte[]{ + 0x0, 0x0, 0x0, 0x0, + 0x0, 0x0, 0x0, 0x0, + -0x37, 0x0, 0x0, 0x0 + }; + + final GenericFixed fixed = CONVERSION.toFixed(durationValue, schema, duration); + + assertArrayEquals(bytes, fixed.bytes()); + } + @Test public void testConvertingDaysToBytes() { LogicalTypes.Duration duration = (LogicalTypes.Duration) logicalType; From 9a9d4695fe4e753116550de0d1956d438375f159 Mon Sep 17 00:00:00 2001 From: R Vadai Date: Sat, 19 Jun 2021 12:16:08 +0100 Subject: [PATCH 5/5] AVRO-2123: Handling null duration value --- lang/java/avro/src/main/java/org/apache/avro/Conversions.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lang/java/avro/src/main/java/org/apache/avro/Conversions.java b/lang/java/avro/src/main/java/org/apache/avro/Conversions.java index efd1fe248ad..776b7eac2d5 100644 --- a/lang/java/avro/src/main/java/org/apache/avro/Conversions.java +++ b/lang/java/avro/src/main/java/org/apache/avro/Conversions.java @@ -216,7 +216,7 @@ public Duration fromFixed(GenericFixed value, Schema schema, LogicalType type) { @Override public GenericFixed toFixed(Duration value, Schema schema, LogicalType type) { - if (value.isZero()) { + if (value == null || value.isZero()) { return new GenericData.Fixed(schema, EMPTY_BYTE_ARRAY); }