From 461f4cb9a6835f55201e9bd03c127a153a2306c4 Mon Sep 17 00:00:00 2001 From: Oscar Westra van Holthe - Kind Date: Tue, 15 Apr 2025 12:06:50 +0200 Subject: [PATCH 1/4] AVRO-4123: fix missing parsed schemas --- .../avro/src/main/java/org/apache/avro/ParseContext.java | 3 +-- .../src/test/java/org/apache/avro/TestSchemaParser.java | 9 +++++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/lang/java/avro/src/main/java/org/apache/avro/ParseContext.java b/lang/java/avro/src/main/java/org/apache/avro/ParseContext.java index b7bc42b9787..0c405dd6404 100644 --- a/lang/java/avro/src/main/java/org/apache/avro/ParseContext.java +++ b/lang/java/avro/src/main/java/org/apache/avro/ParseContext.java @@ -21,7 +21,6 @@ import org.apache.avro.util.Schemas; import java.util.ArrayList; -import java.util.Collection; import java.util.EnumSet; import java.util.HashMap; import java.util.LinkedHashMap; @@ -253,7 +252,7 @@ public void commit() { } public SchemaParser.ParseResult commit(Schema mainSchema) { - Collection parsedNamedSchemas = newSchemas.values(); + List parsedNamedSchemas = new ArrayList<>(newSchemas.values()); SchemaParser.ParseResult parseResult = new SchemaParser.ParseResult() { @Override public Schema mainSchema() { diff --git a/lang/java/avro/src/test/java/org/apache/avro/TestSchemaParser.java b/lang/java/avro/src/test/java/org/apache/avro/TestSchemaParser.java index 29c8f65be66..f382f8ee6ed 100644 --- a/lang/java/avro/src/test/java/org/apache/avro/TestSchemaParser.java +++ b/lang/java/avro/src/test/java/org/apache/avro/TestSchemaParser.java @@ -27,6 +27,7 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; +import java.util.List; import static java.util.Collections.singletonList; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -35,7 +36,7 @@ class TestSchemaParser { private static final Schema SCHEMA_REAL = Schema.createFixed("Real", null, "tests", 42); - private static final String SCHEMA_JSON = SCHEMA_REAL.toString(false); + private static final String SCHEMA_JSON = SchemaFormatter.getInstance("json").format(SCHEMA_REAL); private static final Charset[] UTF_CHARSETS = { StandardCharsets.UTF_8, StandardCharsets.UTF_16LE, StandardCharsets.UTF_16BE }; @@ -88,7 +89,11 @@ void testParseTextWithFallbackJsonParser() { @Test void testParseByCustomParser() { - Schema schema = new SchemaParser().parse(DummySchemaParser.SCHEMA_TEXT_ONE).mainSchema(); + SchemaParser.ParseResult parseResult = new SchemaParser().parse(DummySchemaParser.SCHEMA_TEXT_ONE); + List namedSchemas = parseResult.parsedNamedSchemas(); + assertEquals(1, namedSchemas.size()); + assertEquals(DummySchemaParser.FIXED_SCHEMA, namedSchemas.getFirst()); + Schema schema = parseResult.mainSchema(); assertEquals(DummySchemaParser.FIXED_SCHEMA, schema); } From 2994ab0535bb02639a1ad2c409ccdcc4ab74b3b9 Mon Sep 17 00:00:00 2001 From: Oscar Westra van Holthe - Kind Date: Tue, 15 Apr 2025 12:07:27 +0200 Subject: [PATCH 2/4] AVRO-4123: Remove unnecessary private method --- .../src/main/java/org/apache/avro/idl/IdlReader.java | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/lang/java/idl/src/main/java/org/apache/avro/idl/IdlReader.java b/lang/java/idl/src/main/java/org/apache/avro/idl/IdlReader.java index 96d45ab6dc2..f6d1fb49dcb 100644 --- a/lang/java/idl/src/main/java/org/apache/avro/idl/IdlReader.java +++ b/lang/java/idl/src/main/java/org/apache/avro/idl/IdlReader.java @@ -164,10 +164,6 @@ private Schema namedSchemaOrUnresolved(String fullName) { return parseContext.find(fullName, null); } - private void addSchema(Schema schema) { - parseContext.put(schema); - } - public IdlFile parse(Path location) throws IOException { return parse(location.toUri()); } @@ -437,7 +433,7 @@ public void exitImportStatement(ImportStatementContext importContext) { try (InputStream stream = importLocation.toURL().openStream()) { Protocol importProtocol = Protocol.parse(stream); for (Schema s : importProtocol.getTypes()) { - addSchema(s); + parseContext.put(s); } if (protocol != null) { protocol.getMessages().putAll(importProtocol.getMessages()); @@ -515,7 +511,7 @@ public void exitFixedDeclaration(FixedDeclarationContext ctx) { Schema schema = Schema.createFixed(name, doc, space, size); properties.copyAliases(schema::addAlias); properties.copyProperties(schema); - addSchema(schema); + parseContext.put(schema); } @Override @@ -539,7 +535,7 @@ public void exitEnumDeclaration(EnumDeclarationContext ctx) { enumSymbols.clear(); enumDefaultSymbol = null; - addSchema(schema); + parseContext.put(schema); } @Override @@ -590,7 +586,7 @@ public void enterRecordBody(RecordBodyContext ctx) { public void exitRecordDeclaration(RecordDeclarationContext ctx) { schema.setFields(fields); fields.clear(); - addSchema(schema); + parseContext.put(schema); schema = null; popNamespace(); From b30cc910efc679758554d37d03b1e4661548df00 Mon Sep 17 00:00:00 2001 From: Oscar Westra van Holthe - Kind Date: Fri, 18 Apr 2025 13:09:37 +0200 Subject: [PATCH 3/4] AVRO-4123: Do not use List.getFirst() yet --- .../avro/src/test/java/org/apache/avro/TestSchemaParser.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lang/java/avro/src/test/java/org/apache/avro/TestSchemaParser.java b/lang/java/avro/src/test/java/org/apache/avro/TestSchemaParser.java index f382f8ee6ed..186579bb5d3 100644 --- a/lang/java/avro/src/test/java/org/apache/avro/TestSchemaParser.java +++ b/lang/java/avro/src/test/java/org/apache/avro/TestSchemaParser.java @@ -92,7 +92,7 @@ void testParseByCustomParser() { SchemaParser.ParseResult parseResult = new SchemaParser().parse(DummySchemaParser.SCHEMA_TEXT_ONE); List namedSchemas = parseResult.parsedNamedSchemas(); assertEquals(1, namedSchemas.size()); - assertEquals(DummySchemaParser.FIXED_SCHEMA, namedSchemas.getFirst()); + assertEquals(DummySchemaParser.FIXED_SCHEMA, namedSchemas.get(0)); Schema schema = parseResult.mainSchema(); assertEquals(DummySchemaParser.FIXED_SCHEMA, schema); } From ba8b8176857aff79400009cd129525d49469e1d5 Mon Sep 17 00:00:00 2001 From: Oscar Westra van Holthe - Kind Date: Fri, 18 Apr 2025 16:41:55 +0200 Subject: [PATCH 4/4] AVRO-4123: Fix tests for recent JDKs Recent JDKs alter the requirements for using a security manager. This affects Hadoop code. Also, misplaced javadoc affects tests. When AVRO-4126 is merged, the change to TestSpecificCompiler.java can be reverted. --- .../org/apache/avro/compiler/specific/TestSpecificCompiler.java | 2 +- lang/java/mapred/pom.xml | 2 ++ lang/java/tools/pom.xml | 2 ++ lang/java/trevni/pom.xml | 2 ++ 4 files changed, 7 insertions(+), 1 deletion(-) diff --git a/lang/java/compiler/src/test/java/org/apache/avro/compiler/specific/TestSpecificCompiler.java b/lang/java/compiler/src/test/java/org/apache/avro/compiler/specific/TestSpecificCompiler.java index 19d63d033c7..1916ab43c4d 100644 --- a/lang/java/compiler/src/test/java/org/apache/avro/compiler/specific/TestSpecificCompiler.java +++ b/lang/java/compiler/src/test/java/org/apache/avro/compiler/specific/TestSpecificCompiler.java @@ -87,7 +87,7 @@ public void setUp() { static void assertCompilesWithJavaCompiler(File dstDir, Collection outputs) throws IOException { - assertCompilesWithJavaCompiler(dstDir, outputs, false); + assertCompilesWithJavaCompiler(dstDir, outputs, true); } /** diff --git a/lang/java/mapred/pom.xml b/lang/java/mapred/pom.xml index 5c4d8733cf3..91982cfd110 100644 --- a/lang/java/mapred/pom.xml +++ b/lang/java/mapred/pom.xml @@ -92,6 +92,8 @@ 1 false none + + -Djava.security.manager=allow diff --git a/lang/java/tools/pom.xml b/lang/java/tools/pom.xml index 1342458dd5b..4ed8a5fa8de 100644 --- a/lang/java/tools/pom.xml +++ b/lang/java/tools/pom.xml @@ -158,6 +158,8 @@ 1 false none + + -Djava.security.manager=allow diff --git a/lang/java/trevni/pom.xml b/lang/java/trevni/pom.xml index 1a1e8581acb..64fd16c10f6 100644 --- a/lang/java/trevni/pom.xml +++ b/lang/java/trevni/pom.xml @@ -55,6 +55,8 @@ 1 false none + + -Djava.security.manager=allow