Appender: support composite/nested lists and maps

This PR adds support for appending to `MAP` and composite `LIST`
columns, for example `LIST` of `STRUCT`s, `LIST` of `ARRAY`s or
nested `LIST`s.

It should cover virtually any table schema with composite fields, with
the notable exception of `ARRAY`s with more than 2 dimensions (like
`INT[2][3][4]`) - only 2D `ARRAY`s are supported. Higher dimensions
could be possible, but the interface for them appeared to be tough to
generalize. `LIST`s of `ARRAY`s can be used instead, like `INT[2][3][]`.

`LIST`s can be specified as a `Collection` or `Iterator` (with `count`)
of objects.

`MAP`s are specified as `java.util.Map` instances.

`STRUCT` inside a `LIST` can be specified either as
`java.utils.LinkedHashMap` (keys are ignored) or as a `Collection` of
values for `STRUCT` fields.

`UNION` inside a `LIST` can be specified as an instance of
`java.util.AbstractMap.SimpleEntry`, where a `key` is used as a tag to
choose which `UNION` field to append to.

If some required input cases are missed - please raise an issue.

Testing: new tests added to cover various combinations of nesting.

Author: staticlibs

src/main/java/org/duckdb/DuckDBAppender.java

@@ -169,7 +169,7 @@ enum CAPIType {
         // struct type, only useful as logical type
         DUCKDB_TYPE_STRUCT(25, 0),
         // map type, only useful as logical type
-        DUCKDB_TYPE_MAP(26),
+        DUCKDB_TYPE_MAP(26, 16),
         // duckdb_array, only useful as logical type
         DUCKDB_TYPE_ARRAY(33, 0),
         // duckdb_hugeint

src/main/java/org/duckdb/DuckDBBindings.java

@@ -12,7 +12,7 @@
 import java.time.OffsetDateTime;
 import java.time.OffsetTime;
 import java.util.ArrayList;
-import java.util.HashMap;
+import java.util.LinkedHashMap;
 import java.util.UUID;
 
 public class DuckDBResultSetMetaData implements ResultSetMetaData {
@@ -234,7 +234,7 @@ protected static String type_to_javaString(DuckDBColumnType type) {
         case ARRAY:
             return DuckDBArray.class.getName();
         case MAP:
-            return HashMap.class.getName();
+            return LinkedHashMap.class.getName();
         case STRUCT:
             return DuckDBStruct.class.getName();
         default:

src/main/java/org/duckdb/DuckDBResultSetMetaData.java

@@ -286,7 +286,7 @@ Map<Object, Object> getMap(int idx) throws SQLException {
         }
 
         Object[] entries = (Object[]) (((Array) varlen_data[idx]).getArray());
-        Map<Object, Object> result = new HashMap<>();
+        Map<Object, Object> result = new LinkedHashMap<>();
 
         for (Object entry : entries) {
             Object[] entry_val = ((Struct) entry).getAttributes();

src/main/java/org/duckdb/DuckDBVector.java

@@ -144,8 +144,8 @@ public static void test_appender_uuid() throws Exception {
              Statement stmt = conn.createStatement()) {
 
             stmt.execute("CREATE TABLE tab1(col1 INT, col2 UUID)");
-            UUID uuid1 = UUID.randomUUID();
-            UUID uuid2 = UUID.randomUUID();
+            UUID uuid1 = UUID.fromString("777dfbdb-83e7-40f5-ae1b-e12215bdd798");
+            UUID uuid2 = UUID.fromString("b8708825-3b58-45a1-9a6e-dab053c3f387");
             try (DuckDBAppender appender = conn.createAppender("tab1")) {
                 appender.beginRow().append(1).append(uuid1).endRow();
                 appender.beginRow().append(2).append(uuid2).endRow();

src/test/java/org/duckdb/TestAppender.java

@@ -1,13 +1,18 @@
 package org.duckdb;
 
+import static java.util.Arrays.asList;
+import static java.util.Collections.singletonList;
 import static org.duckdb.TestDuckDBJDBC.JDBC_URL;
 import static org.duckdb.test.Assertions.*;
+import static org.duckdb.test.Helpers.createMap;
 
+import java.math.BigDecimal;
+import java.math.BigInteger;
 import java.sql.DriverManager;
 import java.sql.ResultSet;
 import java.sql.SQLException;
 import java.sql.Statement;
-import java.util.Map;
+import java.util.*;
 
 public class TestAppenderComposite {
 
@@ -474,4 +479,195 @@ public static void test_appender_union_nested() throws Exception {
             }
         }
     }
+
+    private static void assertFetchedStructEquals(Object dbs, Collection<Object> struct) throws Exception {
+        DuckDBStruct dbStruct = (DuckDBStruct) dbs;
+        Map<String, Object> map = dbStruct.getMap();
+        Collection<Object> fetched = map.values();
+        assertEquals(fetched.size(), struct.size());
+        List<Object> structList = new ArrayList<>(struct);
+        int i = 0;
+        for (Object f : fetched) {
+            assertEquals(f, structList.get(i));
+            i++;
+        }
+    }
+
+    public static void test_appender_list_basic_struct() throws Exception {
+        Collection<Object> struct1 = asList(42, "foo");
+        Collection<Object> struct2 = asList(null, "bar");
+        Collection<Object> struct3 = asList(43, null);
+        Collection<Object> struct4 = asList(44, "baz");
+        try (DuckDBConnection conn = DriverManager.getConnection(JDBC_URL).unwrap(DuckDBConnection.class);
+             Statement stmt = conn.createStatement()) {
+            stmt.execute("CREATE TABLE tab1(col1 INT, col2 STRUCT(s1 INT, s2 VARCHAR)[])");
+            try (DuckDBAppender appender = conn.createAppender("tab1")) {
+                appender.beginRow()
+                    .append(42)
+                    .append(asList(struct1, struct2, struct3))
+                    .endRow()
+                    .beginRow()
+                    .append(43)
+                    .append((List<Object>) null)
+                    .endRow()
+                    .beginRow()
+                    .append(44)
+                    .append(asList(null, struct4))
+                    .endRow()
+                    .flush();
+            }
+
+            try (ResultSet rs = stmt.executeQuery("SELECT unnest(col2) from tab1 WHERE col1 = 42")) {
+                assertTrue(rs.next());
+                assertFetchedStructEquals(rs.getObject(1), struct1);
+                assertTrue(rs.next());
+                assertFetchedStructEquals(rs.getObject(1), struct2);
+                assertTrue(rs.next());
+                assertFetchedStructEquals(rs.getObject(1), struct3);
+                assertFalse(rs.next());
+            }
+            try (ResultSet rs = stmt.executeQuery("SELECT col2 from tab1 WHERE col1 = 43")) {
+                assertTrue(rs.next());
+                assertNull(rs.getObject(1));
+                assertTrue(rs.wasNull());
+                assertFalse(rs.next());
+            }
+            try (ResultSet rs = stmt.executeQuery("SELECT unnest(col2) from tab1 WHERE col1 = 44")) {
+                assertTrue(rs.next());
+                assertNull(rs.getObject(1));
+                assertTrue(rs.wasNull());
+                assertTrue(rs.next());
+                assertFetchedStructEquals(rs.getObject(1), struct4);
+                assertFalse(rs.next());
+            }
+        }
+    }
+
+    public static void test_appender_list_basic_struct_as_map() throws Exception {
+        LinkedHashMap<Object, Object> struct1 = createMap("key1", 42, "key2", "foo");
+        LinkedHashMap<Object, Object> struct2 = createMap("key1", null, "key2", "bar");
+        LinkedHashMap<Object, Object> struct3 = createMap("key1", 43, "key2", null);
+        LinkedHashMap<Object, Object> struct4 = createMap("key1", 44, "key2", "baz");
+        try (DuckDBConnection conn = DriverManager.getConnection(JDBC_URL).unwrap(DuckDBConnection.class);
+             Statement stmt = conn.createStatement()) {
+            stmt.execute("CREATE TABLE tab1(col1 INT, col2 STRUCT(s1 INT, s2 VARCHAR)[])");
+            try (DuckDBAppender appender = conn.createAppender("tab1")) {
+                appender.beginRow()
+                    .append(42)
+                    .append(asList(struct1, struct2, struct3))
+                    .endRow()
+                    .beginRow()
+                    .append(43)
+                    .append((List<Object>) null)
+                    .endRow()
+                    .beginRow()
+                    .append(44)
+                    .append(asList(null, struct4))
+                    .endRow()
+                    .flush();
+            }
+
+            try (ResultSet rs = stmt.executeQuery("SELECT unnest(col2) from tab1 WHERE col1 = 42")) {
+                assertTrue(rs.next());
+                assertFetchedStructEquals(rs.getObject(1), struct1.values());
+                assertTrue(rs.next());
+                assertFetchedStructEquals(rs.getObject(1), struct2.values());
+                assertTrue(rs.next());
+                assertFetchedStructEquals(rs.getObject(1), struct3.values());
+                assertFalse(rs.next());
+            }
+            try (ResultSet rs = stmt.executeQuery("SELECT col2 from tab1 WHERE col1 = 43")) {
+                assertTrue(rs.next());
+                assertNull(rs.getObject(1));
+                assertTrue(rs.wasNull());
+                assertFalse(rs.next());
+            }
+            try (ResultSet rs = stmt.executeQuery("SELECT unnest(col2) from tab1 WHERE col1 = 44")) {
+                assertTrue(rs.next());
+                assertNull(rs.getObject(1));
+                assertTrue(rs.wasNull());
+                assertTrue(rs.next());
+                assertFetchedStructEquals(rs.getObject(1), struct4.values());
+                assertFalse(rs.next());
+            }
+        }
+    }
+
+    public static void test_appender_list_basic_struct_with_primitives() throws Exception {
+        Collection<Object> struct1 = asList(true, (byte) 42, (short) 43, 44, 45L, BigInteger.valueOf(46), 47.1F, 48.1D,
+                                            BigDecimal.valueOf(49.123));
+        try (DuckDBConnection conn = DriverManager.getConnection(JDBC_URL).unwrap(DuckDBConnection.class);
+             Statement stmt = conn.createStatement()) {
+            stmt.execute("CREATE TABLE tab1(col1 INT, col2 STRUCT("
+                         + "s1 BOOL,"
+                         + "s2 TINYINT,"
+                         + "s3 SMALLINT,"
+                         + "s4 INTEGER,"
+                         + "s5 BIGINT,"
+                         + "s6 HUGEINT,"
+                         + "s7 FLOAT,"
+                         + "s8 DOUBLE,"
+                         + "s9 DECIMAL"
+                         + ")[])");
+            try (DuckDBAppender appender = conn.createAppender("tab1")) {
+                appender.beginRow().append(42).append(singletonList(struct1)).endRow().flush();
+            }
+
+            try (ResultSet rs = stmt.executeQuery("SELECT unnest(col2) from tab1 WHERE col1 = 42")) {
+                assertTrue(rs.next());
+                assertFetchedStructEquals(rs.getObject(1), struct1);
+                assertFalse(rs.next());
+            }
+        }
+    }
+
+    public static void test_appender_list_basic_union() throws Exception {
+        Map.Entry<String, Object> union1 = new AbstractMap.SimpleEntry<>("u1", 42);
+        Map.Entry<String, Object> union2 = new AbstractMap.SimpleEntry<>("u2", "foo");
+        Map.Entry<String, Object> union3 = new AbstractMap.SimpleEntry<>("u1", null);
+        Map.Entry<String, Object> union4 = new AbstractMap.SimpleEntry<>("u2", "bar");
+        try (DuckDBConnection conn = DriverManager.getConnection(JDBC_URL).unwrap(DuckDBConnection.class);
+             Statement stmt = conn.createStatement()) {
+            stmt.execute("CREATE TABLE tab1(col1 INT, col2 UNION(u1 INT, u2 VARCHAR)[])");
+            try (DuckDBAppender appender = conn.createAppender("tab1")) {
+                appender.beginRow()
+                    .append(42)
+                    .append(asList(union1, union2, union3))
+                    .endRow()
+                    .beginRow()
+                    .append(43)
+                    .append((List<Object>) null)
+                    .endRow()
+                    .beginRow()
+                    .append(44)
+                    .append(asList(null, union4))
+                    .endRow()
+                    .flush();
+            }
+
+            try (ResultSet rs = stmt.executeQuery("SELECT unnest(col2) from tab1 WHERE col1 = 42")) {
+                assertTrue(rs.next());
+                assertEquals(rs.getObject(1), union1.getValue());
+                assertTrue(rs.next());
+                assertEquals(rs.getObject(1), union2.getValue());
+                assertTrue(rs.next());
+                assertEquals(rs.getObject(1), union3.getValue());
+                assertFalse(rs.next());
+            }
+            try (ResultSet rs = stmt.executeQuery("SELECT col2 from tab1 WHERE col1 = 43")) {
+                assertTrue(rs.next());
+                assertNull(rs.getObject(1));
+                assertTrue(rs.wasNull());
+                assertFalse(rs.next());
+            }
+            try (ResultSet rs = stmt.executeQuery("SELECT unnest(col2) from tab1 WHERE col1 = 44")) {
+                assertTrue(rs.next());
+                assertNull(rs.getObject(1));
+                assertTrue(rs.wasNull());
+                assertTrue(rs.next());
+                assertEquals(rs.getObject(1), union4.getValue());
+                assertFalse(rs.next());
+            }
+        }
+    }
 }

src/test/java/org/duckdb/TestAppenderCollection.java

@@ -3156,10 +3156,10 @@ public static void main(String[] args) throws Exception {
             statusCode = runTests(new String[0], clazz);
         } else {
             statusCode = runTests(args, TestDuckDBJDBC.class, TestAppender.class, TestAppenderCollection.class,
-                                  TestAppenderComposite.class, TestSingleValueAppender.class, TestBatch.class,
-                                  TestBindings.class, TestClosure.class, TestExtensionTypes.class, TestSpatial.class,
-                                  TestParameterMetadata.class, TestPrepare.class, TestResults.class,
-                                  TestSessionInit.class, TestTimestamp.class);
+                                  TestAppenderCollection2D.class, TestAppenderComposite.class,
+                                  TestSingleValueAppender.class, TestBatch.class, TestBindings.class, TestClosure.class,
+                                  TestExtensionTypes.class, TestSpatial.class, TestParameterMetadata.class,
+                                  TestPrepare.class, TestResults.class, TestSessionInit.class, TestTimestamp.class);
         }
         System.exit(statusCode);
     }

src/test/java/org/duckdb/TestAppenderCollection2D.java

@@ -6,7 +6,7 @@
 
 import java.math.BigDecimal;
 import java.sql.*;
-import java.util.HashMap;
+import java.util.LinkedHashMap;
 
 public class TestParameterMetadata {
 
@@ -117,7 +117,7 @@ public static void test_parameter_metadata_map() throws Exception {
             try (PreparedStatement ps = conn.prepareStatement("INSERT INTO metadata_test_map_1 VALUES(?)")) {
                 ParameterMetaData meta = ps.getParameterMetaData();
                 assertEquals(meta.getParameterTypeName(1), "MAP(INTEGER, DOUBLE)");
-                assertEquals(meta.getParameterClassName(1), HashMap.class.getName());
+                assertEquals(meta.getParameterClassName(1), LinkedHashMap.class.getName());
                 assertEquals(meta.getPrecision(1), 0);
                 assertEquals(meta.getScale(1), 0);
             }