Skip to content

Commit 64386b7

Browse files
staticlibsstaticlibs
committed
Appender rework
This change implements the access to the Appender interface from Java with the following features: - C API is used to access the native Appender - necessary C API calls are exposed to Java using JNI wrappers as thin as possible - Java calls mirror corresponding C API calls 1 to 1 - the data chunk interface of the Appender API is used: vector data is exposed as a direct ByteBuffer, all primitive appended values are written to this buffer from Java without going through JNI + C API ( that is still used for some types with calls like: `duckdb_vector_assign_string_element_len`) - Java-side Appender/DataChunk/Vector data structures follow the Go lang's Appender implementation closely (with nested arrays initialization etc) - Java Appender usage is made thread-safe for concurrent Appender or Connection closure; `append()` calls are remained not thread-safe (to minimize the overhead), but their usage cannot cause the JNI-side crash - Java API of the new Appender is modeled after the `java.lang.StringBuilder` class and intended to be used with method chaining - support primitive arrays, `NULL` and `DEFAULT` values is added - type checks between Java types and DB column types are enforced Note: previous version of the Appender (that internally creates `Value` instances and appends them one by on) is still available for compatibility. It can be created using `Connection#createSingleValueAppender` method. It is marked as 'deprecated' and intended to be removed in future versions. Testing: existing Appender test suite is extended and adapted to new Appender API. Fixes: #84, #100, #110, #139, #157, #163, #219, #249
1 parent 53f0fc2 commit 64386b7

39 files changed

+4909
-633
lines changed

‎CMakeLists.txt‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -553,7 +553,7 @@ endif()
553553

554554
file(GLOB_RECURSE JAVA_SRC_FILES src/main/java/org/duckdb/*.java)
555555
file(GLOB_RECURSE JAVA_TEST_FILES src/test/java/org/duckdb/*.java)
556-
set(CMAKE_JAVA_COMPILE_FLAGS -encoding utf-8 -g)
556+
set(CMAKE_JAVA_COMPILE_FLAGS -encoding utf-8 -g -Xlint:all)
557557

558558
add_jar(duckdb_jdbc ${JAVA_SRC_FILES} META-INF/services/java.sql.Driver
559559
MANIFEST META-INF/MANIFEST.MF
@@ -570,6 +570,12 @@ else()
570570
endif()
571571

572572
add_library(duckdb_java SHARED
573+
src/jni/bindings_appender.cpp
574+
src/jni/bindings_common.cpp
575+
src/jni/bindings_data_chunk.cpp
576+
src/jni/bindings_logical_type.cpp
577+
src/jni/bindings_validity.cpp
578+
src/jni/bindings_vector.cpp
573579
src/jni/config.cpp
574580
src/jni/duckdb_java.cpp
575581
src/jni/functions.cpp

‎CMakeLists.txt.in‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -83,7 +83,7 @@ endif()
8383

8484
file(GLOB_RECURSE JAVA_SRC_FILES src/main/java/org/duckdb/*.java)
8585
file(GLOB_RECURSE JAVA_TEST_FILES src/test/java/org/duckdb/*.java)
86-
set(CMAKE_JAVA_COMPILE_FLAGS -encoding utf-8 -g)
86+
set(CMAKE_JAVA_COMPILE_FLAGS -encoding utf-8 -g -Xlint:all)
8787

8888
add_jar(duckdb_jdbc ${JAVA_SRC_FILES} META-INF/services/java.sql.Driver
8989
MANIFEST META-INF/MANIFEST.MF
@@ -100,6 +100,12 @@ else()
100100
endif()
101101

102102
add_library(duckdb_java SHARED
103+
src/jni/bindings_appender.cpp
104+
src/jni/bindings_common.cpp
105+
src/jni/bindings_data_chunk.cpp
106+
src/jni/bindings_logical_type.cpp
107+
src/jni/bindings_validity.cpp
108+
src/jni/bindings_vector.cpp
103109
src/jni/config.cpp
104110
src/jni/duckdb_java.cpp
105111
src/jni/functions.cpp

‎duckdb_java.def‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,51 @@ Java_org_duckdb_DuckDBNative_duckdb_1jdbc_1set_1catalog
4747
Java_org_duckdb_DuckDBNative_duckdb_1jdbc_1set_1schema
4848
Java_org_duckdb_DuckDBNative_duckdb_1jdbc_1startup
4949

50+
Java_org_duckdb_DuckDBBindings_duckdb_1vector_1size
51+
Java_org_duckdb_DuckDBBindings_duckdb_1create_1logical_1type
52+
Java_org_duckdb_DuckDBBindings_duckdb_1get_1type_1id
53+
Java_org_duckdb_DuckDBBindings_duckdb_1decimal_1width
54+
Java_org_duckdb_DuckDBBindings_duckdb_1decimal_1scale
55+
Java_org_duckdb_DuckDBBindings_duckdb_1decimal_1internal_1type
56+
Java_org_duckdb_DuckDBBindings_duckdb_1create_1list_1type
57+
Java_org_duckdb_DuckDBBindings_duckdb_1create_1array_1type
58+
Java_org_duckdb_DuckDBBindings_duckdb_1create_1struct_1type
59+
Java_org_duckdb_DuckDBBindings_duckdb_1struct_1type_1child_1count
60+
Java_org_duckdb_DuckDBBindings_duckdb_1array_1type_1array_1size
61+
Java_org_duckdb_DuckDBBindings_duckdb_1destroy_1logical_1type
62+
Java_org_duckdb_DuckDBBindings_duckdb_1create_1vector
63+
Java_org_duckdb_DuckDBBindings_duckdb_1destroy_1vector
64+
Java_org_duckdb_DuckDBBindings_duckdb_1vector_1get_1column_1type
65+
Java_org_duckdb_DuckDBBindings_duckdb_1vector_1get_1data
66+
Java_org_duckdb_DuckDBBindings_duckdb_1vector_1get_1validity
67+
Java_org_duckdb_DuckDBBindings_duckdb_1vector_1ensure_1validity_1writable
68+
Java_org_duckdb_DuckDBBindings_duckdb_1vector_1assign_1string_1element
69+
Java_org_duckdb_DuckDBBindings_duckdb_1vector_1assign_1string_1element_1len
70+
Java_org_duckdb_DuckDBBindings_duckdb_1validity_1row_1is_1valid
71+
Java_org_duckdb_DuckDBBindings_duckdb_1validity_1set_1row_1validity
72+
Java_org_duckdb_DuckDBBindings_duckdb_1list_1vector_1get_1child
73+
Java_org_duckdb_DuckDBBindings_duckdb_1list_1vector_1get_1size
74+
Java_org_duckdb_DuckDBBindings_duckdb_1list_1vector_1set_1size
75+
Java_org_duckdb_DuckDBBindings_duckdb_1list_1vector_1reserve
76+
Java_org_duckdb_DuckDBBindings_duckdb_1struct_1vector_1get_1child
77+
Java_org_duckdb_DuckDBBindings_duckdb_1array_1vector_1get_1child
78+
Java_org_duckdb_DuckDBBindings_duckdb_1create_1data_1chunk
79+
Java_org_duckdb_DuckDBBindings_duckdb_1destroy_1data_1chunk
80+
Java_org_duckdb_DuckDBBindings_duckdb_1data_1chunk_1reset
81+
Java_org_duckdb_DuckDBBindings_duckdb_1data_1chunk_1get_1column_1count
82+
Java_org_duckdb_DuckDBBindings_duckdb_1data_1chunk_1get_1vector
83+
Java_org_duckdb_DuckDBBindings_duckdb_1data_1chunk_1get_1size
84+
Java_org_duckdb_DuckDBBindings_duckdb_1data_1chunk_1set_1size
85+
Java_org_duckdb_DuckDBBindings_duckdb_1appender_1create_1ext
86+
Java_org_duckdb_DuckDBBindings_duckdb_1appender_1error
87+
Java_org_duckdb_DuckDBBindings_duckdb_1appender_1flush
88+
Java_org_duckdb_DuckDBBindings_duckdb_1appender_1close
89+
Java_org_duckdb_DuckDBBindings_duckdb_1appender_1destroy
90+
Java_org_duckdb_DuckDBBindings_duckdb_1appender_1column_1count
91+
Java_org_duckdb_DuckDBBindings_duckdb_1appender_1column_1type
92+
Java_org_duckdb_DuckDBBindings_duckdb_1append_1data_1chunk
93+
Java_org_duckdb_DuckDBBindings_duckdb_1append_1default_1to_1chunk
94+
5095
duckdb_adbc_init
5196
duckdb_add_aggregate_function_to_set
5297
duckdb_add_replacement_scan

‎duckdb_java.exp‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,51 @@ _Java_org_duckdb_DuckDBNative_duckdb_1jdbc_1set_1catalog
4444
_Java_org_duckdb_DuckDBNative_duckdb_1jdbc_1set_1schema
4545
_Java_org_duckdb_DuckDBNative_duckdb_1jdbc_1startup
4646

47+
_Java_org_duckdb_DuckDBBindings_duckdb_1vector_1size
48+
_Java_org_duckdb_DuckDBBindings_duckdb_1create_1logical_1type
49+
_Java_org_duckdb_DuckDBBindings_duckdb_1get_1type_1id
50+
_Java_org_duckdb_DuckDBBindings_duckdb_1decimal_1width
51+
_Java_org_duckdb_DuckDBBindings_duckdb_1decimal_1scale
52+
_Java_org_duckdb_DuckDBBindings_duckdb_1decimal_1internal_1type
53+
_Java_org_duckdb_DuckDBBindings_duckdb_1create_1list_1type
54+
_Java_org_duckdb_DuckDBBindings_duckdb_1create_1array_1type
55+
_Java_org_duckdb_DuckDBBindings_duckdb_1create_1struct_1type
56+
_Java_org_duckdb_DuckDBBindings_duckdb_1struct_1type_1child_1count
57+
_Java_org_duckdb_DuckDBBindings_duckdb_1array_1type_1array_1size
58+
_Java_org_duckdb_DuckDBBindings_duckdb_1destroy_1logical_1type
59+
_Java_org_duckdb_DuckDBBindings_duckdb_1create_1vector
60+
_Java_org_duckdb_DuckDBBindings_duckdb_1destroy_1vector
61+
_Java_org_duckdb_DuckDBBindings_duckdb_1vector_1get_1column_1type
62+
_Java_org_duckdb_DuckDBBindings_duckdb_1vector_1get_1data
63+
_Java_org_duckdb_DuckDBBindings_duckdb_1vector_1get_1validity
64+
_Java_org_duckdb_DuckDBBindings_duckdb_1vector_1ensure_1validity_1writable
65+
_Java_org_duckdb_DuckDBBindings_duckdb_1vector_1assign_1string_1element
66+
_Java_org_duckdb_DuckDBBindings_duckdb_1vector_1assign_1string_1element_1len
67+
_Java_org_duckdb_DuckDBBindings_duckdb_1validity_1row_1is_1valid
68+
_Java_org_duckdb_DuckDBBindings_duckdb_1validity_1set_1row_1validity
69+
_Java_org_duckdb_DuckDBBindings_duckdb_1list_1vector_1get_1child
70+
_Java_org_duckdb_DuckDBBindings_duckdb_1list_1vector_1get_1size
71+
_Java_org_duckdb_DuckDBBindings_duckdb_1list_1vector_1set_1size
72+
_Java_org_duckdb_DuckDBBindings_duckdb_1list_1vector_1reserve
73+
_Java_org_duckdb_DuckDBBindings_duckdb_1struct_1vector_1get_1child
74+
_Java_org_duckdb_DuckDBBindings_duckdb_1array_1vector_1get_1child
75+
_Java_org_duckdb_DuckDBBindings_duckdb_1create_1data_1chunk
76+
_Java_org_duckdb_DuckDBBindings_duckdb_1destroy_1data_1chunk
77+
_Java_org_duckdb_DuckDBBindings_duckdb_1data_1chunk_1reset
78+
_Java_org_duckdb_DuckDBBindings_duckdb_1data_1chunk_1get_1column_1count
79+
_Java_org_duckdb_DuckDBBindings_duckdb_1data_1chunk_1get_1vector
80+
_Java_org_duckdb_DuckDBBindings_duckdb_1data_1chunk_1get_1size
81+
_Java_org_duckdb_DuckDBBindings_duckdb_1data_1chunk_1set_1size
82+
_Java_org_duckdb_DuckDBBindings_duckdb_1appender_1create_1ext
83+
_Java_org_duckdb_DuckDBBindings_duckdb_1appender_1error
84+
_Java_org_duckdb_DuckDBBindings_duckdb_1appender_1flush
85+
_Java_org_duckdb_DuckDBBindings_duckdb_1appender_1close
86+
_Java_org_duckdb_DuckDBBindings_duckdb_1appender_1destroy
87+
_Java_org_duckdb_DuckDBBindings_duckdb_1appender_1column_1count
88+
_Java_org_duckdb_DuckDBBindings_duckdb_1appender_1column_1type
89+
_Java_org_duckdb_DuckDBBindings_duckdb_1append_1data_1chunk
90+
_Java_org_duckdb_DuckDBBindings_duckdb_1append_1default_1to_1chunk
91+
4792
_duckdb_adbc_init
4893
_duckdb_add_aggregate_function_to_set
4994
_duckdb_add_replacement_scan

‎duckdb_java.map‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,51 @@ DUCKDB_JAVA {
4646
Java_org_duckdb_DuckDBNative_duckdb_1jdbc_1set_1schema;
4747
Java_org_duckdb_DuckDBNative_duckdb_1jdbc_1startup;
4848

49+
Java_org_duckdb_DuckDBBindings_duckdb_1vector_1size;
50+
Java_org_duckdb_DuckDBBindings_duckdb_1create_1logical_1type;
51+
Java_org_duckdb_DuckDBBindings_duckdb_1get_1type_1id;
52+
Java_org_duckdb_DuckDBBindings_duckdb_1decimal_1width;
53+
Java_org_duckdb_DuckDBBindings_duckdb_1decimal_1scale;
54+
Java_org_duckdb_DuckDBBindings_duckdb_1decimal_1internal_1type;
55+
Java_org_duckdb_DuckDBBindings_duckdb_1create_1list_1type;
56+
Java_org_duckdb_DuckDBBindings_duckdb_1create_1array_1type;
57+
Java_org_duckdb_DuckDBBindings_duckdb_1create_1struct_1type;
58+
Java_org_duckdb_DuckDBBindings_duckdb_1struct_1type_1child_1count;
59+
Java_org_duckdb_DuckDBBindings_duckdb_1array_1type_1array_1size;
60+
Java_org_duckdb_DuckDBBindings_duckdb_1destroy_1logical_1type;
61+
Java_org_duckdb_DuckDBBindings_duckdb_1create_1vector;
62+
Java_org_duckdb_DuckDBBindings_duckdb_1destroy_1vector;
63+
Java_org_duckdb_DuckDBBindings_duckdb_1vector_1get_1column_1type;
64+
Java_org_duckdb_DuckDBBindings_duckdb_1vector_1get_1data;
65+
Java_org_duckdb_DuckDBBindings_duckdb_1vector_1get_1validity;
66+
Java_org_duckdb_DuckDBBindings_duckdb_1vector_1ensure_1validity_1writable;
67+
Java_org_duckdb_DuckDBBindings_duckdb_1vector_1assign_1string_1element;
68+
Java_org_duckdb_DuckDBBindings_duckdb_1vector_1assign_1string_1element_1len;
69+
Java_org_duckdb_DuckDBBindings_duckdb_1validity_1row_1is_1valid;
70+
Java_org_duckdb_DuckDBBindings_duckdb_1validity_1set_1row_1validity;
71+
Java_org_duckdb_DuckDBBindings_duckdb_1list_1vector_1get_1child;
72+
Java_org_duckdb_DuckDBBindings_duckdb_1list_1vector_1get_1size;
73+
Java_org_duckdb_DuckDBBindings_duckdb_1list_1vector_1set_1size;
74+
Java_org_duckdb_DuckDBBindings_duckdb_1list_1vector_1reserve;
75+
Java_org_duckdb_DuckDBBindings_duckdb_1struct_1vector_1get_1child;
76+
Java_org_duckdb_DuckDBBindings_duckdb_1array_1vector_1get_1child;
77+
Java_org_duckdb_DuckDBBindings_duckdb_1create_1data_1chunk;
78+
Java_org_duckdb_DuckDBBindings_duckdb_1destroy_1data_1chunk;
79+
Java_org_duckdb_DuckDBBindings_duckdb_1data_1chunk_1reset;
80+
Java_org_duckdb_DuckDBBindings_duckdb_1data_1chunk_1get_1column_1count;
81+
Java_org_duckdb_DuckDBBindings_duckdb_1data_1chunk_1get_1vector;
82+
Java_org_duckdb_DuckDBBindings_duckdb_1data_1chunk_1get_1size;
83+
Java_org_duckdb_DuckDBBindings_duckdb_1data_1chunk_1set_1size;
84+
Java_org_duckdb_DuckDBBindings_duckdb_1appender_1create_1ext;
85+
Java_org_duckdb_DuckDBBindings_duckdb_1appender_1error;
86+
Java_org_duckdb_DuckDBBindings_duckdb_1appender_1flush;
87+
Java_org_duckdb_DuckDBBindings_duckdb_1appender_1close;
88+
Java_org_duckdb_DuckDBBindings_duckdb_1appender_1destroy;
89+
Java_org_duckdb_DuckDBBindings_duckdb_1appender_1column_1count;
90+
Java_org_duckdb_DuckDBBindings_duckdb_1appender_1column_1type;
91+
Java_org_duckdb_DuckDBBindings_duckdb_1append_1data_1chunk;
92+
Java_org_duckdb_DuckDBBindings_duckdb_1append_1default_1to_1chunk;
93+
4994
duckdb_adbc_init;
5095
duckdb_add_aggregate_function_to_set;
5196
duckdb_add_replacement_scan;

‎src/jni/bindings.hpp‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
#pragma once
2+
3+
extern "C" {
4+
#include "duckdb.h"
5+
}
6+
7+
#include "org_duckdb_DuckDBBindings.h"
8+
9+
duckdb_logical_type logical_type_buf_to_logical_type(JNIEnv *env, jobject logical_type_buf);
10+
11+
duckdb_data_chunk chunk_buf_to_chunk(JNIEnv *env, jobject chunk_buf);

0 commit comments

Comments
 (0)