Update vendored DuckDB sources to 535b7a6a14

Author: duckdblabs-bot

src/duckdb/extension/json/include/json_structure.hpp

@@ -80,6 +80,9 @@ struct JSONStructureDescription {
 
 	//! Candidate types (if auto-detecting and type == LogicalTypeId::VARCHAR)
 	vector<LogicalTypeId> candidate_types;
+
+	//! Whether any UBIGINT value exceeds the BIGINT range (i.e., >= 2^63)
+	bool has_large_ubigint = false;
 };
 
 struct JSONStructure {

src/duckdb/extension/json/json_functions/copy_json.cpp

@@ -55,6 +55,9 @@ static BoundStatement CopyToJSONPlan(Binder &binder, CopyStatement &stmt) {
 				csv_copy_options["suffix"] = {"\n]\n"};
 				csv_copy_options["new_line"] = {",\n\t"};
 			}
+		} else if (loption == "file_extension") {
+			// Since we set the file extension to "json" above, we need to override it
+			csv_copy_options["file_extension"] = {StringValue::Get(kv.second.back())};
 		} else if (SUPPORTED_BASE_OPTIONS.find(loption) != SUPPORTED_BASE_OPTIONS.end()) {
 			// We support these base options
 			csv_copy_options.insert(kv);

src/duckdb/extension/json/json_functions/json_structure.cpp

@@ -9,14 +9,23 @@
 namespace duckdb {
 
 static bool IsNumeric(LogicalTypeId type) {
-	return type == LogicalTypeId::DOUBLE || type == LogicalTypeId::UBIGINT || type == LogicalTypeId::BIGINT;
+	return type == LogicalTypeId::DOUBLE || type == LogicalTypeId::UBIGINT || type == LogicalTypeId::BIGINT ||
+	       type == LogicalTypeId::HUGEINT;
 }
 
 static LogicalTypeId MaxNumericType(const LogicalTypeId &a, const LogicalTypeId &b) {
 	D_ASSERT(a != b);
 	if (a == LogicalTypeId::DOUBLE || b == LogicalTypeId::DOUBLE) {
 		return LogicalTypeId::DOUBLE;
 	}
+	if (a == LogicalTypeId::HUGEINT || b == LogicalTypeId::HUGEINT) {
+		return LogicalTypeId::HUGEINT;
+	}
+	// One is BIGINT and the other is UBIGINT - need HUGEINT to represent both ranges
+	if ((a == LogicalTypeId::BIGINT && b == LogicalTypeId::UBIGINT) ||
+	    (a == LogicalTypeId::UBIGINT && b == LogicalTypeId::BIGINT)) {
+		return LogicalTypeId::HUGEINT;
+	}
 	return LogicalTypeId::BIGINT;
 }
 
@@ -343,6 +352,7 @@ static void SwapJSONStructureDescription(JSONStructureDescription &a, JSONStruct
 	std::swap(a.key_map, b.key_map);
 	std::swap(a.children, b.children);
 	std::swap(a.candidate_types, b.candidate_types);
+	std::swap(a.has_large_ubigint, b.has_large_ubigint);
 }
 
 JSONStructureDescription::JSONStructureDescription(JSONStructureDescription &&other) noexcept {
@@ -427,7 +437,11 @@ static void ExtractStructureObject(yyjson_val *obj, JSONStructureNode &node, con
 
 static void ExtractStructureVal(yyjson_val *val, JSONStructureNode &node) {
 	D_ASSERT(!yyjson_is_arr(val) && !yyjson_is_obj(val));
-	node.GetOrCreateDescription(JSONCommon::ValTypeToLogicalTypeId(val));
+	auto &desc = node.GetOrCreateDescription(JSONCommon::ValTypeToLogicalTypeId(val));
+	if (desc.type == LogicalTypeId::UBIGINT &&
+	    unsafe_yyjson_get_uint(val) > static_cast<uint64_t>(NumericLimits<int64_t>::Maximum())) {
+		desc.has_large_ubigint = true;
+	}
 }
 
 void JSONStructure::ExtractStructure(yyjson_val *val, JSONStructureNode &node, const bool ignore_errors) {
@@ -558,6 +572,9 @@ static void MergeNodeVal(JSONStructureNode &merged, const JSONStructureDescripti
                          const bool node_initialized) {
 	D_ASSERT(child_desc.type != LogicalTypeId::LIST && child_desc.type != LogicalTypeId::STRUCT);
 	auto &merged_desc = merged.GetOrCreateDescription(child_desc.type);
+	if (child_desc.has_large_ubigint) {
+		merged_desc.has_large_ubigint = true;
+	}
 	if (merged_desc.type != LogicalTypeId::VARCHAR || !node_initialized || merged.descriptions.size() != 1) {
 		return;
 	}
@@ -798,7 +815,10 @@ LogicalType JSONStructure::StructureToType(ClientContext &context, const JSONStr
 	case LogicalTypeId::VARCHAR:
 		return StructureToTypeString(node);
 	case LogicalTypeId::UBIGINT:
-		return LogicalTypeId::BIGINT; // We prefer not to return UBIGINT in our type auto-detection
+		if (desc.has_large_ubigint) {
+			return LogicalTypeId::HUGEINT;
+		}
+		return LogicalTypeId::BIGINT;
 	case LogicalTypeId::SQLNULL:
 		return null_type;
 	default:

src/duckdb/extension/json/json_reader.cpp

@@ -1,10 +1,13 @@
 #include "json_reader.hpp"
 
+#include <utility>
+
 #include "duckdb/common/file_opener.hpp"
+#include "duckdb/common/file_open_flags.hpp"
 #include "duckdb/common/serializer/deserializer.hpp"
 #include "duckdb/common/serializer/serializer.hpp"
+#include "duckdb/storage/caching_mode.hpp"
 #include "json_scan.hpp"
-#include <utility>
 
 namespace duckdb {
 
@@ -183,7 +186,9 @@ void JSONReader::OpenJSONFile() {
 	lock_guard<mutex> guard(lock);
 	if (!IsOpen()) {
 		auto &fs = FileSystem::GetFileSystem(context);
-		auto regular_file_handle = fs.OpenFile(file, FileFlags::FILE_FLAGS_READ | options.compression);
+		FileOpenFlags flags = FileFlags::FILE_FLAGS_READ | options.compression;
+		flags.SetCachingMode(CachingMode::CACHE_REMOTE_ONLY);
+		auto regular_file_handle = fs.OpenFile(file, flags);
 		file_handle = make_uniq<JSONFileHandle>(context, std::move(regular_file_handle), BufferAllocator::Get(context));
 	}
 	Reset();
@@ -1052,8 +1057,9 @@ void JSONReader::ReadNextBufferSeek(JSONReaderScanState &scan_state) {
 			if (!raw_handle.OnDiskFile() && raw_handle.CanSeek()) {
 				if (!scan_state.thread_local_filehandle ||
 				    scan_state.thread_local_filehandle->GetPath() != raw_handle.GetPath()) {
-					scan_state.thread_local_filehandle = scan_state.fs.OpenFile(
-					    raw_handle.GetPath(), FileFlags::FILE_FLAGS_READ | FileFlags::FILE_FLAGS_DIRECT_IO);
+					FileOpenFlags flags = FileFlags::FILE_FLAGS_READ | FileFlags::FILE_FLAGS_DIRECT_IO;
+					flags.SetCachingMode(CachingMode::CACHE_REMOTE_ONLY);
+					scan_state.thread_local_filehandle = scan_state.fs.OpenFile(raw_handle.GetPath(), flags);
 				}
 			} else if (scan_state.thread_local_filehandle) {
 				scan_state.thread_local_filehandle = nullptr;

src/duckdb/extension/parquet/include/parquet_geometry.hpp

@@ -85,14 +85,14 @@ class GeoParquetFileMetadata {
 public:
 	explicit GeoParquetFileMetadata(GeoParquetVersion geo_parquet_version) : version(geo_parquet_version) {
 	}
-	void AddGeoParquetStats(const string &column_name, const LogicalType &type, const GeometryStatsData &stats,
-	                        GeoParquetVersion version);
+	void AddGeoParquetStats(ClientContext &context, const string &column_name, const LogicalType &type,
+	                        const GeometryStatsData &stats, GeoParquetVersion version);
 	void Write(duckdb_parquet::FileMetaData &file_meta_data);
 
 	// Try to read GeoParquet metadata. Returns nullptr if not found, invalid or the required spatial extension is not
 	// available.
 	static unique_ptr<GeoParquetFileMetadata> TryRead(const duckdb_parquet::FileMetaData &file_meta_data,
-	                                                  const ClientContext &context);
+	                                                  ClientContext &context);
 	const unordered_map<string, GeoParquetColumnMetadata> &GetColumnMeta() const;
 	optional_ptr<const GeoParquetColumnMetadata> GetColumnMeta(const string &column_name) const;
 

src/duckdb/extension/parquet/include/parquet_reader.hpp

@@ -109,7 +109,6 @@ struct ParquetOptions {
 	bool binary_as_string = false;
 	bool file_row_number = false;
 	shared_ptr<ParquetEncryptionConfig> encryption_config;
-	bool debug_use_openssl = true;
 
 	vector<ParquetColumnDefinition> schema;
 	idx_t explicit_cardinality = 0;

src/duckdb/extension/parquet/include/parquet_writer.hpp

@@ -110,8 +110,8 @@ class ParquetWriter {
 	              ShreddingType shredding_types, const vector<pair<string, string>> &kv_metadata,
 	              shared_ptr<ParquetEncryptionConfig> encryption_config, optional_idx dictionary_size_limit,
 	              idx_t string_dictionary_page_size_limit, bool enable_bloom_filters,
-	              double bloom_filter_false_positive_ratio, int64_t compression_level, bool debug_use_openssl,
-	              ParquetVersion parquet_version, GeoParquetVersion geoparquet_version);
+	              double bloom_filter_false_positive_ratio, int64_t compression_level, ParquetVersion parquet_version,
+	              GeoParquetVersion geoparquet_version);
 	~ParquetWriter();
 
 public:
@@ -123,7 +123,7 @@ class ParquetWriter {
 
 	static duckdb_parquet::Type::type DuckDBTypeToParquetType(const LogicalType &duckdb_type);
 	static void SetSchemaProperties(const LogicalType &duckdb_type, duckdb_parquet::SchemaElement &schema_ele,
-	                                bool allow_geometry);
+	                                bool allow_geometry, ClientContext &context);
 
 	ClientContext &GetContext() {
 		return context;
@@ -207,7 +207,6 @@ class ParquetWriter {
 	bool enable_bloom_filters;
 	double bloom_filter_false_positive_ratio;
 	int64_t compression_level;
-	bool debug_use_openssl;
 	shared_ptr<EncryptionUtil> encryption_util;
 	ParquetVersion parquet_version;
 	GeoParquetVersion geoparquet_version;

src/duckdb/extension/parquet/parquet_crypto.cpp

@@ -163,8 +163,11 @@ class EncryptionTransport : public TTransport {
 public:
 	EncryptionTransport(TProtocol &prot_p, const string &key, const EncryptionUtil &encryption_util_p)
 	    : prot(prot_p), trans(*prot.getTransport()),
-	      aes(encryption_util_p.CreateEncryptionState(EncryptionTypes::GCM, key.size())),
 	      allocator(Allocator::DefaultAllocator(), ParquetCrypto::CRYPTO_BLOCK_SIZE) {
+		auto metadata = make_uniq<EncryptionStateMetadata>(EncryptionTypes::GCM, key.size(),
+		                                                   EncryptionTypes::EncryptionVersion::NONE);
+		aes = encryption_util_p.CreateEncryptionState(std::move(metadata));
+
 		Initialize(key);
 	}
 
@@ -190,8 +193,8 @@ class EncryptionTransport : public TTransport {
 		const uint32_t total_length = ParquetCrypto::NONCE_BYTES + ciphertext_length + ParquetCrypto::TAG_BYTES;
 
 		trans.write(const_data_ptr_cast(&total_length), ParquetCrypto::LENGTH_BYTES);
-		// Write nonce at beginning of encrypted chunk
-		trans.write(nonce, ParquetCrypto::NONCE_BYTES);
+		// Write nonce at the start of encrypted chunk
+		trans.write(nonce.data(), ParquetCrypto::NONCE_BYTES);
 
 		data_t aes_buffer[ParquetCrypto::CRYPTO_BLOCK_SIZE];
 		auto current = allocator.GetTail();
@@ -220,10 +223,9 @@ class EncryptionTransport : public TTransport {
 private:
 	void Initialize(const string &key) {
 		// Generate Nonce
-		aes->GenerateRandomData(nonce, ParquetCrypto::NONCE_BYTES);
+		aes->GenerateRandomData(nonce.data(), nonce.size());
 		// Initialize Encryption
-		aes->InitializeEncryption(nonce, ParquetCrypto::NONCE_BYTES, reinterpret_cast<const_data_ptr_t>(key.data()),
-		                          key.size());
+		aes->InitializeEncryption(nonce, reinterpret_cast<const_data_ptr_t>(key.data()));
 	}
 
 private:
@@ -235,7 +237,7 @@ class EncryptionTransport : public TTransport {
 	shared_ptr<EncryptionState> aes;
 
 	//! Nonce created by Initialize()
-	data_t nonce[ParquetCrypto::NONCE_BYTES];
+	EncryptionNonce nonce;
 
 	//! Arena Allocator to fully materialize in memory before encrypting
 	ArenaAllocator allocator;
@@ -246,9 +248,10 @@ class DecryptionTransport : public TTransport {
 public:
 	DecryptionTransport(TProtocol &prot_p, const string &key, const EncryptionUtil &encryption_util_p,
 	                    const CryptoMetaData &crypto_meta_data)
-	    : prot(prot_p), trans(*prot.getTransport()),
-	      aes(encryption_util_p.CreateEncryptionState(EncryptionTypes::GCM, key.size())), read_buffer_size(0),
-	      read_buffer_offset(0) {
+	    : prot(prot_p), trans(*prot.getTransport()), read_buffer_size(0), read_buffer_offset(0) {
+		auto metadata = make_uniq<EncryptionStateMetadata>(EncryptionTypes::GCM, key.size(),
+		                                                   EncryptionTypes::EncryptionVersion::NONE);
+		aes = encryption_util_p.CreateEncryptionState(std::move(metadata));
 		Initialize(key, crypto_meta_data);
 	}
 
@@ -306,16 +309,15 @@ class DecryptionTransport : public TTransport {
 		total_bytes = Load<uint32_t>(length_buf);
 		transport_remaining = total_bytes;
 		// Read nonce and initialize AES
-		transport_remaining -= trans.read(nonce, ParquetCrypto::NONCE_BYTES);
+		transport_remaining -= trans.read(nonce.data(), nonce.total_size());
 		// check whether context is initialized
 		if (!crypto_meta_data.IsEmpty()) {
-			aes->InitializeDecryption(nonce, ParquetCrypto::NONCE_BYTES, reinterpret_cast<const_data_ptr_t>(key.data()),
-			                          key.size(), crypto_meta_data.additional_authenticated_data->data(),
+			aes->InitializeDecryption(nonce, reinterpret_cast<const_data_ptr_t>(key.data()),
+			                          crypto_meta_data.additional_authenticated_data->data(),
 			                          crypto_meta_data.additional_authenticated_data->size());
 			crypto_meta_data.additional_authenticated_data->Rewind();
 		} else {
-			aes->InitializeDecryption(nonce, ParquetCrypto::NONCE_BYTES, reinterpret_cast<const_data_ptr_t>(key.data()),
-			                          key.size());
+			aes->InitializeDecryption(nonce, reinterpret_cast<const_data_ptr_t>(key.data()));
 		}
 	}
 
@@ -353,7 +355,7 @@ class DecryptionTransport : public TTransport {
 	uint32_t total_bytes;
 	uint32_t transport_remaining;
 	//! Nonce read by Initialize()
-	data_t nonce[ParquetCrypto::NONCE_BYTES];
+	EncryptionNonce nonce;
 };
 
 class SimpleReadTransport : public TTransport {

src/duckdb/extension/parquet/parquet_extension.cpp

@@ -67,9 +67,8 @@ struct ParquetWriteBindData : public TableFunctionData {
 	idx_t row_group_size = DEFAULT_ROW_GROUP_SIZE;
 	idx_t row_group_size_bytes = NumericLimits<idx_t>::Maximum();
 
-	//! How/Whether to encrypt the data
+	//! Encryption configuration
 	shared_ptr<ParquetEncryptionConfig> encryption_config;
-	bool debug_use_openssl = true;
 
 	//! After how many distinct values should we abandon dictionary compression and bloom filters?
 	//! Defaults to 1/5th of the row group size if unset (in templated_column_writer.hpp)
@@ -131,8 +130,8 @@ static void ParquetListCopyOptions(ClientContext &context, CopyOptionsInput &inp
 	copy_options["dictionary_size_limit"] = CopyOption(LogicalType::BIGINT, CopyOptionMode::WRITE_ONLY);
 	copy_options["string_dictionary_page_size_limit"] = CopyOption(LogicalType::UBIGINT, CopyOptionMode::WRITE_ONLY);
 	copy_options["bloom_filter_false_positive_ratio"] = CopyOption(LogicalType::DOUBLE, CopyOptionMode::WRITE_ONLY);
-	copy_options["write_bloom_filter"] = CopyOption(LogicalType::BOOLEAN, CopyOptionMode::WRITE_ONLY);
 	copy_options["debug_use_openssl"] = CopyOption(LogicalType::BOOLEAN, CopyOptionMode::READ_WRITE);
+	copy_options["write_bloom_filter"] = CopyOption(LogicalType::BOOLEAN, CopyOptionMode::WRITE_ONLY);
 	copy_options["compression_level"] = CopyOption(LogicalType::BIGINT, CopyOptionMode::WRITE_ONLY);
 	copy_options["parquet_version"] = CopyOption(LogicalType::VARCHAR, CopyOptionMode::WRITE_ONLY);
 	copy_options["binary_as_string"] = CopyOption(LogicalType::BOOLEAN, CopyOptionMode::READ_ONLY);
@@ -272,7 +271,7 @@ static unique_ptr<FunctionData> ParquetWriteBind(ClientContext &context, CopyFun
 			}
 		} else if (loption == "encryption_config") {
 			bind_data->encryption_config = ParquetEncryptionConfig::Create(context, option.second[0]);
-		} else if (loption == "dictionary_compression_ratio_threshold") {
+		} else if (loption == "dictionary_compression_ratio_threshold" || loption == "debug_use_openssl") {
 			// deprecated, ignore setting
 		} else if (loption == "dictionary_size_limit") {
 			auto val = option.second[0].GetValue<int64_t>();
@@ -296,15 +295,6 @@ static unique_ptr<FunctionData> ParquetWriteBind(ClientContext &context, CopyFun
 				throw BinderException("bloom_filter_false_positive_ratio must be greater than 0");
 			}
 			bind_data->bloom_filter_false_positive_ratio = val;
-		} else if (loption == "debug_use_openssl") {
-			auto val = StringUtil::Lower(option.second[0].GetValue<std::string>());
-			if (val == "false") {
-				bind_data->debug_use_openssl = false;
-			} else if (val == "true") {
-				bind_data->debug_use_openssl = true;
-			} else {
-				throw BinderException("Expected debug_use_openssl to be a BOOLEAN");
-			}
 		} else if (loption == "compression_level") {
 			const auto val = option.second[0].GetValue<int64_t>();
 			if (val < ZStdFileSystem::MinimumCompressionLevel() || val > ZStdFileSystem::MaximumCompressionLevel()) {
@@ -367,8 +357,8 @@ static unique_ptr<GlobalFunctionData> ParquetWriteInitializeGlobal(ClientContext
 	    parquet_bind.field_ids.Copy(), parquet_bind.shredding_types.Copy(), parquet_bind.kv_metadata,
 	    parquet_bind.encryption_config, parquet_bind.dictionary_size_limit,
 	    parquet_bind.string_dictionary_page_size_limit, parquet_bind.enable_bloom_filters,
-	    parquet_bind.bloom_filter_false_positive_ratio, parquet_bind.compression_level, parquet_bind.debug_use_openssl,
-	    parquet_bind.parquet_version, parquet_bind.geoparquet_version);
+	    parquet_bind.bloom_filter_false_positive_ratio, parquet_bind.compression_level, parquet_bind.parquet_version,
+	    parquet_bind.geoparquet_version);
 	return std::move(global_state);
 }
 
@@ -611,8 +601,6 @@ static void ParquetCopySerialize(Serializer &serializer, const FunctionData &bin
 	serializer.WritePropertyWithDefault(109, "compression_level", compression_level);
 	serializer.WritePropertyWithDefault(110, "row_groups_per_file", bind_data.row_groups_per_file,
 	                                    default_value.row_groups_per_file);
-	serializer.WritePropertyWithDefault(111, "debug_use_openssl", bind_data.debug_use_openssl,
-	                                    default_value.debug_use_openssl);
 	serializer.WritePropertyWithDefault(112, "dictionary_size_limit", bind_data.dictionary_size_limit,
 	                                    default_value.dictionary_size_limit);
 	serializer.WritePropertyWithDefault(113, "bloom_filter_false_positive_ratio",
@@ -648,8 +636,7 @@ static unique_ptr<FunctionData> ParquetCopyDeserialize(Deserializer &deserialize
 	ParquetWriteBindData default_value;
 	data->row_groups_per_file = deserializer.ReadPropertyWithExplicitDefault<optional_idx>(
 	    110, "row_groups_per_file", default_value.row_groups_per_file);
-	data->debug_use_openssl =
-	    deserializer.ReadPropertyWithExplicitDefault<bool>(111, "debug_use_openssl", default_value.debug_use_openssl);
+	deserializer.ReadDeletedProperty<bool>(111, "debug_use_openssl");
 	data->dictionary_size_limit =
 	    deserializer.ReadPropertyWithExplicitDefault<optional_idx>(112, "dictionary_size_limit", optional_idx());
 	data->bloom_filter_false_positive_ratio = deserializer.ReadPropertyWithExplicitDefault<double>(
@@ -660,8 +647,7 @@ static unique_ptr<FunctionData> ParquetCopyDeserialize(Deserializer &deserialize
 	    115, "string_dictionary_page_size_limit", default_value.string_dictionary_page_size_limit);
 	data->geoparquet_version =
 	    deserializer.ReadPropertyWithExplicitDefault(116, "geoparquet_version", default_value.geoparquet_version);
-	data->shredding_types =
-	    deserializer.ReadPropertyWithExplicitDefault<ShreddingType>(117, "shredding_types", ShreddingType());
+	data->shredding_types = deserializer.ReadProperty<ShreddingType>(117, "shredding_types");
 
 	return std::move(data);
 }