Skip to content

Commit 6091ecd

Browse files
duckdblabs-botgithub-actions[bot]
duckdblabs-bot
authored and
github-actions[bot]
committed
Update vendored DuckDB sources to 783f08ffd8
1 parent f08e565 commit 6091ecd

14 files changed

Lines changed: 217 additions & 74 deletions

File tree

‎src/duckdb/src/catalog/default/default_table_functions.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,7 @@ FROM histogram_values(source, col_name, bin_count := bin_count, technique := tec
6969
{DEFAULT_SCHEMA, "duckdb_logs_parsed", {"log_type"}, {}, R"(
7070
SELECT * EXCLUDE (message), UNNEST(parse_duckdb_log_message(log_type, message))
7171
FROM duckdb_logs(denormalized_table=1)
72-
WHERE type = log_type
72+
WHERE type ILIKE log_type
7373
)"},
7474
{nullptr, nullptr, {nullptr}, {{nullptr, nullptr}}, nullptr}
7575
};

‎src/duckdb/src/function/scalar/system/parse_log_message.cpp‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -77,8 +77,10 @@ void ParseLogMessageFunction(DataChunk &args, ExpressionState &state, Vector &re
7777
} // namespace
7878

7979
ScalarFunction ParseLogMessage::GetFunction() {
80-
return ScalarFunction({LogicalType::VARCHAR, LogicalType::VARCHAR}, LogicalType::ANY, ParseLogMessageFunction,
81-
ParseLogMessageBind, nullptr, nullptr, nullptr, LogicalType(LogicalTypeId::INVALID));
80+
auto fun = ScalarFunction({LogicalType::VARCHAR, LogicalType::VARCHAR}, LogicalType::ANY, ParseLogMessageFunction,
81+
ParseLogMessageBind, nullptr, nullptr, nullptr, LogicalType(LogicalTypeId::INVALID));
82+
fun.errors = FunctionErrors::CAN_THROW_RUNTIME_ERROR;
83+
return fun;
8284
}
8385

8486
} // namespace duckdb

‎src/duckdb/src/function/table/version/pragma_version.cpp‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
#ifndef DUCKDB_PATCH_VERSION
2-
#define DUCKDB_PATCH_VERSION "2-dev266"
2+
#define DUCKDB_PATCH_VERSION "2-dev279"
33
#endif
44
#ifndef DUCKDB_MINOR_VERSION
55
#define DUCKDB_MINOR_VERSION 4
@@ -8,10 +8,10 @@
88
#define DUCKDB_MAJOR_VERSION 1
99
#endif
1010
#ifndef DUCKDB_VERSION
11-
#define DUCKDB_VERSION "v1.4.2-dev266"
11+
#define DUCKDB_VERSION "v1.4.2-dev279"
1212
#endif
1313
#ifndef DUCKDB_SOURCE_ID
14-
#define DUCKDB_SOURCE_ID "2d69f075ee"
14+
#define DUCKDB_SOURCE_ID "783f08ffd8"
1515
#endif
1616
#include "duckdb/function/table/system_functions.hpp"
1717
#include "duckdb/main/database.hpp"

‎src/duckdb/src/include/duckdb/main/settings.hpp‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -337,6 +337,15 @@ struct DebugSkipCheckpointOnCommitSetting {
337337
static constexpr SetScope DefaultScope = SetScope::GLOBAL;
338338
};
339339

340+
struct DebugVerifyBlocksSetting {
341+
using RETURN_TYPE = bool;
342+
static constexpr const char *Name = "debug_verify_blocks";
343+
static constexpr const char *Description = "DEBUG SETTING: verify block metadata during checkpointing";
344+
static constexpr const char *InputType = "BOOLEAN";
345+
static constexpr const char *DefaultValue = "false";
346+
static constexpr SetScope DefaultScope = SetScope::GLOBAL;
347+
};
348+
340349
struct DebugVerifyVectorSetting {
341350
using RETURN_TYPE = DebugVectorVerification;
342351
static constexpr const char *Name = "debug_verify_vector";

‎src/duckdb/src/include/duckdb/storage/block.hpp‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,11 @@ struct MetaBlockPointer {
6565
return block_pointer == rhs.block_pointer && offset == rhs.offset;
6666
}
6767

68+
friend std::ostream &operator<<(std::ostream &os, const MetaBlockPointer &obj) {
69+
return os << "{block_id: " << obj.GetBlockId() << " index: " << obj.GetBlockIndex() << " offset: " << obj.offset
70+
<< "}";
71+
}
72+
6873
void Serialize(Serializer &serializer) const;
6974
static MetaBlockPointer Deserialize(Deserializer &source);
7075
};

‎src/duckdb/src/include/duckdb/storage/metadata/metadata_manager.hpp‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,8 @@ class MetadataManager {
7777
//! Flush all blocks to disk
7878
void Flush();
7979

80+
bool BlockHasBeenCleared(const MetaBlockPointer &ptr);
81+
8082
void MarkBlocksAsModified();
8183
void ClearModifiedBlocks(const vector<MetaBlockPointer> &pointers);
8284

‎src/duckdb/src/include/duckdb/storage/table/row_group.hpp‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -65,7 +65,8 @@ struct RowGroupWriteInfo {
6565
struct RowGroupWriteData {
6666
vector<unique_ptr<ColumnCheckpointState>> states;
6767
vector<BaseStatistics> statistics;
68-
vector<MetaBlockPointer> existing_pointers;
68+
bool reuse_existing_metadata_blocks = false;
69+
vector<idx_t> existing_extra_metadata_blocks;
6970
};
7071

7172
class RowGroup : public SegmentBase<RowGroup> {
@@ -94,7 +95,10 @@ class RowGroup : public SegmentBase<RowGroup> {
9495
return collection.get();
9596
}
9697
//! Returns the list of meta block pointers used by the columns
97-
vector<MetaBlockPointer> GetColumnPointers();
98+
vector<idx_t> GetOrComputeExtraMetadataBlocks(bool force_compute = false);
99+
100+
const vector<MetaBlockPointer> &GetColumnStartPointers() const;
101+
98102
//! Returns the list of meta block pointers used by the deletes
99103
const vector<MetaBlockPointer> &GetDeletesPointers() const {
100104
return deletes_pointers;

‎src/duckdb/src/main/config.cpp‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,7 @@ static const ConfigurationOption internal_options[] = {
8686
DUCKDB_LOCAL(DebugForceExternalSetting),
8787
DUCKDB_SETTING(DebugForceNoCrossProductSetting),
8888
DUCKDB_SETTING(DebugSkipCheckpointOnCommitSetting),
89+
DUCKDB_SETTING(DebugVerifyBlocksSetting),
8990
DUCKDB_SETTING_CALLBACK(DebugVerifyVectorSetting),
9091
DUCKDB_SETTING_CALLBACK(DebugWindowModeSetting),
9192
DUCKDB_GLOBAL(DefaultBlockSizeSetting),
@@ -177,12 +178,12 @@ static const ConfigurationOption internal_options[] = {
177178
DUCKDB_GLOBAL(ZstdMinStringLengthSetting),
178179
FINAL_SETTING};
179180

180-
static const ConfigurationAlias setting_aliases[] = {DUCKDB_SETTING_ALIAS("memory_limit", 83),
181-
DUCKDB_SETTING_ALIAS("null_order", 33),
182-
DUCKDB_SETTING_ALIAS("profiling_output", 102),
183-
DUCKDB_SETTING_ALIAS("user", 116),
181+
static const ConfigurationAlias setting_aliases[] = {DUCKDB_SETTING_ALIAS("memory_limit", 84),
182+
DUCKDB_SETTING_ALIAS("null_order", 34),
183+
DUCKDB_SETTING_ALIAS("profiling_output", 103),
184+
DUCKDB_SETTING_ALIAS("user", 117),
184185
DUCKDB_SETTING_ALIAS("wal_autocheckpoint", 20),
185-
DUCKDB_SETTING_ALIAS("worker_threads", 115),
186+
DUCKDB_SETTING_ALIAS("worker_threads", 116),
186187
FINAL_ALIAS};
187188

188189
vector<ConfigurationOption> DBConfig::GetOptions() {

‎src/duckdb/src/storage/checkpoint/table_data_writer.cpp‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
#include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp"
55
#include "duckdb/common/serializer/binary_serializer.hpp"
66
#include "duckdb/main/database.hpp"
7+
#include "duckdb/main/settings.hpp"
78
#include "duckdb/parallel/task_scheduler.hpp"
89
#include "duckdb/storage/table/column_checkpoint_state.hpp"
910
#include "duckdb/storage/table/table_statistics.hpp"
@@ -119,15 +120,16 @@ void SingleFileTableDataWriter::FinalizeTable(const TableStatistics &global_stat
119120
}
120121
auto index_storage_infos = info.GetIndexes().SerializeToDisk(context, options);
121122

122-
#ifdef DUCKDB_BLOCK_VERIFICATION
123-
for (auto &entry : index_storage_infos) {
124-
for (auto &allocator : entry.allocator_infos) {
125-
for (auto &block : allocator.block_pointers) {
126-
checkpoint_manager.verify_block_usage_count[block.block_id]++;
123+
auto debug_verify_blocks = DBConfig::GetSetting<DebugVerifyBlocksSetting>(GetDatabase());
124+
if (debug_verify_blocks) {
125+
for (auto &entry : index_storage_infos) {
126+
for (auto &allocator : entry.allocator_infos) {
127+
for (auto &block : allocator.block_pointers) {
128+
checkpoint_manager.verify_block_usage_count[block.block_id]++;
129+
}
127130
}
128131
}
129132
}
130-
#endif
131133

132134
// write empty block pointers for forwards compatibility
133135
vector<BlockPointer> compat_block_pointers;

‎src/duckdb/src/storage/checkpoint_manager.cpp‎

Lines changed: 24 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -214,33 +214,35 @@ void SingleFileCheckpointWriter::CreateCheckpoint() {
214214
header.vector_size = STANDARD_VECTOR_SIZE;
215215
block_manager.WriteHeader(context, header);
216216

217-
#ifdef DUCKDB_BLOCK_VERIFICATION
218-
// extend verify_block_usage_count
219-
auto metadata_info = storage_manager.GetMetadataInfo();
220-
for (auto &info : metadata_info) {
221-
verify_block_usage_count[info.block_id]++;
222-
}
223-
for (auto &entry_ref : catalog_entries) {
224-
auto &entry = entry_ref.get();
225-
if (entry.type == CatalogType::TABLE_ENTRY) {
226-
auto &table = entry.Cast<DuckTableEntry>();
227-
auto &storage = table.GetStorage();
228-
auto segment_info = storage.GetColumnSegmentInfo();
229-
for (auto &segment : segment_info) {
230-
verify_block_usage_count[segment.block_id]++;
231-
if (StringUtil::Contains(segment.segment_info, "Overflow String Block Ids: ")) {
232-
auto overflow_blocks = StringUtil::Replace(segment.segment_info, "Overflow String Block Ids: ", "");
233-
auto splits = StringUtil::Split(overflow_blocks, ", ");
234-
for (auto &split : splits) {
235-
auto overflow_block_id = std::stoll(split);
236-
verify_block_usage_count[overflow_block_id]++;
217+
auto debug_verify_blocks = DBConfig::GetSetting<DebugVerifyBlocksSetting>(db.GetDatabase());
218+
if (debug_verify_blocks) {
219+
// extend verify_block_usage_count
220+
auto metadata_info = storage_manager.GetMetadataInfo();
221+
for (auto &info : metadata_info) {
222+
verify_block_usage_count[info.block_id]++;
223+
}
224+
for (auto &entry_ref : catalog_entries) {
225+
auto &entry = entry_ref.get();
226+
if (entry.type == CatalogType::TABLE_ENTRY) {
227+
auto &table = entry.Cast<DuckTableEntry>();
228+
auto &storage = table.GetStorage();
229+
auto segment_info = storage.GetColumnSegmentInfo();
230+
for (auto &segment : segment_info) {
231+
verify_block_usage_count[segment.block_id]++;
232+
if (StringUtil::Contains(segment.segment_info, "Overflow String Block Ids: ")) {
233+
auto overflow_blocks =
234+
StringUtil::Replace(segment.segment_info, "Overflow String Block Ids: ", "");
235+
auto splits = StringUtil::Split(overflow_blocks, ", ");
236+
for (auto &split : splits) {
237+
auto overflow_block_id = std::stoll(split);
238+
verify_block_usage_count[overflow_block_id]++;
239+
}
237240
}
238241
}
239242
}
240243
}
244+
block_manager.VerifyBlocks(verify_block_usage_count);
241245
}
242-
block_manager.VerifyBlocks(verify_block_usage_count);
243-
#endif
244246

245247
if (debug_checkpoint_abort == CheckpointAbort::DEBUG_ABORT_BEFORE_TRUNCATE) {
246248
throw FatalException("Checkpoint aborted before truncate because of PRAGMA checkpoint_abort flag");

0 commit comments

Comments
 (0)