From 45432071c0b3e726689aac3cac390d27b4163a89 Mon Sep 17 00:00:00 2001 From: yiguolei Date: Thu, 26 Mar 2026 11:12:47 +0800 Subject: [PATCH 1/4] [refactor](rowcursor) using field in rowcursor to skip many string converters (#61300) Issue Number: close #xxx Related PR: #xxx Problem Summary: None - Test - [ ] Regression test - [ ] Unit Test - [ ] Manual test (add detailed scripts or steps below) - [ ] No need to test or manual test. Explain why: - [ ] This is a refactor/code format and no logic has been changed. - [ ] Previous test can cover this change. - [ ] No code files have been changed. - [ ] Other reason - Behavior changed: - [ ] No. - [ ] Yes. - Does this need documentation? - [ ] No. - [ ] Yes. - [ ] Confirm the release note - [ ] Confirm test cases - [ ] Confirm document - [ ] Add branch pick label --- be/src/core/field.cpp | 94 +- be/src/core/field.h | 5 + be/src/exec/operator/olap_scan_operator.cpp | 35 +- be/src/exec/operator/scan_operator.cpp | 66 +- be/src/exec/scan/olap_scanner.cpp | 7 +- be/src/exprs/function/cast/cast_to_string.h | 2 +- be/src/service/point_query_executor.cpp | 44 +- be/src/storage/field.h | 89 -- be/src/storage/olap_scan_common.cpp | 17 + be/src/storage/olap_scan_common.h | 302 +++-- be/src/storage/olap_tuple.h | 80 +- be/src/storage/olap_utils.h | 63 +- be/src/storage/row_cursor.cpp | 396 +++--- be/src/storage/row_cursor.h | 174 +-- be/src/storage/row_cursor_cell.h | 36 - be/src/storage/schema.cpp | 37 - be/src/storage/schema.h | 40 +- be/src/storage/segment/segment_iterator.cpp | 19 +- be/src/storage/segment/segment_iterator.h | 53 +- be/src/storage/segment/segment_writer.cpp | 34 +- be/src/storage/segment/segment_writer.h | 3 +- be/src/storage/tablet/tablet_reader.cpp | 35 +- be/src/storage/tablet/tablet_reader.h | 18 +- be/src/storage/types.h | 473 ------- .../delete/delete_bitmap_calculator_test.cpp | 18 +- be/test/storage/olap_type_test.cpp | 79 +- be/test/storage/row_cursor_test.cpp | 1102 +++++++++++++---- .../segment/column_meta_accessor_test.cpp | 8 +- .../segment/external_col_meta_util_test.cpp | 8 +- .../segment/segment_corruption_test.cpp | 19 +- .../segment/segment_footer_cache_test.cpp | 8 +- be/test/storage/segment/test_segment_writer.h | 136 ++ be/test/storage/storage_types_test.cpp | 18 - .../apache/doris/qe/PointQueryExecutor.java | 41 +- gensrc/proto/internal_service.proto | 5 + 35 files changed, 1922 insertions(+), 1642 deletions(-) delete mode 100644 be/src/storage/row_cursor_cell.h create mode 100644 be/test/storage/segment/test_segment_writer.h diff --git a/be/src/core/field.cpp b/be/src/core/field.cpp index b93362a0e3996b..a2fae427350538 100644 --- a/be/src/core/field.cpp +++ b/be/src/core/field.cpp @@ -20,6 +20,7 @@ #include "core/field.h" +#include "common/compare.h" #include "core/accurate_comparison.h" #include "core/data_type/data_type_decimal.h" #include "core/data_type/define_primitive_type.h" @@ -678,6 +679,13 @@ std::strong_ordering Field::operator<=>(const Field& rhs) const { return type <=> rhs.type; } if (type != rhs.type) { + // String-family types (STRING, CHAR, VARCHAR) all store String internally + // and are inter-comparable. This arises when comparing RowCursor fields + // (which carry the declared column type) against Column::operator[] results + // (which always return TYPE_STRING for ColumnString). + if (is_string_type(type) && is_string_type(rhs.type)) { + return get() <=> rhs.get(); + } throw Exception(Status::FatalError("lhs type not equal with rhs, lhs={}, rhs={}", get_type_name(), rhs.get_type_name())); } @@ -724,17 +732,31 @@ std::strong_ordering Field::operator<=>(const Field& rhs) const { case PrimitiveType::TYPE_IPV4: return get() <=> rhs.get(); case PrimitiveType::TYPE_FLOAT: - return get() < rhs.get() ? std::strong_ordering::less - : get() == rhs.get() ? std::strong_ordering::equal - : std::strong_ordering::greater; + switch (Compare::compare(get(), rhs.get())) { + case -1: + return std::strong_ordering::less; + case 0: + return std::strong_ordering::equal; + case 1: + return std::strong_ordering::greater; + default: + LOG(FATAL) << "unexpected float compare result"; + } case PrimitiveType::TYPE_TIMEV2: return get() < rhs.get() ? std::strong_ordering::less : get() == rhs.get() ? std::strong_ordering::equal : std::strong_ordering::greater; case PrimitiveType::TYPE_DOUBLE: - return get() < rhs.get() ? std::strong_ordering::less - : get() == rhs.get() ? std::strong_ordering::equal - : std::strong_ordering::greater; + switch (Compare::compare(get(), rhs.get())) { + case -1: + return std::strong_ordering::less; + case 0: + return std::strong_ordering::equal; + case 1: + return std::strong_ordering::greater; + default: + LOG(FATAL) << "unexpected double compare result"; + } case PrimitiveType::TYPE_STRING: return get() <=> rhs.get(); case PrimitiveType::TYPE_CHAR: @@ -822,6 +844,66 @@ std::string_view Field::as_string_view() const { #undef MATCH_PRIMITIVE_TYPE +// Important!!! This method is not accurate, for example, decimal to string, it uses scale == 0, because +// it do not know the actual scale of the decimal value. It is only used for debug printing, so it is fine. +std::string Field::to_debug_string(int scale) const { + if (is_null()) { + return "NULL"; + } + switch (type) { + case PrimitiveType::TYPE_BOOLEAN: + return get() ? "true" : "false"; + case PrimitiveType::TYPE_TINYINT: + return CastToString::from_number(get()); + case PrimitiveType::TYPE_SMALLINT: + return CastToString::from_number(get()); + case PrimitiveType::TYPE_INT: + return CastToString::from_number(get()); + case PrimitiveType::TYPE_BIGINT: + return CastToString::from_number(get()); + case PrimitiveType::TYPE_LARGEINT: + return CastToString::from_number(get()); + case PrimitiveType::TYPE_FLOAT: + return CastToString::from_number(get()); + case PrimitiveType::TYPE_DOUBLE: + return CastToString::from_number(get()); + case PrimitiveType::TYPE_STRING: + case PrimitiveType::TYPE_CHAR: + case PrimitiveType::TYPE_VARCHAR: + return get(); + case PrimitiveType::TYPE_VARBINARY: + return get(); + case PrimitiveType::TYPE_DATE: + return CastToString::from_date_or_datetime(get()); + case PrimitiveType::TYPE_DATETIME: + return CastToString::from_date_or_datetime(get()); + case PrimitiveType::TYPE_DATEV2: + return CastToString::from_datev2(get()); + case PrimitiveType::TYPE_DATETIMEV2: + return CastToString::from_datetimev2(get(), scale); + case PrimitiveType::TYPE_TIMESTAMPTZ: + return CastToString::from_timestamptz(get(), scale); + case PrimitiveType::TYPE_DECIMALV2: + return get().to_string(); + case PrimitiveType::TYPE_DECIMAL32: + return CastToString::from_decimal(get(), scale); + case PrimitiveType::TYPE_DECIMAL64: + return CastToString::from_decimal(get(), scale); + case PrimitiveType::TYPE_DECIMAL128I: + return CastToString::from_decimal(get(), scale); + case PrimitiveType::TYPE_DECIMAL256: + return CastToString::from_decimal(get(), scale); + case PrimitiveType::TYPE_IPV4: + return CastToString::from_ip(get()); + case PrimitiveType::TYPE_IPV6: + return CastToString::from_ip(get()); + default: + throw Exception(Status::FatalError("type not supported for to_debug_string, type={}", + get_type_name())); + __builtin_unreachable(); + } +} + #define DECLARE_FUNCTION(FUNC_NAME) \ template void Field::FUNC_NAME(typename PrimitiveTypeTraits::CppType && \ rhs); \ diff --git a/be/src/core/field.h b/be/src/core/field.h index 2c31172794174f..75ce72f7553d44 100644 --- a/be/src/core/field.h +++ b/be/src/core/field.h @@ -281,6 +281,11 @@ class Field { std::string_view as_string_view() const; + // Return a human-readable representation of the stored value for debugging. + // Unlike get_type_name() which returns the type, this prints the actual value. + // For decimal types, caller can provide scale for accurate formatting. + std::string to_debug_string(int scale) const; + private: std::aligned_union_t* scanners) { _output_column_ids.emplace(uid); } - // ranges constructed from scan keys + // Step 3: convert accumulated scan key pairs into OlapScanRange objects. + // Each OlapScanRange carries real begin/end OlapTuples with has_lower_bound = true. RETURN_IF_ERROR(_scan_keys.get_key_range(&_cond_ranges)); - // if we can't get ranges from conditions, we give it a total range + // If no key predicates were pushed down, _cond_ranges is empty. + // Create a single default-constructed OlapScanRange (has_lower_bound = false) + // to represent a full table scan. Consumers detect this and skip pushing + // key range to the tablet reader. if (_cond_ranges.empty()) { _cond_ranges.emplace_back(new doris::OlapScanRange()); } @@ -492,10 +496,11 @@ Status OlapScanLocalState::_init_scanners(std::list* scanners) { if (enable_parallel_scan && !p._should_run_serial && p._push_down_agg_type == TPushAggOp::NONE && (_storage_no_merge() || p._olap_scan_node.is_preaggregation)) { + // Filter out the "full scan" placeholder range (has_lower_bound == false) + // so that only ranges with real key bounds are forwarded to the parallel scanner. std::vector key_ranges; for (auto& range : _cond_ranges) { - if (range->begin_scan_range.size() == 1 && - range->begin_scan_range.get_value(0) == NEGATIVE_INFINITY) { + if (!range->has_lower_bound) { continue; } key_ranges.emplace_back(range.get()); @@ -878,6 +883,28 @@ inline std::string push_down_agg_to_string(const TPushAggOp::type& op) { } } +/// Step 2 of the scan-key generation pipeline. +/// +/// Iterate key columns in schema order; for each one, look up its ColumnValueRange +/// from _slot_id_to_value_range (populated by _normalize_conjuncts) and call +/// _scan_keys.extend_scan_key() to grow the multi-column prefix key set. +/// +/// Example – table t(k1 INT, k2 INT, v INT), key columns = (k1, k2): +/// Input ColumnValueRanges: +/// k1: fixed_values = {1, 2} +/// k2: fixed_values = {10} +/// After extend_scan_key(k1): +/// _begin_scan_keys = [(1), (2)] _end_scan_keys = [(1), (2)] +/// After extend_scan_key(k2): +/// _begin_scan_keys = [(1,10), (2,10)] _end_scan_keys = [(1,10), (2,10)] +/// +/// Loop terminates when: +/// - A key column has no predicate (break) +/// - A range column was appended (_has_range_value, cannot extend further) +/// - The ColumnValueRange is provably empty (eos) +/// - The fixed-value set exceeds max_scan_key_num (should_break or fall back to range) +/// +/// At the end, _scan_keys.get_key_range() converts these into OlapScanRange objects. Status OlapScanLocalState::_build_key_ranges_and_filters() { auto& p = _parent->cast(); if (p._push_down_agg_type == TPushAggOp::NONE || diff --git a/be/src/exec/operator/scan_operator.cpp b/be/src/exec/operator/scan_operator.cpp index fd1e5bbd5f7db7..111cae9af0196e 100644 --- a/be/src/exec/operator/scan_operator.cpp +++ b/be/src/exec/operator/scan_operator.cpp @@ -229,30 +229,29 @@ static void init_slot_value_range( slot_id_to_value_range[slot->id()] = std::move(range); \ break; \ } -#define APPLY_FOR_PRIMITIVE_TYPE(M) \ - M(TINYINT) \ - M(SMALLINT) \ - M(INT) \ - M(BIGINT) \ - M(LARGEINT) \ - M(FLOAT) \ - M(DOUBLE) \ - M(CHAR) \ - M(DATE) \ - M(DATETIME) \ - M(DATEV2) \ - M(DATETIMEV2) \ - M(TIMESTAMPTZ) \ - M(VARCHAR) \ - M(STRING) \ - M(HLL) \ - M(DECIMAL32) \ - M(DECIMAL64) \ - M(DECIMAL128I) \ - M(DECIMAL256) \ - M(DECIMALV2) \ - M(BOOLEAN) \ - M(IPV4) \ +#define APPLY_FOR_SCALAR_TYPE(M) \ + M(TINYINT) \ + M(SMALLINT) \ + M(INT) \ + M(BIGINT) \ + M(LARGEINT) \ + M(FLOAT) \ + M(DOUBLE) \ + M(CHAR) \ + M(DATE) \ + M(DATETIME) \ + M(DATEV2) \ + M(DATETIMEV2) \ + M(TIMESTAMPTZ) \ + M(VARCHAR) \ + M(STRING) \ + M(DECIMAL32) \ + M(DECIMAL64) \ + M(DECIMAL128I) \ + M(DECIMAL256) \ + M(DECIMALV2) \ + M(BOOLEAN) \ + M(IPV4) \ M(IPV6) APPLY_FOR_PRIMITIVE_TYPE(M) #undef M @@ -263,6 +262,21 @@ static void init_slot_value_range( } } +/// Step 1 of the scan-key generation pipeline. +/// +/// Parse SQL WHERE conjuncts into per-column ColumnValueRange objects stored in +/// _slot_id_to_value_range. Each ColumnValueRange captures all constraints on +/// one column (fixed values from IN / =, or min/max bounds from < / <= / > / >=). +/// +/// Example – "WHERE k1 IN (1, 2) AND k2 >= 5 AND k2 < 10 AND v > 100": +/// => ColumnValueRange: fixed_values = {1, 2} +/// => ColumnValueRange: scope [5, 10) (low=5 >=, high=10 <) +/// => ColumnValueRange: scope (100, MAX] (low=100 >, high=MAX <=) +/// The k1/k2 ranges will later become scan keys (since they're key columns); +/// v's range stays as a residual predicate / olap filter. +/// +/// After this step, _build_key_ranges_and_filters() picks up the key-column +/// ColumnValueRanges and feeds them to OlapScanKeys::extend_scan_key(). template Status ScanLocalState::_normalize_conjuncts(RuntimeState* state) { auto& p = _parent->cast(); @@ -937,10 +951,6 @@ Status ScanLocalState::_change_value_range(bool is_equal_op, (PrimitiveType == TYPE_DATEV2) || (PrimitiveType == TYPE_TIMESTAMPTZ) || (PrimitiveType == TYPE_DATETIME) || is_string_type(PrimitiveType)) { func(temp_range, to_olap_filter_type(fn_name), value.template get()); - } else if constexpr (PrimitiveType == TYPE_HLL) { - auto tmp = value.template get(); - func(temp_range, to_olap_filter_type(fn_name), - StringRef(reinterpret_cast(&tmp), sizeof(tmp))); } else { static_assert(always_false_v); } diff --git a/be/src/exec/scan/olap_scanner.cpp b/be/src/exec/scan/olap_scanner.cpp index 9cf652f9d50d1e..5df196ddf2637f 100644 --- a/be/src/exec/scan/olap_scanner.cpp +++ b/be/src/exec/scan/olap_scanner.cpp @@ -395,10 +395,11 @@ Status OlapScanner::_init_tablet_reader_params( tablet_schema->merge_dropped_columns(*del_pred->tablet_schema()); } - // Range + // Push key ranges to the tablet reader. + // Skip the "full scan" placeholder (has_lower_bound == false) — when no key + // predicates exist, start_key/end_key remain empty and the reader does a full scan. for (auto* key_range : key_ranges) { - if (key_range->begin_scan_range.size() == 1 && - key_range->begin_scan_range.get_value(0) == NEGATIVE_INFINITY) { + if (!key_range->has_lower_bound) { continue; } diff --git a/be/src/exprs/function/cast/cast_to_string.h b/be/src/exprs/function/cast/cast_to_string.h index 5c37fa7e30865e..c0c2a6b1ab4ebf 100644 --- a/be/src/exprs/function/cast/cast_to_string.h +++ b/be/src/exprs/function/cast/cast_to_string.h @@ -61,7 +61,7 @@ struct CastToString { static inline void push_datev2(const DateV2Value& from, BufferWritable& bw); static inline std::string from_datetimev2(const DateV2Value& from, - UInt32 scale = -1); + UInt32 scale); static inline std::string from_timestamptz(const TimestampTzValue& from, UInt32 scale, const cctz::time_zone* timezone = nullptr); static inline void push_datetimev2(const DateV2Value& from, UInt32 scale, diff --git a/be/src/service/point_query_executor.cpp b/be/src/service/point_query_executor.cpp index ccb9d3634362ab..faa52c6546780a 100644 --- a/be/src/service/point_query_executor.cpp +++ b/be/src/service/point_query_executor.cpp @@ -36,6 +36,7 @@ #include "common/cast_set.h" #include "common/consts.h" #include "common/status.h" +#include "core/data_type/data_type_factory.hpp" #include "core/data_type_serde/data_type_serde.h" #include "exec/sink/writer/vmysql_result_writer.h" #include "exprs/vexpr.h" @@ -48,7 +49,6 @@ #include "runtime/runtime_profile.h" #include "runtime/runtime_state.h" #include "runtime/thread_context.h" -#include "storage/olap_tuple.h" #include "storage/row_cursor.h" #include "storage/rowset/beta_rowset.h" #include "storage/rowset/rowset_fwd.h" @@ -387,21 +387,43 @@ void PointQueryExecutor::print_profile() { Status PointQueryExecutor::_init_keys(const PTabletKeyLookupRequest* request) { SCOPED_TIMER(&_profile_metrics.init_key_ns); + const auto& schema = _tablet->tablet_schema(); + // Point query is only supported on merge-on-write unique key tables. + DCHECK(schema->keys_type() == UNIQUE_KEYS && _tablet->enable_unique_key_merge_on_write()); + if (schema->keys_type() != UNIQUE_KEYS || !_tablet->enable_unique_key_merge_on_write()) { + return Status::InvalidArgument( + "Point query is only supported on merge-on-write unique key tables, " + "tablet_id={}", + _tablet->tablet_id()); + } // 1. get primary key from conditions - std::vector olap_tuples; - olap_tuples.resize(request->key_tuples().size()); + _row_read_ctxs.resize(request->key_tuples().size()); + // get row cursor and encode keys for (int i = 0; i < request->key_tuples().size(); ++i) { const KeyTuple& key_tuple = request->key_tuples(i); - for (const std::string& key_col : key_tuple.key_column_rep()) { - olap_tuples[i].add_value(key_col); + if (UNLIKELY(cast_set(key_tuple.key_column_literals_size()) != + schema->num_key_columns())) { + return Status::InvalidArgument( + "Key column count mismatch. expected={}, actual={}, tablet_id={}", + schema->num_key_columns(), key_tuple.key_column_literals_size(), + _tablet->tablet_id()); } - } - _row_read_ctxs.resize(olap_tuples.size()); - // get row cursor and encode keys - for (size_t i = 0; i < olap_tuples.size(); ++i) { RowCursor cursor; - RETURN_IF_ERROR(cursor.init_scan_key(_tablet->tablet_schema(), olap_tuples[i].values())); - RETURN_IF_ERROR(cursor.from_tuple(olap_tuples[i])); + std::vector key_fields; + key_fields.reserve(key_tuple.key_column_literals_size()); + for (int j = 0; j < key_tuple.key_column_literals_size(); ++j) { + const auto& literal_bytes = key_tuple.key_column_literals(j); + TExprNode expr_node; + auto len = cast_set(literal_bytes.size()); + RETURN_IF_ERROR( + deserialize_thrift_msg(reinterpret_cast(literal_bytes.data()), + &len, false, &expr_node)); + const auto& col = schema->column(j); + auto data_type = DataTypeFactory::instance().create_data_type( + col.type(), col.precision(), col.frac(), col.length()); + key_fields.push_back(data_type->get_field(expr_node)); + } + RETURN_IF_ERROR(cursor.init_scan_key(_tablet->tablet_schema(), std::move(key_fields))); cursor.encode_key_with_padding(&_row_read_ctxs[i]._primary_key, _tablet->tablet_schema()->num_key_columns(), true); } diff --git a/be/src/storage/field.h b/be/src/storage/field.h index 9eb2cd75027ef1..acfedf29d0baaa 100644 --- a/be/src/storage/field.h +++ b/be/src/storage/field.h @@ -27,7 +27,6 @@ #include "storage/key_coder.h" #include "storage/olap_common.h" #include "storage/olap_define.h" -#include "storage/row_cursor_cell.h" #include "storage/tablet/tablet_schema.h" #include "storage/types.h" #include "storage/utils.h" @@ -70,70 +69,12 @@ class StorageField { virtual void set_to_min(char* buf) const { return _type_info->set_to_min(buf); } - void set_long_text_buf(char** buf) { _long_text_buf = buf; } - - virtual size_t get_variable_len() const { return 0; } - virtual StorageField* clone() const { auto* local = new StorageField(_desc); this->clone(local); return local; } - // Only compare column content, without considering nullptr condition. - // RETURNS: - // 0 means equal, - // -1 means left less than right, - // 1 means left bigger than right - int compare(const void* left, const void* right) const { return _type_info->cmp(left, right); } - - // Compare two types of cell. - // This function differs compare in that this function compare cell which - // will consider the condition which cell may be nullptr. While compare only - // compare column content without considering nullptr condition. - // Only compare column content, without considering nullptr condition. - // RETURNS: - // 0 means equal, - // -1 means left less than right, - // 1 means left bigger than right - template - int compare_cell(const LhsCellType& lhs, const RhsCellType& rhs) const { - bool l_null = lhs.is_null(); - bool r_null = rhs.is_null(); - if (l_null != r_null) { - return l_null ? -1 : 1; - } - return l_null ? 0 : _type_info->cmp(lhs.cell_ptr(), rhs.cell_ptr()); - } - - // deep copy source cell' content to destination cell. - // For string type, this will allocate data form arena, - // and copy source's content. - template - void deep_copy(DstCellType* dst, const SrcCellType& src, Arena& arena) const { - bool is_null = src.is_null(); - dst->set_is_null(is_null); - if (is_null) { - return; - } - _type_info->deep_copy(dst->mutable_cell_ptr(), src.cell_ptr(), arena); - } - - // used by init scan key stored in string format - // value_string should end with '\0' - Status from_string(char* buf, const std::string& value_string, const int precision = 0, - const int scale = 0) const { - if (type() == FieldType::OLAP_FIELD_TYPE_STRING && !value_string.empty()) { - auto slice = reinterpret_cast(buf); - if (slice->size < value_string.size()) { - *_long_text_buf = static_cast(realloc(*_long_text_buf, value_string.size())); - slice->data = *_long_text_buf; - slice->size = value_string.size(); - } - } - return _type_info->from_string(buf, value_string, precision, scale); - } - FieldType type() const { return _type_info->type(); } const TypeInfo* type_info() const { return _type_info.get(); } bool is_nullable() const { return _is_nullable; } @@ -173,20 +114,6 @@ class StorageField { // its number of subfields is a variable, so the actual length of // a struct field is not fixed. size_t _length; - // Since the length of the STRING type cannot be determined, - // only dynamic memory can be used. Arena cannot realize realloc. - // The schema information is shared globally. Therefore, - // dynamic memory can only be managed in thread local mode. - // The memory will be created and released in rowcursor. - char** _long_text_buf = nullptr; - - char* allocate_string_value(Arena& arena) const { - char* type_value = arena.alloc(sizeof(Slice)); - auto slice = reinterpret_cast(type_value); - slice->size = _length; - slice->data = arena.alloc(slice->size); - return type_value; - } void clone(StorageField* other) const { other->_type_info = clone_type_info(this->_type_info.get()); @@ -225,36 +152,22 @@ class StorageField { class MapField : public StorageField { public: MapField(const TabletColumn& column) : StorageField(column) {} - - size_t get_variable_len() const override { return _length; } }; class StructField : public StorageField { public: StructField(const TabletColumn& column) : StorageField(column) {} - - size_t get_variable_len() const override { - size_t variable_len = _length; - for (size_t i = 0; i < get_sub_field_count(); i++) { - variable_len += get_sub_field(i)->get_variable_len(); - } - return variable_len; - } }; class ArrayField : public StorageField { public: ArrayField(const TabletColumn& column) : StorageField(column) {} - - size_t get_variable_len() const override { return _length; } }; class CharField : public StorageField { public: CharField(const TabletColumn& column) : StorageField(column) {} - size_t get_variable_len() const override { return _length; } - CharField* clone() const override { auto* local = new CharField(_desc); StorageField::clone(local); @@ -272,8 +185,6 @@ class VarcharField : public StorageField { public: VarcharField(const TabletColumn& column) : StorageField(column) {} - size_t get_variable_len() const override { return _length - OLAP_VARCHAR_MAX_BYTES; } - VarcharField* clone() const override { auto* local = new VarcharField(_desc); StorageField::clone(local); diff --git a/be/src/storage/olap_scan_common.cpp b/be/src/storage/olap_scan_common.cpp index c522b5989f4718..934fe97e886f87 100644 --- a/be/src/storage/olap_scan_common.cpp +++ b/be/src/storage/olap_scan_common.cpp @@ -36,6 +36,21 @@ template <> const typename ColumnValueRange::CppType ColumnValueRange::TYPE_MAX = std::numeric_limits::quiet_NaN(); +/// Convert the internal scan key pairs (_begin_scan_keys / _end_scan_keys) +/// into a vector of OlapScanRange objects that the scanner / tablet reader can consume. +/// +/// Each pair (_begin_scan_keys[i], _end_scan_keys[i]) becomes one OlapScanRange with +/// has_lower_bound = true and has_upper_bound = true, because these tuples always +/// carry real typed boundary values produced by extend_scan_key(). +/// +/// If _begin_scan_keys is empty (no key predicates were pushed down), the caller +/// (_init_scanners) will create a single default-constructed OlapScanRange with +/// has_lower_bound = false / has_upper_bound = false to represent a full table scan. +/// +/// Example – two scan key pairs from "k1 IN (1, 2) AND k2 = 10": +/// _begin_scan_keys = [ (1, 10), (2, 10) ] +/// _end_scan_keys = [ (1, 10), (2, 10) ] +/// => two OlapScanRange objects, both with has_lower_bound=true, has_upper_bound=true. Status OlapScanKeys::get_key_range(std::vector>* key_range) { key_range->clear(); @@ -43,6 +58,8 @@ Status OlapScanKeys::get_key_range(std::vector>* std::unique_ptr range(new OlapScanRange()); range->begin_scan_range = _begin_scan_keys[i]; range->end_scan_range = _end_scan_keys[i]; + range->has_lower_bound = true; + range->has_upper_bound = true; range->begin_include = _begin_include; range->end_include = _end_include; key_range->emplace_back(std::move(range)); diff --git a/be/src/storage/olap_scan_common.h b/be/src/storage/olap_scan_common.h index 5a2faedabf33f6..54052c965d786a 100644 --- a/be/src/storage/olap_scan_common.h +++ b/be/src/storage/olap_scan_common.h @@ -56,51 +56,13 @@ namespace doris { #include "common/compile_check_begin.h" -template -std::string cast_to_string(T value, int scale) { - if constexpr (primitive_type == TYPE_DECIMAL32) { - return ((Decimal)value).to_string(scale); - } else if constexpr (primitive_type == TYPE_DECIMAL64) { - return ((Decimal)value).to_string(scale); - } else if constexpr (primitive_type == TYPE_DECIMAL128I) { - return ((Decimal)value).to_string(scale); - } else if constexpr (primitive_type == TYPE_DECIMAL256) { - return ((Decimal)value).to_string(scale); - } else if constexpr (primitive_type == TYPE_TINYINT) { - return std::to_string(static_cast(value)); - } else if constexpr (primitive_type == TYPE_LARGEINT) { - return int128_to_string(value); - } else if constexpr (primitive_type == TYPE_DATETIMEV2) { - auto datetimev2_val = static_cast>(value); - char buf[30]; - datetimev2_val.to_string(buf); - std::stringstream ss; - ss << buf; - return ss.str(); - } else if constexpr (primitive_type == TYPE_TIMESTAMPTZ) { - auto timestamptz_val = static_cast(value); - return timestamptz_val.to_string(cctz::utc_time_zone(), scale); - } else if constexpr (primitive_type == TYPE_TIMEV2) { - return TimeValue::to_string(value, scale); - } else if constexpr (primitive_type == TYPE_IPV4) { - return IPv4Value::to_string(value); - } else if constexpr (primitive_type == TYPE_IPV6) { - return IPv6Value::to_string(value); - } else if constexpr (primitive_type == TYPE_BOOLEAN) { - return CastToString::from_number(value); - } else { - return boost::lexical_cast(value); - } -} - /** * @brief Column's value range **/ template class ColumnValueRange { public: - using CppType = std::conditional_t::CppType>; + using CppType = typename PrimitiveTypeTraits::CppType; using SetType = std::set>; using IteratorType = typename SetType::iterator; @@ -273,7 +235,6 @@ class ColumnValueRange { primitive_type == PrimitiveType::TYPE_DOUBLE || primitive_type == PrimitiveType::TYPE_LARGEINT || primitive_type == PrimitiveType::TYPE_DECIMALV2 || - primitive_type == PrimitiveType::TYPE_HLL || primitive_type == PrimitiveType::TYPE_VARCHAR || primitive_type == PrimitiveType::TYPE_CHAR || primitive_type == PrimitiveType::TYPE_STRING || @@ -292,13 +253,53 @@ const typename ColumnValueRange::CppType ColumnValueRange const typename ColumnValueRange::CppType ColumnValueRange::TYPE_MAX; +/// OlapScanKeys accumulates multi-column prefix scan keys from per-column ColumnValueRange +/// constraints, and converts them into OlapScanRange objects for the storage layer. +/// +/// Overall pipeline (with examples for table t(k1 INT, k2 INT, v INT)): +/// +/// 1. _normalize_conjuncts() (scan_operator.cpp) +/// Parses SQL WHERE conjuncts into per-column ColumnValueRange objects. +/// e.g. "WHERE k1 IN (1,2) AND k2 = 10" +/// => ColumnValueRange: fixed_values = {1, 2} +/// => ColumnValueRange: fixed_values = {10} +/// +/// 2. _build_key_ranges_and_filters() (olap_scan_operator.cpp) +/// Iterates key columns in schema order, calling extend_scan_key() for each column +/// to expand internal _begin_scan_keys / _end_scan_keys. +/// +/// 3. extend_scan_key() (this class) +/// Appends one more column dimension to existing scan keys (Cartesian product for +/// fixed values, or min/max for range values). +/// After k1: _begin_scan_keys = [(1), (2)] _end_scan_keys = [(1), (2)] +/// After k2: _begin_scan_keys = [(1,10), (2,10)] _end_scan_keys = [(1,10), (2,10)] +/// +/// 4. get_key_range() (olap_scan_common.cpp) +/// Converts each (_begin_scan_keys[i], _end_scan_keys[i]) pair into an OlapScanRange. +/// => OlapScanRange{ begin=(1,10), end=(1,10), has_lower_bound=true, ... } +/// => OlapScanRange{ begin=(2,10), end=(2,10), has_lower_bound=true, ... } +/// +/// 5. If no key predicates exist, get_key_range returns empty; the caller creates a single +/// default OlapScanRange with has_lower_bound=false (represents full table scan). +/// class OlapScanKeys { public: - // TODO(gabriel): use ColumnPredicate to extend scan key + /// Extend internal scan key pairs with the next key column's ColumnValueRange. + /// + /// - If the range has fixed values, produces a Cartesian product of existing keys + /// and the fixed values (subject to max_scan_key_num limit). + /// - If the range is a scope (min..max), appends min to begin keys and max to end keys, + /// and sets _has_range_value=true (no further columns can be appended). + /// + /// @param exact_value [out]: true if the range covers the column's values exactly + /// (can be erased from residual predicates). + /// @param eos [out]: true if the range is provably empty (no rows to scan). + /// @param should_break[out]: true if the range cannot be encoded and we should stop. template Status extend_scan_key(ColumnValueRange& range, int32_t max_scan_key_num, bool* exact_value, bool* eos, bool* should_break); + /// Convert accumulated scan key pairs into OlapScanRange objects for the storage layer. Status get_key_range(std::vector>* key_range); bool has_range_value() const { return _has_range_value; } @@ -315,8 +316,8 @@ class OlapScanKeys { ss << "ScanKeys:"; for (int i = 0; i < _begin_scan_keys.size(); ++i) { - ss << "ScanKey=" << (_begin_include ? "[" : "(") << _begin_scan_keys[i] << " : " - << _end_scan_keys[i] << (_end_include ? "]" : ")"); + ss << "ScanKey=" << (_begin_include ? "[" : "(") << _begin_scan_keys[i].debug_string() + << " : " << _end_scan_keys[i].debug_string() << (_end_include ? "]" : ")"); } return ss.str(); } @@ -345,9 +346,9 @@ using ColumnValueRangeType = std::variant< ColumnValueRange, ColumnValueRange, ColumnValueRange, ColumnValueRange, ColumnValueRange, ColumnValueRange, ColumnValueRange, - ColumnValueRange, ColumnValueRange, - ColumnValueRange, ColumnValueRange, - ColumnValueRange, ColumnValueRange>; + ColumnValueRange, ColumnValueRange, + ColumnValueRange, ColumnValueRange, + ColumnValueRange>; template const typename ColumnValueRange::CppType @@ -362,20 +363,6 @@ template ColumnValueRange::ColumnValueRange() : _column_type(INVALID_TYPE), _precision(-1), _scale(-1) {} -template -ColumnValueRange::ColumnValueRange(std::string col_name, const CppType& min, - const CppType& max, bool contain_null) - : _column_name(std::move(col_name)), - _column_type(primitive_type), - _low_value(min), - _high_value(max), - _low_op(FILTER_LARGER_OR_EQUAL), - _high_op(FILTER_LESS_OR_EQUAL), - _is_nullable_col(true), - _contain_null(contain_null), - _precision(-1), - _scale(-1) {} - template ColumnValueRange::ColumnValueRange(std::string col_name, const CppType& min, const CppType& max, bool is_nullable_col, @@ -522,13 +509,19 @@ bool ColumnValueRange::convert_to_avg_range_value( auto no_split = [&]() -> bool { begin_scan_keys.emplace_back(); - begin_scan_keys.back().add_value( - cast_to_string(get_range_min_value(), scale()), - contain_null()); + if (contain_null()) { + begin_scan_keys.back().add_null(); + } else { + begin_scan_keys.back().add_field( + Field::create_field(get_range_min_value())); + } end_scan_keys.emplace_back(); - end_scan_keys.back().add_value( - cast_to_string(get_range_max_value(), scale()), - empty_range_only_null ? true : false); + if (empty_range_only_null) { + end_scan_keys.back().add_null(); + } else { + end_scan_keys.back().add_field( + Field::create_field(get_range_max_value())); + } return true; }; if (empty_range_only_null || max_scan_key_num == 1) { @@ -565,8 +558,7 @@ bool ColumnValueRange::convert_to_avg_range_value( } while (true) { begin_scan_keys.emplace_back(); - begin_scan_keys.back().add_value( - cast_to_string(min_value, scale())); + begin_scan_keys.back().add_field(Field::create_field(min_value)); if (cast(max_value) - min_value < step_size) { min_value = max_value; @@ -575,8 +567,7 @@ bool ColumnValueRange::convert_to_avg_range_value( } end_scan_keys.emplace_back(); - end_scan_keys.back().add_value( - cast_to_string(min_value, scale())); + end_scan_keys.back().add_field(Field::create_field(min_value)); if (Compare::equal(min_value, max_value)) { break; @@ -585,11 +576,8 @@ bool ColumnValueRange::convert_to_avg_range_value( ++real_step_size; if (real_step_size > MAX_STEP_SIZE) { throw Exception(Status::InternalError( - "convert_to_avg_range_value meet error. type={}, step_size={}, " - "min_value={}, max_value={}", - int(primitive_type), step_size, - cast_to_string(min_value, scale()), - cast_to_string(max_value, scale()))); + "convert_to_avg_range_value meet error. type={}, step_size={}", + int(primitive_type), step_size)); } } @@ -819,27 +807,96 @@ void ColumnValueRange::intersection(ColumnValueRange / >= / < / <= predicates (begin = min, end = max) +/// +/// ======== Example 1: Two fixed-value columns (IN + =) ======== +/// Table t(k1 INT, k2 INT, v INT), key columns = (k1, k2). +/// WHERE k1 IN (1, 2) AND k2 = 10 +/// +/// Call 1: extend_scan_key(k1's range {fixed_values={1,2}}) +/// _begin_scan_keys was empty, so create one pair per fixed value: +/// _begin = [(1), (2)] _end = [(1), (2)] include=[true, true] +/// +/// Call 2: extend_scan_key(k2's range {fixed_values={10}}) +/// _begin is non-empty, so do Cartesian product (existing keys × new fixed values): +/// _begin = [(1,10), (2,10)] _end = [(1,10), (2,10)] include=[true, true] +/// +/// ======== Example 2: Fixed + range (IN + between) ======== +/// WHERE k1 IN (1, 2) AND k2 >= 5 AND k2 < 10 +/// +/// Call 1: extend_scan_key(k1's range {fixed_values={1,2}}) +/// _begin = [(1), (2)] _end = [(1), (2)] +/// +/// Call 2: extend_scan_key(k2's range {scope [5, 10)}) +/// k2 is a scope range, so append min=5 to all begin keys, max=10 to all end keys: +/// _begin = [(1,5), (2,5)] _end = [(1,10), (2,10)] +/// _begin_include = true (>=) _end_include = false (<) +/// Set _has_range_value = true → no further columns can be appended. +/// +/// ======== Example 3: Single range column ======== +/// WHERE k1 >= 100 AND k1 <= 200 +/// +/// Call 1: extend_scan_key(k1's range {scope [100, 200]}) +/// _begin was empty, so create one pair: +/// _begin = [(100)] _end = [(200)] include=[true, true] +/// Set _has_range_value = true. +/// +/// ======== Example 4: Too many fixed values (exceeds max_scan_key_num) ======== +/// WHERE k1 IN (1, 2, ..., 10000) — exceeds limit +/// +/// If is_range_value_convertible(): convert fixed set {1..10000} to scope [1, 10000], +/// then extend as a range (same as Example 3), and set *exact_value = false +/// (the predicate must be kept for residual filtering). +/// +/// If NOT convertible (e.g. BOOLEAN/NULL type): set *should_break = true, stop extending. +/// +/// ======== Example 5: Range splitting (convert_to_avg_range_value) ======== +/// WHERE k1 >= 1 AND k1 <= 100, with max_scan_key_num = 4 +/// If k1 is an integer type that supports splitting: +/// convert_to_close_range: adjust to closed range [1, 100] +/// convert_to_avg_range_value: split into ~4 sub-ranges: +/// _begin = [(1), (26), (51), (76)] _end = [(25), (50), (75), (100)] +/// Set _has_range_value = true. +/// +/// @param range [in/out] The next key column's ColumnValueRange (may be mutated +/// if fixed values must be converted to a range). +/// @param max_scan_key_num [in] Upper limit on total number of scan key pairs. +/// @param exact_value [out] Set to true if the column's predicate is fully captured +/// by scan keys (can be erased from residual filters). +/// @param eos [out] Set to true if the range is provably empty. +/// @param should_break [out] Set to true if extending must stop (un-convertible overflow). template Status OlapScanKeys::extend_scan_key(ColumnValueRange& range, int32_t max_scan_key_num, bool* exact_value, bool* eos, bool* should_break) { - using CppType = std::conditional_t::CppType>; using ConstIterator = typename ColumnValueRange::SetType::const_iterator; - // 1. clear ScanKey if some column range is empty + // 1. If the column's value range is empty (contradictory predicates, e.g. k1 > 10 AND k1 < 5), + // clear all accumulated keys — no rows can match. if (range.is_empty_value_range()) { _begin_scan_keys.clear(); _end_scan_keys.clear(); return Status::OK(); } - // 2. stop extend ScanKey when it's already extend a range value + // 2. Once a previous column was extended as a scope range, we cannot append more columns, + // because the begin/end keys would have different semantics per pair. + // e.g. after k1 in [5, 10), appending k2 values is meaningless for short-key index. if (_has_range_value) { return Status::OK(); } - //if a column doesn't have any predicate, we will try converting the range to fixed values + // 3. Overflow check: if fixed_value_count × existing_key_count > max_scan_key_num, + // Cartesian product would be too large. + // - If convertible: degrade fixed values {v1,v2,...} to scope [min(v), max(v)], + // set *exact_value = false (keep predicate as residual filter). + // - If not convertible (BOOLEAN etc.): stop extending (*should_break = true). auto scan_keys_size = _begin_scan_keys.empty() ? 1 : _begin_scan_keys.size(); if (range.is_fixed_value_range()) { if (range.get_fixed_value_size() > max_scan_key_num / scan_keys_size) { @@ -852,6 +909,10 @@ Status OlapScanKeys::extend_scan_key(ColumnValueRange& range, } } } else { + // 4. Range-splitting optimization: if this is the FIRST key column and it's a scope + // range on a splittable integer type, try to split [low, high] into multiple + // sub-ranges for parallel / pipelined scanning. + // e.g. k1 in [1, 100] with max_scan_key_num=4 → [(1,25), (26,50), (51,75), (76,100)] if (_begin_scan_keys.empty() && range.is_fixed_value_convertible() && _is_convertible && !range.is_reject_split_type()) { *eos |= range.convert_to_close_range(_begin_scan_keys, _end_scan_keys, _begin_include, @@ -866,20 +927,23 @@ Status OlapScanKeys::extend_scan_key(ColumnValueRange& range, } } - // 3.1 extend ScanKey with FixedValueRange + // ==================================================================== + // 5. Actually extend scan keys with this column's values. + // ==================================================================== + if (range.is_fixed_value_range()) { - // 3.1.1 construct num of fixed value ScanKey (begin_key == end_key) + // ---- 5a. Fixed values (IN / =): point lookup, begin == end per value. ---- if (_begin_scan_keys.empty()) { + // First column: create one key pair per fixed value. + // e.g. k1 IN (1, 2) → _begin=[(1),(2)] _end=[(1),(2)] auto fixed_value_set = range.get_fixed_value_set(); ConstIterator iter = fixed_value_set.begin(); for (; iter != fixed_value_set.end(); ++iter) { _begin_scan_keys.emplace_back(); - _begin_scan_keys.back().add_value( - cast_to_string(*iter, range.scale())); + _begin_scan_keys.back().add_field(Field::create_field(*iter)); _end_scan_keys.emplace_back(); - _end_scan_keys.back().add_value( - cast_to_string(*iter, range.scale())); + _end_scan_keys.back().add_field(Field::create_field(*iter)); } if (range.contain_null()) { @@ -888,8 +952,10 @@ Status OlapScanKeys::extend_scan_key(ColumnValueRange& range, _end_scan_keys.emplace_back(); _end_scan_keys.back().add_null(); } - } // 3.1.2 produces the Cartesian product of ScanKey and fixed_value - else { + } else { + // Subsequent column: Cartesian product of existing keys × new fixed values. + // e.g. existing = [(1),(2)], k2 IN (10, 20) + // → [(1,10),(1,20),(2,10),(2,20)] auto fixed_value_set = range.get_fixed_value_set(); size_t original_key_range_size = _begin_scan_keys.size(); @@ -900,20 +966,16 @@ Status OlapScanKeys::extend_scan_key(ColumnValueRange& range, ConstIterator iter = fixed_value_set.begin(); for (; iter != fixed_value_set.end(); ++iter) { - // alter the first ScanKey in original place + // Reuse i-th slot for the first value, append new slots for the rest. if (iter == fixed_value_set.begin()) { - _begin_scan_keys[i].add_value( - cast_to_string(*iter, range.scale())); - _end_scan_keys[i].add_value( - cast_to_string(*iter, range.scale())); - } // append follow ScanKey - else { + _begin_scan_keys[i].add_field(Field::create_field(*iter)); + _end_scan_keys[i].add_field(Field::create_field(*iter)); + } else { _begin_scan_keys.push_back(start_base_key_range); - _begin_scan_keys.back().add_value( - cast_to_string(*iter, range.scale())); + _begin_scan_keys.back().add_field( + Field::create_field(*iter)); _end_scan_keys.push_back(end_base_key_range); - _end_scan_keys.back().add_value( - cast_to_string(*iter, range.scale())); + _end_scan_keys.back().add_field(Field::create_field(*iter)); } } @@ -926,13 +988,20 @@ Status OlapScanKeys::extend_scan_key(ColumnValueRange& range, } } + // Fixed values are always closed intervals (begin == end, point lookup). _begin_include = true; _end_include = true; - } // Extend ScanKey with range value - else { + } else { + // ---- 5b. Scope range (> / >= / < / <=): append min to begin, max to end. ---- + // After this, no more columns can be appended (_has_range_value = true), + // because the range semantics only apply to the last appended column. + // e.g. existing = [(1),(2)], k2 >= 5 AND k2 < 10 + // → _begin = [(1,5),(2,5)] _end = [(1,10),(2,10)] + // → begin_include=true, end_include=false _has_range_value = true; - /// if max < min, this range should only contains a null value. + // Special case: max < min means the range itself is empty, + // but contain_null() is true, so only null values match this column. if (Compare::less(range.get_range_max_value(), range.get_range_min_value())) { CHECK(range.contain_null()); if (_begin_scan_keys.empty()) { @@ -947,23 +1016,32 @@ Status OlapScanKeys::extend_scan_key(ColumnValueRange& range, } } } else if (_begin_scan_keys.empty()) { + // First column as a range: + // e.g. k1 >= 100 AND k1 <= 200 → _begin=[(100)] _end=[(200)] _begin_scan_keys.emplace_back(); - _begin_scan_keys.back().add_value(cast_to_string( - range.get_range_min_value(), range.scale()), - range.contain_null()); + if (range.contain_null()) { + _begin_scan_keys.back().add_null(); + } else { + _begin_scan_keys.back().add_field( + Field::create_field(range.get_range_min_value())); + } _end_scan_keys.emplace_back(); - _end_scan_keys.back().add_value(cast_to_string( - range.get_range_max_value(), range.scale())); + _end_scan_keys.back().add_field( + Field::create_field(range.get_range_max_value())); } else { + // Subsequent column as a range: append min/max to every existing key pair. for (int i = 0; i < _begin_scan_keys.size(); ++i) { - _begin_scan_keys[i].add_value(cast_to_string( - range.get_range_min_value(), range.scale()), - range.contain_null()); + if (range.contain_null()) { + _begin_scan_keys[i].add_null(); + } else { + _begin_scan_keys[i].add_field( + Field::create_field(range.get_range_min_value())); + } } for (int i = 0; i < _end_scan_keys.size(); ++i) { - _end_scan_keys[i].add_value(cast_to_string( - range.get_range_max_value(), range.scale())); + _end_scan_keys[i].add_field( + Field::create_field(range.get_range_max_value())); } } _begin_include = range.is_begin_include(); diff --git a/be/src/storage/olap_tuple.h b/be/src/storage/olap_tuple.h index 9a3ed19a3aef73..3deec5823957f8 100644 --- a/be/src/storage/olap_tuple.h +++ b/be/src/storage/olap_tuple.h @@ -17,71 +17,45 @@ #pragma once -#include #include +#include "core/field.h" + namespace doris { class OlapTuple { public: OlapTuple() {} - OlapTuple(const std::vector& values) - : _values(values), _nulls(values.size(), false) {} - - void add_null() { - _values.push_back(""); - _nulls.push_back(true); - } - - void add_value(const std::string& value, bool is_null = false) { - _values.push_back(value); - _nulls.push_back(is_null); - } - - size_t size() const { return _values.size(); } - - void reserve(size_t size) { - _values.reserve(size); - _nulls.reserve(size); - } - - void set_value(size_t i, const std::string& value, bool is_null = false) { - _values[i] = value; - _nulls[i] = is_null; - } - - bool is_null(size_t i) const { return _nulls[i]; } - const std::string& get_value(size_t i) const { return _values[i]; } - const std::vector& values() const { return _values; } - void reset() { - _values.clear(); - _nulls.clear(); + void add_null() { _fields.emplace_back(PrimitiveType::TYPE_NULL); } + + void add_field(Field f) { _fields.push_back(std::move(f)); } + + size_t size() const { return _fields.size(); } + + // Return debug string for profile/logging only. + // NOTE: this output may be inaccurate for decimal types because OlapTuple + // does not carry decimal scale metadata and falls back to scale=0. + std::string debug_string() const { + std::string result; + for (size_t i = 0; i < _fields.size(); ++i) { + if (i > 0) { + result.append(","); + } + if (_fields[i].is_null()) { + result.append("null"); + } else { + result.append(_fields[i].to_debug_string(0)); + } + } + return result; } - std::string operator[](size_t index) const { return _values[index]; } + const Field& get_field(size_t i) const { return _fields[i]; } + Field& get_field(size_t i) { return _fields[i]; } private: - friend std::ostream& operator<<(std::ostream& os, const OlapTuple& tuple); - - std::vector _values; - std::vector _nulls; + std::vector _fields; }; -inline std::ostream& operator<<(std::ostream& os, const OlapTuple& tuple) { - for (int i = 0; i < tuple._values.size(); ++i) { - if (i > 0) { - os << ","; - } - if (tuple._nulls[i]) { - os << "null("; - } - os << tuple._values[i]; - if (tuple._nulls[i]) { - os << ")"; - } - } - return os; -} - } // namespace doris diff --git a/be/src/storage/olap_utils.h b/be/src/storage/olap_utils.h index 9c37849271123f..680b1793b731a7 100644 --- a/be/src/storage/olap_utils.h +++ b/be/src/storage/olap_utils.h @@ -17,11 +17,12 @@ #pragma once -#include #include #include #include +#include + #include "common/logging.h" #include "core/data_type/primitive_type.h" #include "storage/olap_tuple.h" @@ -30,34 +31,56 @@ namespace doris { using CompareLargeFunc = bool (*)(const void*, const void*); -static const char* NEGATIVE_INFINITY = "-oo"; -static const char* POSITIVE_INFINITY = "+oo"; - +/// OlapScanRange represents a single key-range interval used to scan an OLAP tablet. +/// +/// It is the final product of the scan-key generation pipeline: +/// +/// SQL WHERE conjuncts +/// -> ColumnValueRange (per-column value constraints, see olap_scan_common.h) +/// -> OlapScanKeys::extend_scan_key() (combine columns into multi-column prefix keys) +/// -> OlapScanKeys::get_key_range() (emit one OlapScanRange per key pair) +/// -> OlapScanner / tablet reader (use ranges for short-key index lookup) +/// +/// Example – table t(k1 INT, k2 INT, v INT) with key columns (k1, k2): +/// +/// WHERE k1 IN (1, 2) AND k2 = 10 +/// => two OlapScanRange objects: +/// range0: begin=(1, 10) end=(1, 10) include=[true, true] -- point lookup +/// range1: begin=(2, 10) end=(2, 10) include=[true, true] -- point lookup +/// +/// WHERE k1 >= 5 AND k1 < 10 +/// => one OlapScanRange: +/// begin=(5) end=(10) begin_include=true end_include=false +/// +/// No key predicates at all (full table scan): +/// => one default-constructed OlapScanRange with has_lower_bound=false, has_upper_bound=false. +/// Consumers detect this and skip pushing key range to the reader (fall back to full scan). +/// struct OlapScanRange { public: - OlapScanRange() : begin_include(true), end_include(true) { - begin_scan_range.add_value(NEGATIVE_INFINITY); - end_scan_range.add_value(POSITIVE_INFINITY); - } - OlapScanRange(bool begin, bool end, std::vector& begin_range, - std::vector& end_range) - : begin_include(begin), - end_include(end), - begin_scan_range(begin_range), - end_scan_range(end_range) {} + OlapScanRange() + : begin_include(true), + end_include(true), + has_lower_bound(false), + has_upper_bound(false) {} bool begin_include; bool end_include; + + /// Whether this range carries real begin/end bounds. + /// false only for the default-constructed "full scan" placeholder + /// (created when no key predicates exist at all). + bool has_lower_bound; + bool has_upper_bound; + OlapTuple begin_scan_range; OlapTuple end_scan_range; std::string debug_string() const { - fmt::memory_buffer buf; - DCHECK_EQ(begin_scan_range.size(), end_scan_range.size()); - for (int i = 0; i < begin_scan_range.size(); i++) { - fmt::format_to(buf, "({}, {})\n", begin_scan_range[i], end_scan_range[i]); - } - return fmt::to_string(buf); + std::ostringstream buf; + buf << "begin=(" << begin_scan_range.debug_string() << "), end=(" + << end_scan_range.debug_string() << ")"; + return buf.str(); } }; diff --git a/be/src/storage/row_cursor.cpp b/be/src/storage/row_cursor.cpp index c1494d320edfca..2250781b0ba9ca 100644 --- a/be/src/storage/row_cursor.cpp +++ b/be/src/storage/row_cursor.cpp @@ -18,171 +18,90 @@ #include "storage/row_cursor.h" #include -#include #include -#include #include #include #include "common/cast_set.h" +#include "common/consts.h" +#include "core/data_type/primitive_type.h" +#include "core/field.h" #include "storage/field.h" #include "storage/olap_common.h" #include "storage/olap_define.h" #include "storage/tablet/tablet_schema.h" +#include "storage/types.h" #include "util/slice.h" -using std::nothrow; -using std::string; -using std::vector; - namespace doris { #include "common/compile_check_begin.h" using namespace ErrorCode; -RowCursor::RowCursor() - : _fixed_len(0), _variable_len(0), _string_field_count(0), _long_text_buf(nullptr) {} - -RowCursor::~RowCursor() { - delete[] _owned_fixed_buf; - delete[] _variable_buf; - if (_string_field_count > 0 && _long_text_buf != nullptr) { - for (int i = 0; i < _string_field_count; ++i) { - free(_long_text_buf[i]); - } - free(_long_text_buf); - } -} -Status RowCursor::_init(const std::vector& columns) { - _variable_len = 0; - for (auto cid : columns) { - if (_schema->column(cid) == nullptr) { - return Status::Error("Fail to malloc _fixed_buf."); - } - _variable_len += column_schema(cid)->get_variable_len(); - if (_schema->column(cid)->type() == FieldType::OLAP_FIELD_TYPE_STRING) { - ++_string_field_count; - } - } - - _fixed_len = _schema->schema_size(); - _fixed_buf = new (nothrow) char[_fixed_len](); - if (_fixed_buf == nullptr) { - return Status::Error("Fail to malloc _fixed_buf."); - } - _owned_fixed_buf = _fixed_buf; +RowCursor::RowCursor() = default; +RowCursor::~RowCursor() = default; +RowCursor::RowCursor(RowCursor&&) noexcept = default; +RowCursor& RowCursor::operator=(RowCursor&&) noexcept = default; - return Status::OK(); +void RowCursor::_init_schema(TabletSchemaSPtr schema, uint32_t column_count) { + std::vector columns(column_count); + std::iota(columns.begin(), columns.end(), 0); + _schema.reset(new Schema(schema->columns(), columns)); } -Status RowCursor::_init(const std::shared_ptr& shared_schema, - const std::vector& columns) { +void RowCursor::_init_schema(const std::shared_ptr& shared_schema, uint32_t column_count) { _schema.reset(new Schema(*shared_schema)); - return _init(columns); } -Status RowCursor::_init(const std::vector& schema, - const std::vector& columns) { - _schema.reset(new Schema(schema, columns)); - return _init(columns); -} - -Status RowCursor::_init_scan_key(TabletSchemaSPtr schema, - const std::vector& scan_keys) { - // NOTE: cid equal with column index - // Hyperloglog cannot be key, no need to handle it - _variable_len = 0; - for (auto cid : _schema->column_ids()) { - const TabletColumn& column = schema->column(cid); - FieldType type = column.type(); - if (type == FieldType::OLAP_FIELD_TYPE_VARCHAR) { - _variable_len += scan_keys[cid].length(); - } else if (type == FieldType::OLAP_FIELD_TYPE_CHAR || - type == FieldType::OLAP_FIELD_TYPE_ARRAY) { - _variable_len += - std::max(scan_keys[cid].length(), static_cast(column.length())); - } else if (type == FieldType::OLAP_FIELD_TYPE_STRING) { - ++_string_field_count; - } - } - - // variable_len for null bytes - RETURN_IF_ERROR(_alloc_buf()); - char* fixed_ptr = _fixed_buf; - char* variable_ptr = _variable_buf; - char** long_text_ptr = _long_text_buf; - for (auto cid : _schema->column_ids()) { - const TabletColumn& column = schema->column(cid); - fixed_ptr = _fixed_buf + _schema->column_offset(cid); - FieldType type = column.type(); - if (type == FieldType::OLAP_FIELD_TYPE_VARCHAR) { - // Use memcpy to avoid misaligned store on fixed_ptr + 1 - Slice slice(variable_ptr, scan_keys[cid].length()); - memcpy(fixed_ptr + 1, &slice, sizeof(Slice)); - variable_ptr += scan_keys[cid].length(); - } else if (type == FieldType::OLAP_FIELD_TYPE_CHAR) { - // Use memcpy to avoid misaligned store on fixed_ptr + 1 - size_t len = std::max(scan_keys[cid].length(), static_cast(column.length())); - Slice slice(variable_ptr, len); - memcpy(fixed_ptr + 1, &slice, sizeof(Slice)); - variable_ptr += len; - } else if (type == FieldType::OLAP_FIELD_TYPE_STRING) { - // Use memcpy to avoid misaligned store on fixed_ptr + 1 - _schema->mutable_column(cid)->set_long_text_buf(long_text_ptr); - Slice slice(*(long_text_ptr), DEFAULT_TEXT_LENGTH); - memcpy(fixed_ptr + 1, &slice, sizeof(Slice)); - ++long_text_ptr; - } +Status RowCursor::init(TabletSchemaSPtr schema, size_t num_columns) { + if (num_columns > schema->num_columns()) { + return Status::Error( + "Input param are invalid. Column count is bigger than num_columns of schema. " + "column_count={}, schema.num_columns={}", + num_columns, schema->num_columns()); } - + _init_schema(schema, cast_set(num_columns)); + // Initialize all fields as null (TYPE_NULL). + _fields.resize(num_columns); return Status::OK(); } -Status RowCursor::_init(TabletSchemaSPtr schema, uint32_t column_count) { - if (column_count > schema->num_columns()) { +Status RowCursor::init(TabletSchemaSPtr schema, const OlapTuple& tuple) { + size_t key_size = tuple.size(); + if (key_size > schema->num_columns()) { return Status::Error( "Input param are invalid. Column count is bigger than num_columns of schema. " "column_count={}, schema.num_columns={}", - column_count, schema->num_columns()); + key_size, schema->num_columns()); } - std::vector columns; - for (auto i = 0; i < column_count; ++i) { - columns.push_back(i); - } - RETURN_IF_ERROR(_init(schema->columns(), columns)); - return Status::OK(); + _init_schema(schema, cast_set(key_size)); + return from_tuple(tuple); } -Status RowCursor::init_scan_key(TabletSchemaSPtr schema, - const std::vector& scan_keys) { - size_t scan_key_size = scan_keys.size(); - if (scan_key_size > schema->num_columns()) { +Status RowCursor::init(TabletSchemaSPtr schema, const OlapTuple& tuple, + const std::shared_ptr& shared_schema) { + size_t key_size = tuple.size(); + if (key_size > schema->num_columns()) { return Status::Error( "Input param are invalid. Column count is bigger than num_columns of schema. " "column_count={}, schema.num_columns={}", - scan_key_size, schema->num_columns()); + key_size, schema->num_columns()); } - - std::vector columns(scan_key_size); - std::iota(columns.begin(), columns.end(), 0); - - RETURN_IF_ERROR(_init(schema->columns(), columns)); - - return _init_scan_key(schema, scan_keys); + _init_schema(shared_schema, cast_set(key_size)); + return from_tuple(tuple); } -Status RowCursor::init_scan_key(TabletSchemaSPtr schema, const std::vector& scan_keys, - const std::shared_ptr& shared_schema) { - size_t scan_key_size = scan_keys.size(); - - std::vector columns; - for (uint32_t i = 0; i < scan_key_size; ++i) { - columns.push_back(i); +Status RowCursor::init_scan_key(TabletSchemaSPtr schema, std::vector fields) { + size_t key_size = fields.size(); + if (key_size > schema->num_columns()) { + return Status::Error( + "Input param are invalid. Column count is bigger than num_columns of schema. " + "column_count={}, schema.num_columns={}", + key_size, schema->num_columns()); } - - RETURN_IF_ERROR(_init(shared_schema, columns)); - - return _init_scan_key(schema, scan_keys); + _init_schema(schema, cast_set(key_size)); + _fields = std::move(fields); + return Status::OK(); } Status RowCursor::from_tuple(const OlapTuple& tuple) { @@ -191,70 +110,209 @@ Status RowCursor::from_tuple(const OlapTuple& tuple) { "column count does not match. tuple_size={}, field_count={}", tuple.size(), _schema->num_column_ids()); } - _row_string.resize(tuple.size()); - + _fields.resize(tuple.size()); for (size_t i = 0; i < tuple.size(); ++i) { - auto cid = _schema->column_ids()[i]; - const StorageField* field = column_schema(cid); - if (tuple.is_null(i)) { - _set_null(cid); - continue; - } - _set_not_null(cid); - _row_string[i] = tuple.get_value(i); - char* buf = _cell_ptr(cid); - Status res = field->from_string(buf, tuple.get_value(i), field->get_precision(), - field->get_scale()); - if (!res.ok()) { - LOG(WARNING) << "fail to convert field from string. string=" << tuple.get_value(i) - << ", res=" << res; - return res; - } + _fields[i] = tuple.get_field(i); } - return Status::OK(); } +RowCursor RowCursor::clone() const { + RowCursor result; + result._schema = std::make_unique(*_schema); + result._fields = _fields; + return result; +} + +void RowCursor::pad_char_fields() { + for (size_t i = 0; i < _fields.size(); ++i) { + const StorageField* col = _schema->column(cast_set(i)); + if (col->type() == FieldType::OLAP_FIELD_TYPE_CHAR && !_fields[i].is_null()) { + String padded = _fields[i].get(); + padded.resize(col->length(), '\0'); + _fields[i] = Field::create_field(std::move(padded)); + } + } +} + std::string RowCursor::to_string() const { std::string result; - size_t i = 0; - for (auto cid : _schema->column_ids()) { + for (size_t i = 0; i < _fields.size(); ++i) { if (i > 0) { result.append("|"); } - - result.append(std::to_string(_is_null(cid))); - result.append("&"); - if (_is_null(cid)) { - result.append("NULL"); + if (_fields[i].is_null()) { + result.append("1&NULL"); } else { - result.append(_row_string[i]); + result.append("0&"); + result.append(_fields[i].to_debug_string( + _schema->column(cast_set(i))->get_scale())); } - ++i; } - return result; } -Status RowCursor::_alloc_buf() { - // variable_len for null bytes - _variable_buf = new (nothrow) char[_variable_len](); - if (_variable_buf == nullptr) { - return Status::Error("Fail to malloc _variable_buf."); + +// Convert a Field value to its storage representation via PrimitiveTypeConvertor and encode. +// For most types this is an identity conversion; for DATE, DATETIME, DECIMALV2 it does +// actual conversion to the olap storage format. +template +static void encode_non_string_field(const StorageField* storage_field, const Field& f, + bool full_encode, std::string* buf) { + auto storage_val = PrimitiveTypeConvertor::to_storage_field_type(f.get()); + if (full_encode) { + storage_field->full_encode_ascending(&storage_val, buf); + } else { + storage_field->encode_ascending(&storage_val, buf); } - if (_string_field_count > 0) { - _long_text_buf = (char**)malloc(_string_field_count * sizeof(char*)); - if (_long_text_buf == nullptr) { - return Status::Error("Fail to malloc _long_text_buf."); +} + +void RowCursor::_encode_field(const StorageField* storage_field, const Field& f, bool full_encode, + std::string* buf) const { + FieldType ft = storage_field->type(); + + if (field_is_slice_type(ft)) { + // String types: CHAR, VARCHAR, STRING — all stored as String in Field. + const String& str = f.get(); + + if (ft == FieldType::OLAP_FIELD_TYPE_CHAR) { + // CHAR type: must pad with \0 to the declared column length + size_t col_len = storage_field->length(); + String padded(col_len, '\0'); + memcpy(padded.data(), str.data(), std::min(str.size(), col_len)); + + Slice slice(padded.data(), col_len); + if (full_encode) { + storage_field->full_encode_ascending(&slice, buf); + } else { + storage_field->encode_ascending(&slice, buf); + } + } else { + // VARCHAR / STRING: use actual length + Slice slice(str.data(), str.size()); + if (full_encode) { + storage_field->full_encode_ascending(&slice, buf); + } else { + storage_field->encode_ascending(&slice, buf); + } } - for (int i = 0; i < _string_field_count; ++i) { - _long_text_buf[i] = (char*)malloc(DEFAULT_TEXT_LENGTH * sizeof(char)); - if (_long_text_buf[i] == nullptr) { - return Status::Error("Fail to malloc _long_text_buf."); + return; + } + + // Non-string types: convert Field value to storage format via PrimitiveTypeConvertor, + // then encode. For most types this is an identity conversion. + switch (ft) { + case FieldType::OLAP_FIELD_TYPE_BOOL: + encode_non_string_field(storage_field, f, full_encode, buf); + break; + case FieldType::OLAP_FIELD_TYPE_TINYINT: + encode_non_string_field(storage_field, f, full_encode, buf); + break; + case FieldType::OLAP_FIELD_TYPE_SMALLINT: + encode_non_string_field(storage_field, f, full_encode, buf); + break; + case FieldType::OLAP_FIELD_TYPE_INT: + encode_non_string_field(storage_field, f, full_encode, buf); + break; + case FieldType::OLAP_FIELD_TYPE_BIGINT: + encode_non_string_field(storage_field, f, full_encode, buf); + break; + case FieldType::OLAP_FIELD_TYPE_LARGEINT: + encode_non_string_field(storage_field, f, full_encode, buf); + break; + case FieldType::OLAP_FIELD_TYPE_FLOAT: + encode_non_string_field(storage_field, f, full_encode, buf); + break; + case FieldType::OLAP_FIELD_TYPE_DOUBLE: + encode_non_string_field(storage_field, f, full_encode, buf); + break; + case FieldType::OLAP_FIELD_TYPE_DATE: + encode_non_string_field(storage_field, f, full_encode, buf); + break; + case FieldType::OLAP_FIELD_TYPE_DATETIME: + encode_non_string_field(storage_field, f, full_encode, buf); + break; + case FieldType::OLAP_FIELD_TYPE_DATEV2: + encode_non_string_field(storage_field, f, full_encode, buf); + break; + case FieldType::OLAP_FIELD_TYPE_DATETIMEV2: + encode_non_string_field(storage_field, f, full_encode, buf); + break; + case FieldType::OLAP_FIELD_TYPE_TIMESTAMPTZ: + encode_non_string_field(storage_field, f, full_encode, buf); + break; + case FieldType::OLAP_FIELD_TYPE_DECIMAL: + encode_non_string_field(storage_field, f, full_encode, buf); + break; + case FieldType::OLAP_FIELD_TYPE_DECIMAL32: + encode_non_string_field(storage_field, f, full_encode, buf); + break; + case FieldType::OLAP_FIELD_TYPE_DECIMAL64: + encode_non_string_field(storage_field, f, full_encode, buf); + break; + case FieldType::OLAP_FIELD_TYPE_DECIMAL128I: + encode_non_string_field(storage_field, f, full_encode, buf); + break; + case FieldType::OLAP_FIELD_TYPE_DECIMAL256: + encode_non_string_field(storage_field, f, full_encode, buf); + break; + case FieldType::OLAP_FIELD_TYPE_IPV4: + encode_non_string_field(storage_field, f, full_encode, buf); + break; + case FieldType::OLAP_FIELD_TYPE_IPV6: + encode_non_string_field(storage_field, f, full_encode, buf); + break; + default: + LOG(FATAL) << "unsupported field type for encoding: " << int(ft); + break; + } +} + +template +void RowCursor::encode_key_with_padding(std::string* buf, size_t num_keys, + bool padding_minimal) const { + for (uint32_t cid = 0; cid < num_keys; cid++) { + auto* storage_field = _schema->column(cid); + if (storage_field == nullptr) { + if (padding_minimal) { + buf->push_back(KeyConsts::KEY_MINIMAL_MARKER); + } else { + if (is_mow) { + buf->push_back(KeyConsts::KEY_NORMAL_NEXT_MARKER); + } else { + buf->push_back(KeyConsts::KEY_MAXIMAL_MARKER); + } } + break; + } + + if (cid >= _fields.size() || _fields[cid].is_null()) { + buf->push_back(KeyConsts::KEY_NULL_FIRST_MARKER); + continue; } + + buf->push_back(KeyConsts::KEY_NORMAL_MARKER); + _encode_field(storage_field, _fields[cid], is_mow, buf); } - return Status::OK(); } +// Explicit template instantiations +template void RowCursor::encode_key_with_padding(std::string*, size_t, bool) const; +template void RowCursor::encode_key_with_padding(std::string*, size_t, bool) const; + +template +void RowCursor::encode_key(std::string* buf, size_t num_keys) const { + for (uint32_t cid = 0; cid < num_keys; cid++) { + if (cid >= _fields.size() || _fields[cid].is_null()) { + buf->push_back(KeyConsts::KEY_NULL_FIRST_MARKER); + continue; + } + buf->push_back(KeyConsts::KEY_NORMAL_MARKER); + _encode_field(_schema->column(cid), _fields[cid], full_encode, buf); + } +} + +template void RowCursor::encode_key(std::string*, size_t) const; +template void RowCursor::encode_key(std::string*, size_t) const; + #include "common/compile_check_end.h" } // namespace doris diff --git a/be/src/storage/row_cursor.h b/be/src/storage/row_cursor.h index d94591104eac53..8e8a70daf587fb 100644 --- a/be/src/storage/row_cursor.h +++ b/be/src/storage/row_cursor.h @@ -27,8 +27,9 @@ #include "common/consts.h" #include "common/status.h" +#include "core/data_type/primitive_type.h" +#include "core/field.h" #include "storage/olap_tuple.h" -#include "storage/row_cursor_cell.h" #include "storage/schema.h" #include "storage/tablet/tablet_schema.h" @@ -36,153 +37,82 @@ namespace doris { #include "common/compile_check_begin.h" class StorageField; -// Delegate the operation of a row of data +// Delegate the operation of a row of data. +// Stores values as core::Field objects instead of raw byte buffers. class RowCursor { public: - static const int DEFAULT_TEXT_LENGTH = 128; - RowCursor(); - - // Traverse and destroy the field cursor ~RowCursor(); + RowCursor(const RowCursor&) = delete; + RowCursor& operator=(const RowCursor&) = delete; + RowCursor(RowCursor&&) noexcept; + RowCursor& operator=(RowCursor&&) noexcept; - // Initialize with the size of the key, currently only used when splitting the range of key - Status init_scan_key(TabletSchemaSPtr schema, const std::vector& keys); + // Initialize from OlapTuple (which now stores Fields). + // Sets up the schema and copies Fields from the tuple. + Status init(TabletSchemaSPtr schema, const OlapTuple& tuple); + Status init(TabletSchemaSPtr schema, const OlapTuple& tuple, + const std::shared_ptr& shared_schema); - Status init_scan_key(TabletSchemaSPtr schema, const std::vector& keys, - const std::shared_ptr& shared_schema); + // Initialize with schema and num_columns, creating null Fields. + // Caller sets individual fields via mutable_field(). + Status init(TabletSchemaSPtr schema, size_t num_columns); - RowCursorCell cell(uint32_t cid) const { return RowCursorCell(_nullable_cell_ptr(cid)); } + // Initialize from typed Fields directly. + Status init_scan_key(TabletSchemaSPtr schema, std::vector fields); - // Deserialize the value of each field from the string array, - // Each array item must be a \0 terminated string - // and the input string and line cursor need the same number of columns - Status from_tuple(const OlapTuple& tuple); + const Field& field(uint32_t cid) const { return _fields[cid]; } + Field& mutable_field(uint32_t cid) { return _fields[cid]; } - // Output row cursor content in string format - std::string to_string() const; - - OlapTuple to_tuple() const; + size_t field_count() const { return _fields.size(); } - bool is_delete() const { - auto sign_idx = _schema->delete_sign_idx(); - if (sign_idx < 0) { - return false; - } - return *reinterpret_cast(cell(sign_idx).cell_ptr()) > 0; - } - - char* get_buf() const { return _fixed_buf; } + const StorageField* column_schema(uint32_t cid) const { return _schema->column(cid); } + const Schema* schema() const { return _schema.get(); } - // this two functions is used in unit test - size_t get_fixed_len() const { return _fixed_len; } - size_t get_variable_len() const { return _variable_len; } + // Returns a deep copy of this RowCursor with the same schema and field values. + RowCursor clone() const; - const StorageField* column_schema(uint32_t cid) const { return _schema->column(cid); } + // Pad all CHAR-type fields in-place to their declared column length using '\0'. + // RowCursor holds CHAR values in compute format (unpadded). Call this before + // comparing against storage-format data (e.g. _seek_block) where CHAR is padded. + void pad_char_fields(); - const Schema* schema() const { return _schema.get(); } + // Output row cursor content in string format + std::string to_string() const; // Encode one row into binary according given num_keys. - // A cell will be encoded in the format of a marker and encoded content. - // When encoding row, if any cell isn't found in row, this function will - // fill a marker and return. If padding_minimal is true, KEY_MINIMAL_MARKER will - // be added, if padding_minimal is false, KEY_MAXIMAL_MARKER will be added. - // If all num_keys are found in row, no marker will be added. + // Internally converts each core::Field to its storage representation via + // PrimitiveTypeConvertor before passing to KeyCoder. + // CHAR fields are zero-padded to column.length() for encoding. template - void encode_key_with_padding(std::string* buf, size_t num_keys, bool padding_minimal) const { - for (uint32_t cid = 0; cid < num_keys; cid++) { - auto field = _schema->column(cid); - if (field == nullptr) { - if (padding_minimal) { - buf->push_back(KeyConsts::KEY_MINIMAL_MARKER); - } else { - if (is_mow) { - buf->push_back(KeyConsts::KEY_NORMAL_NEXT_MARKER); - } else { - buf->push_back(KeyConsts::KEY_MAXIMAL_MARKER); - } - } - break; - } - - auto c = cell(cid); - if (c.is_null()) { - buf->push_back(KeyConsts::KEY_NULL_FIRST_MARKER); - continue; - } - buf->push_back(KeyConsts::KEY_NORMAL_MARKER); - if (is_mow) { - field->full_encode_ascending(c.cell_ptr(), buf); - } else { - field->encode_ascending(c.cell_ptr(), buf); - } - } - } + void encode_key_with_padding(std::string* buf, size_t num_keys, bool padding_minimal) const; // Encode one row into binary according given num_keys. - // Client call this function must assure that row contains the first - // num_keys columns. + // Client must ensure that row contains the first num_keys columns. template - void encode_key(std::string* buf, size_t num_keys) const { - for (uint32_t cid = 0; cid < num_keys; cid++) { - auto c = cell(cid); - if (c.is_null()) { - buf->push_back(KeyConsts::KEY_NULL_FIRST_MARKER); - continue; - } - buf->push_back(KeyConsts::KEY_NORMAL_MARKER); - if (full_encode) { - _schema->column(cid)->full_encode_ascending(c.cell_ptr(), buf); - } else { - _schema->column(cid)->encode_ascending(c.cell_ptr(), buf); - } - } + void encode_key(std::string* buf, size_t num_keys) const; + + // Encode a single field at column index 'cid' into 'buf'. + void encode_single_field(uint32_t cid, std::string* buf, bool full_encode) const { + const auto& f = _fields[cid]; + DCHECK(!f.is_null()); + _encode_field(_schema->column(cid), f, full_encode, buf); } private: - Status _init(TabletSchemaSPtr schema, uint32_t column_count); - Status _init(const std::vector& columns); - Status _init(const std::shared_ptr& shared_schema, - const std::vector& columns); - // common init function - Status _init(const std::vector& schema, const std::vector& columns); - Status _alloc_buf(); - - Status _init_scan_key(TabletSchemaSPtr schema, const std::vector& scan_keys); - - // Get column nullable pointer with column id - // TODO(zc): make this return const char* - char* _nullable_cell_ptr(uint32_t cid) const { - return _fixed_buf + _schema->column_offset(cid); - } - char* _cell_ptr(uint32_t cid) const { return _fixed_buf + _schema->column_offset(cid) + 1; } + // Copy Fields from an OlapTuple into this cursor. + Status from_tuple(const OlapTuple& tuple); - void _set_null(uint32_t index) const { - *reinterpret_cast(_nullable_cell_ptr(index)) = true; - } + void _init_schema(TabletSchemaSPtr schema, uint32_t column_count); + void _init_schema(const std::shared_ptr& shared_schema, uint32_t column_count); - void _set_not_null(uint32_t index) const { - *reinterpret_cast(_nullable_cell_ptr(index)) = false; - } - - bool _is_null(uint32_t index) const { - return *reinterpret_cast(_nullable_cell_ptr(index)); - } + // Helper: encode a single non-null field for the given column. + // Converts the core::Field to storage format and calls KeyCoder. + void _encode_field(const StorageField* storage_field, const Field& f, bool full_encode, + std::string* buf) const; std::unique_ptr _schema; - - char* _fixed_buf = nullptr; // point to fixed buf - size_t _fixed_len; - char* _owned_fixed_buf = nullptr; // point to buf allocated in init function - - char* _variable_buf = nullptr; - size_t _variable_len; - size_t _string_field_count; - char** _long_text_buf = nullptr; - - std::vector _row_string; - - DISALLOW_COPY_AND_ASSIGN(RowCursor); + std::vector _fields; }; #include "common/compile_check_end.h" } // namespace doris diff --git a/be/src/storage/row_cursor_cell.h b/be/src/storage/row_cursor_cell.h deleted file mode 100644 index f874dba14ad37e..00000000000000 --- a/be/src/storage/row_cursor_cell.h +++ /dev/null @@ -1,36 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -namespace doris { - -struct RowCursorCell { - RowCursorCell(void* ptr) : _ptr(ptr) {} - RowCursorCell(const void* ptr) : _ptr((void*)ptr) {} - bool is_null() const { return *reinterpret_cast(_ptr); } - void set_is_null(bool is_null) const { *reinterpret_cast(_ptr) = is_null; } - void set_null() const { *reinterpret_cast(_ptr) = true; } - void set_not_null() const { *reinterpret_cast(_ptr) = false; } - const void* cell_ptr() const { return (char*)_ptr + 1; } - void* mutable_cell_ptr() const { return (char*)_ptr + 1; } - -private: - void* _ptr = nullptr; -}; - -} // namespace doris \ No newline at end of file diff --git a/be/src/storage/schema.cpp b/be/src/storage/schema.cpp index 3fab5850ca5d9d..9a7c59a24d6ab9 100644 --- a/be/src/storage/schema.cpp +++ b/be/src/storage/schema.cpp @@ -53,10 +53,7 @@ Schema& Schema::operator=(const Schema& other) { void Schema::_copy_from(const Schema& other) { _col_ids = other._col_ids; - _col_offsets = other._col_offsets; - _num_key_columns = other._num_key_columns; - _schema_size = other._schema_size; // Deep copy _cols // TODO(lingbin): really need clone? @@ -72,48 +69,14 @@ void Schema::_init(const std::vector& cols, const std::vector col_id_set(col_ids.begin(), col_ids.end()); for (int cid = 0; cid < cols.size(); ++cid) { if (col_id_set.find(cid) == col_id_set.end()) { continue; } _cols[cid] = StorageFieldFactory::create(*cols[cid]); - - _col_offsets[cid] = offset; - // Plus 1 byte for null byte - offset += _cols[cid]->size() + 1; - } - - _schema_size = offset; -} - -void Schema::_init(const std::vector& cols, - const std::vector& col_ids, size_t num_key_columns) { - _col_ids = col_ids; - _num_key_columns = num_key_columns; - - _cols.resize(cols.size(), nullptr); - _col_offsets.resize(_cols.size(), -1); - - size_t offset = 0; - std::unordered_set col_id_set(col_ids.begin(), col_ids.end()); - for (int cid = 0; cid < cols.size(); ++cid) { - if (col_id_set.find(cid) == col_id_set.end()) { - continue; - } - // TODO(lingbin): is it necessary to clone StorageField? each SegmentIterator will - // use this func, can we avoid clone? - _cols[cid] = cols[cid]->clone(); - - _col_offsets[cid] = offset; - // Plus 1 byte for null byte - offset += _cols[cid]->size() + 1; } - - _schema_size = offset; } Schema::~Schema() { diff --git a/be/src/storage/schema.h b/be/src/storage/schema.h index 6331653b8a154d..b036def75dfff6 100644 --- a/be/src/storage/schema.h +++ b/be/src/storage/schema.h @@ -121,23 +121,6 @@ class Schema { _init(columns, col_ids, num_key_columns); } - Schema(const std::vector& cols, size_t num_key_columns) { - std::vector col_ids(cols.size()); - _unique_ids.resize(cols.size()); - for (uint32_t cid = 0; cid < cols.size(); ++cid) { - col_ids[cid] = cid; - if (cols.at(cid)->name() == DELETE_SIGN) { - _delete_sign_idx = cid; - } - if (cols.at(cid)->name() == VERSION_COL) { - _version_col_idx = cid; - } - _unique_ids[cid] = cols[cid]->unique_id(); - } - - _init(cols, col_ids, num_key_columns); - } - Schema(const Schema&); Schema& operator=(const Schema& other); @@ -154,26 +137,11 @@ class Schema { const doris::StorageField* column(ColumnId cid) const { return _cols[cid]; } - doris::StorageField* mutable_column(ColumnId cid) const { return _cols[cid]; } - size_t num_key_columns() const { return _num_key_columns; } - size_t schema_size() const { return _schema_size; } - - size_t column_offset(ColumnId cid) const { return _col_offsets[cid]; } - - size_t column_size(ColumnId cid) const { return _cols[cid]->size(); } - - bool is_null(const char* row, int index) const { - return *reinterpret_cast(row + _col_offsets[index]); - } - void set_is_null(void* row, uint32_t cid, bool is_null) const { - *reinterpret_cast((char*)row + _col_offsets[cid]) = is_null; - } size_t num_columns() const { return _cols.size(); } size_t num_column_ids() const { return _col_ids.size(); } const std::vector& column_ids() const { return _col_ids; } - const std::vector& unique_ids() const { return _unique_ids; } ColumnId column_id(size_t index) const { return _col_ids[index]; } int32_t unique_id(size_t index) const { return _unique_ids[index]; } int32_t delete_sign_idx() const { return _delete_sign_idx; } @@ -188,8 +156,6 @@ class Schema { private: void _init(const std::vector& cols, const std::vector& col_ids, size_t num_key_columns); - void _init(const std::vector& cols, - const std::vector& col_ids, size_t num_key_columns); void _copy_from(const Schema& other); @@ -197,15 +163,11 @@ class Schema { // a column in current row, not the unique id-identifier of each column std::vector _col_ids; std::vector _unique_ids; - // NOTE: Both _cols[cid] and _col_offsets[cid] can only be accessed when the cid is + // NOTE: _cols[cid] can only be accessed when the cid is // contained in _col_ids std::vector _cols; - // The value of each item indicates the starting offset of the corresponding column in - // current row. e.g. _col_offsets[idx] is the offset of _cols[idx] (idx must in _col_ids) - std::vector _col_offsets; size_t _num_key_columns; - size_t _schema_size; int32_t _delete_sign_idx = -1; bool _has_sequence_col = false; int32_t _rowid_col_idx = -1; diff --git a/be/src/storage/segment/segment_iterator.cpp b/be/src/storage/segment/segment_iterator.cpp index e78ffff0f14282..3b2adc5f68c3ee 100644 --- a/be/src/storage/segment/segment_iterator.cpp +++ b/be/src/storage/segment/segment_iterator.cpp @@ -610,7 +610,14 @@ Status SegmentIterator::_prepare_seek(const StorageReadOptions::KeyRange& key_ra } } if (!_seek_schema) { - _seek_schema = std::make_unique(key_fields, key_fields.size()); + // Schema constructors accept a vector of TabletColumnPtr. Convert + // StorageField pointers to TabletColumnPtr by copying their descriptors. + std::vector cols; + cols.reserve(key_fields.size()); + for (const StorageField* f : key_fields) { + cols.emplace_back(std::make_shared(f->get_desc())); + } + _seek_schema = std::make_unique(cols, cols.size()); } // todo(wb) need refactor here, when using pk to search, _seek_block is useless if (_seek_block.size() == 0) { @@ -1503,7 +1510,13 @@ Status SegmentIterator::_lookup_ordinal_from_sk_index(const RowCursor& key, bool is_include); const auto& key_col_ids = key.schema()->column_ids(); - _convert_rowcursor_to_short_key(key, key_col_ids.size()); + + // Clone the key once and pad CHAR fields to storage format before the binary search. + // _seek_block holds storage-format data where CHAR is zero-padded to column length, + // while RowCursor holds CHAR in compute format (unpadded). Padding once here avoids + // repeated allocation inside the comparison loop. + RowCursor padded_key = key.clone(); + padded_key.pad_char_fields(); ssize_t start_block_id = 0; auto start_iter = sk_index_decoder->lower_bound(index_key); @@ -1532,7 +1545,7 @@ Status SegmentIterator::_lookup_ordinal_from_sk_index(const RowCursor& key, bool while (start < end) { rowid_t mid = (start + end) / 2; RETURN_IF_ERROR(_seek_and_peek(mid)); - int cmp = _compare_short_key_with_seek_block(key_col_ids); + int cmp = _compare_short_key_with_seek_block(padded_key, key_col_ids); if (cmp > 0) { start = mid + 1; } else if (cmp == 0) { diff --git a/be/src/storage/segment/segment_iterator.h b/be/src/storage/segment/segment_iterator.h index 9d8748893562ac..4edef9c77d61ae 100644 --- a/be/src/storage/segment/segment_iterator.h +++ b/be/src/storage/segment/segment_iterator.h @@ -37,6 +37,8 @@ #include "core/block/columns_with_type_and_name.h" #include "core/column/column.h" #include "core/data_type/data_type.h" +#include "core/data_type/primitive_type.h" +#include "core/field.h" #include "exec/common/variant_util.h" #include "exprs/score_runtime.h" #include "exprs/vexpr_fwd.h" @@ -50,7 +52,6 @@ #include "storage/predicate/block_column_predicate.h" #include "storage/predicate/column_predicate.h" #include "storage/row_cursor.h" -#include "storage/row_cursor_cell.h" #include "storage/schema.h" #include "storage/segment/common.h" #include "storage/segment/segment.h" @@ -300,49 +301,12 @@ class SegmentIterator : public RowwiseIterator { Status _construct_compound_expr_context(); - // todo(wb) remove this method after RowCursor is removed - void NO_SANITIZE_UNDEFINED _convert_rowcursor_to_short_key(const RowCursor& key, - size_t num_keys) { - if (_short_key.size() == 0) { - _short_key.resize(num_keys); - for (auto cid = 0; cid < num_keys; cid++) { - auto* field = key.schema()->column(cid); - _short_key[cid] = Schema::get_column_by_field(*field); - } - } else { - for (int i = 0; i < num_keys; i++) { - _short_key[i]->clear(); - } - } - - for (auto cid = 0; cid < num_keys; cid++) { - auto field = key.schema()->column(cid); - if (field == nullptr) { - break; - } - auto cell = key.cell(cid); - if (cell.is_null()) { - _short_key[cid]->insert_default(); - } else { - if (field->type() == FieldType::OLAP_FIELD_TYPE_VARCHAR || - field->type() == FieldType::OLAP_FIELD_TYPE_CHAR || - field->type() == FieldType::OLAP_FIELD_TYPE_STRING) { - const Slice* slice = reinterpret_cast(cell.cell_ptr()); - _short_key[cid]->insert_data(slice->data, slice->size); - } else { - _short_key[cid]->insert_many_fix_len_data( - reinterpret_cast(cell.cell_ptr()), 1); - } - } - } - } - - int _compare_short_key_with_seek_block(const std::vector& col_ids) { + int _compare_short_key_with_seek_block(const RowCursor& key, + const std::vector& col_ids) { for (auto cid : col_ids) { - // todo(wb) simd compare when memory layout in row - auto res = _short_key[cid]->compare_at(0, 0, *_seek_block[cid], -1); - if (res != 0) { - return res; + auto ord = key.field(cid) <=> (*_seek_block[cid])[0]; + if (ord != std::strong_ordering::equal) { + return ord < 0 ? -1 : 1; } } return 0; @@ -453,9 +417,6 @@ class SegmentIterator : public RowwiseIterator { // only used in `_get_row_ranges_by_keys` MutableColumns _seek_block; - //todo(wb) remove this field after Rowcursor is removed - MutableColumns _short_key; - io::FileReaderSPtr _file_reader; // char_type or array type columns cid diff --git a/be/src/storage/segment/segment_writer.cpp b/be/src/storage/segment/segment_writer.cpp index aa010e8ed99331..3f5fae5792c45c 100644 --- a/be/src/storage/segment/segment_writer.cpp +++ b/be/src/storage/segment/segment_writer.cpp @@ -35,6 +35,8 @@ #include "core/block/block.h" #include "core/block/column_with_type_and_name.h" #include "core/column/column_nullable.h" +#include "core/data_type/primitive_type.h" +#include "core/field.h" #include "core/types.h" #include "core/value/vdatetime_value.h" #include "exec/common/variant_util.h" @@ -57,7 +59,6 @@ #include "storage/olap_common.h" #include "storage/olap_define.h" #include "storage/partial_update_info.h" -#include "storage/row_cursor.h" // RowCursor // IWYU pragma: keep #include "storage/rowset/rowset_writer_context.h" // RowsetWriterContext #include "storage/rowset/segment_creator.h" #include "storage/segment/column_writer.h" // ColumnWriter @@ -890,37 +891,6 @@ std::string SegmentWriter::_encode_keys(const std::vector(cid)); - RETURN_IF_ERROR(_column_writers[cid]->append(cell)); - } - std::string full_encoded_key; - row.encode_key(&full_encoded_key, _num_sort_key_columns); - if (_tablet_schema->has_sequence_col()) { - full_encoded_key.push_back(KEY_NORMAL_MARKER); - auto cid = _tablet_schema->sequence_col_idx(); - auto cell = row.cell(cid); - row.schema()->column(cid)->full_encode_ascending(cell.cell_ptr(), &full_encoded_key); - } - - if (_is_mow_with_cluster_key()) { - return Status::InternalError( - "SegmentWriter::append_row does not support mow tables with cluster key"); - } else if (_is_mow()) { - RETURN_IF_ERROR(_primary_key_index_builder->add_item(full_encoded_key)); - } else { - // At the beginning of one block, so add a short key index entry - if ((_num_rows_written % _opts.num_rows_per_block) == 0) { - std::string encoded_key; - row.encode_key(&encoded_key, _num_short_key_columns); - RETURN_IF_ERROR(_short_key_index_builder->add_item(encoded_key)); - } - set_min_max_key(full_encoded_key); - } - ++_num_rows_written; - return Status::OK(); -} // TODO(lingbin): Currently this function does not include the size of various indexes, // We should make this more precise. diff --git a/be/src/storage/segment/segment_writer.h b/be/src/storage/segment/segment_writer.h index fa11947bdfa770..ebda86e3c13872 100644 --- a/be/src/storage/segment/segment_writer.h +++ b/be/src/storage/segment/segment_writer.h @@ -89,8 +89,6 @@ class SegmentWriter { // for vertical compaction Status init(const std::vector& col_ids, bool has_key); - Status append_row(const RowCursor& row); - Status append_block(const Block* block, size_t row_pos, size_t num_rows); Status probe_key_for_mow(std::string key, std::size_t segment_pos, bool have_input_seq_column, bool have_delete_sign, @@ -151,6 +149,7 @@ class SegmentWriter { uint64_t primary_keys_size() const { return _primary_keys_size; } private: + friend class TestSegmentWriter; DISALLOW_COPY_AND_ASSIGN(SegmentWriter); Status _create_column_writer(uint32_t cid, const TabletColumn& column, const TabletSchemaSPtr& schema); diff --git a/be/src/storage/tablet/tablet_reader.cpp b/be/src/storage/tablet/tablet_reader.cpp index 25e72405553369..d56e37672f8e20 100644 --- a/be/src/storage/tablet/tablet_reader.cpp +++ b/be/src/storage/tablet/tablet_reader.cpp @@ -67,23 +67,6 @@ void TabletReader::ReaderParams::check_validation() const { } } -std::string TabletReader::ReaderParams::to_string() const { - std::stringstream ss; - ss << "tablet=" << tablet->tablet_id() << " reader_type=" << int(reader_type) - << " aggregation=" << aggregation << " version=" << version - << " start_key_include=" << start_key_include << " end_key_include=" << end_key_include; - - for (const auto& key : start_key) { - ss << " keys=" << key; - } - - for (const auto& key : end_key) { - ss << " end_keys=" << key; - } - - return ss.str(); -} - Status TabletReader::init(const ReaderParams& read_params) { SCOPED_RAW_TIMER(&_stats.tablet_reader_init_timer_ns); @@ -371,17 +354,12 @@ Status TabletReader::_init_keys_param(const ReaderParams& read_params) { read_params.start_key[i].size(), scan_key_size); } - Status res = _keys_param.start_keys[i].init_scan_key( - _tablet_schema, read_params.start_key[i].values(), schema); + Status res = + _keys_param.start_keys[i].init(_tablet_schema, read_params.start_key[i], schema); if (!res.ok()) { LOG(WARNING) << "fail to init row cursor. res = " << res; return res; } - res = _keys_param.start_keys[i].from_tuple(read_params.start_key[i]); - if (!res.ok()) { - LOG(WARNING) << "fail to init row cursor from Keys. res=" << res << "key_index=" << i; - return res; - } } size_t end_key_size = read_params.end_key.size(); @@ -394,18 +372,11 @@ Status TabletReader::_init_keys_param(const ReaderParams& read_params) { read_params.end_key[i].size(), scan_key_size); } - Status res = _keys_param.end_keys[i].init_scan_key(_tablet_schema, - read_params.end_key[i].values(), schema); + Status res = _keys_param.end_keys[i].init(_tablet_schema, read_params.end_key[i], schema); if (!res.ok()) { LOG(WARNING) << "fail to init row cursor. res = " << res; return res; } - - res = _keys_param.end_keys[i].from_tuple(read_params.end_key[i]); - if (!res.ok()) { - LOG(WARNING) << "fail to init row cursor from Keys. res=" << res << " key_index=" << i; - return res; - } } //TODO:check the valid of start_key and end_key.(eg. start_key <= end_key) diff --git a/be/src/storage/tablet/tablet_reader.h b/be/src/storage/tablet/tablet_reader.h index 8616d8eeac0dfd..b0fbacfac7ca44 100644 --- a/be/src/storage/tablet/tablet_reader.h +++ b/be/src/storage/tablet/tablet_reader.h @@ -70,12 +70,20 @@ class VExprContext; // // NOTE: if you are not sure if you can use it, please don't use this function. inline int compare_row_key(const RowCursor& lhs, const RowCursor& rhs) { - auto cmp_cids = std::min(lhs.schema()->num_column_ids(), rhs.schema()->num_column_ids()); + auto cmp_cids = std::min(lhs.field_count(), rhs.field_count()); for (uint32_t cid = 0; cid < cmp_cids; ++cid) { - auto res = lhs.schema()->column(cid)->compare_cell(lhs.cell(cid), rhs.cell(cid)); - if (res != 0) { - return res; + const auto& lf = lhs.field(cid); + const auto& rf = rhs.field(cid); + // Handle nulls: null < non-null + if (lf.is_null() != rf.is_null()) { + return lf.is_null() ? -1 : 1; } + if (lf.is_null()) { + continue; // both null + } + auto cmp = lf <=> rf; + if (cmp < 0) return -1; + if (cmp > 0) return 1; } return 0; } @@ -191,8 +199,6 @@ class TabletReader { void check_validation() const; - std::string to_string() const; - int64_t batch_size = -1; std::map virtual_column_exprs; diff --git a/be/src/storage/types.h b/be/src/storage/types.h index 39b51246df41ee..9fa092996875d8 100644 --- a/be/src/storage/types.h +++ b/be/src/storage/types.h @@ -49,7 +49,6 @@ #include "storage/olap_common.h" #include "storage/olap_define.h" #include "util/slice.h" -#include "util/string_parser.hpp" namespace doris { #include "common/compile_check_begin.h" @@ -60,8 +59,6 @@ class ColumnMetaPB; class TabletColumn; -extern bool is_olap_string_type(FieldType field_type); - class TypeInfo; using TypeInfoPtr = std::unique_ptr; @@ -76,11 +73,6 @@ class TypeInfo { virtual void deep_copy(void* dest, const void* src, Arena& arena) const = 0; - virtual void direct_copy(void* dest, const void* src) const = 0; - - virtual Status from_string(void* buf, const std::string& scan_key, const int precision = 0, - const int scale = 0) const = 0; - virtual void set_to_max(void* buf) const = 0; virtual void set_to_min(void* buf) const = 0; @@ -97,13 +89,6 @@ class ScalarTypeInfo : public TypeInfo { _deep_copy(dest, src, arena); } - void direct_copy(void* dest, const void* src) const override { _direct_copy(dest, src); } - - Status from_string(void* buf, const std::string& scan_key, const int precision = 0, - const int scale = 0) const override { - return _from_string(buf, scan_key, precision, scale); - } - void set_to_max(void* buf) const override { _set_to_max(buf); } void set_to_min(void* buf) const override { _set_to_min(buf); } size_t size() const override { return _size; } @@ -114,8 +99,6 @@ class ScalarTypeInfo : public TypeInfo { ScalarTypeInfo(TypeTraitsClass t) : _cmp(TypeTraitsClass::cmp), _deep_copy(TypeTraitsClass::deep_copy), - _direct_copy(TypeTraitsClass::direct_copy), - _from_string(TypeTraitsClass::from_string), _set_to_max(TypeTraitsClass::set_to_max), _set_to_min(TypeTraitsClass::set_to_min), _size(TypeTraitsClass::size), @@ -125,10 +108,6 @@ class ScalarTypeInfo : public TypeInfo { int (*_cmp)(const void* left, const void* right); void (*_deep_copy)(void* dest, const void* src, Arena& arena); - void (*_direct_copy)(void* dest, const void* src); - - Status (*_from_string)(void* buf, const std::string& scan_key, const int precision, - const int scale); void (*_set_to_max)(void* buf); void (*_set_to_min)(void* buf); @@ -223,63 +202,6 @@ class ArrayTypeInfo : public TypeInfo { } } - void direct_copy(void* dest, const void* src) const override { - auto dest_value = static_cast(dest); - // NOTICE: The address pointed by null_signs of the dest_value can NOT be modified here. - auto base = reinterpret_cast(dest_value->mutable_null_signs()); - direct_copy(&base, dest, src); - } - - void direct_copy(uint8_t** base, void* dest, const void* src) const { - auto dest_value = static_cast(dest); - auto src_value = static_cast(src); - - auto nulls_size = src_value->has_null() ? src_value->length() : 0; - dest_value->set_data(src_value->length() ? (*base + nulls_size) : nullptr); - dest_value->set_length(src_value->length()); - dest_value->set_has_null(src_value->has_null()); - if (src_value->has_null()) { - // direct copy null_signs - dest_value->set_null_signs(reinterpret_cast(*base)); - memcpy(dest_value->mutable_null_signs(), src_value->null_signs(), src_value->length()); - } - *base += nulls_size + src_value->length() * _item_type_info->size(); - - // Direct copy item. - if (_item_type_info->type() == FieldType::OLAP_FIELD_TYPE_ARRAY) { - for (uint32_t i = 0; i < src_value->length(); ++i) { - if (dest_value->is_null_at(i)) { - continue; - } - dynamic_cast(_item_type_info.get()) - ->direct_copy(base, (uint8_t*)(dest_value->mutable_data()) + i * _item_size, - (uint8_t*)(src_value->data()) + i * _item_size); - } - } else { - for (uint32_t i = 0; i < src_value->length(); ++i) { - if (dest_value->is_null_at(i)) { - continue; - } - auto dest_address = (uint8_t*)(dest_value->mutable_data()) + i * _item_size; - auto src_address = (uint8_t*)(src_value->data()) + i * _item_size; - if (is_olap_string_type(_item_type_info->type())) { - auto dest_slice = reinterpret_cast(dest_address); - auto src_slice = reinterpret_cast(src_address); - dest_slice->data = reinterpret_cast(*base); - dest_slice->size = src_slice->size; - *base += src_slice->size; - } - _item_type_info->direct_copy(dest_address, src_address); - } - } - } - - Status from_string(void* buf, const std::string& scan_key, const int precision = 0, - const int scale = 0) const override { - return Status::Error( - "ArrayTypeInfo not support from_string"); - } - void set_to_max(void* buf) const override { DCHECK(false) << "set_to_max of list is not implemented."; } @@ -333,16 +255,6 @@ class MapTypeInfo : public TypeInfo { void deep_copy(void* dest, const void* src, Arena& arena) const override { DCHECK(false); } - void direct_copy(void* dest, const void* src) const override { CHECK(false); } - - void direct_copy(uint8_t** base, void* dest, const void* src) const { CHECK(false); } - - Status from_string(void* buf, const std::string& scan_key, const int precision = 0, - const int scale = 0) const override { - return Status::Error( - "MapTypeInfo not support from_string"); - } - void set_to_max(void* buf) const override { DCHECK(false) << "set_to_max of list is not implemented."; } @@ -454,58 +366,6 @@ class StructTypeInfo : public TypeInfo { } } - void direct_copy(void* dest, const void* src) const override { - auto dest_value = static_cast(dest); - auto base = reinterpret_cast(dest_value->mutable_values()); - direct_copy(&base, dest, src); - } - - void direct_copy(uint8_t** base, void* dest, const void* src) const { - auto dest_value = static_cast(dest); - auto src_value = static_cast(src); - - dest_value->set_size(src_value->size()); - dest_value->set_has_null(src_value->has_null()); - *base += src_value->size() * sizeof(*src_value->values()); - - for (uint32_t i = 0; i < src_value->size(); ++i) { - dest_value->set_child_value(nullptr, i); - if (src_value->is_null_at(i)) continue; - dest_value->set_child_value(*base, i); - *base += _type_infos[i]->size(); - } - - for (uint32_t i = 0; i < src_value->size(); ++i) { - if (dest_value->is_null_at(i)) { - continue; - } - auto dest_address = dest_value->mutable_child_value(i); - auto src_address = src_value->child_value(i); - if (_type_infos[i]->type() == FieldType::OLAP_FIELD_TYPE_STRUCT) { - dynamic_cast(_type_infos[i].get()) - ->direct_copy(base, dest_address, src_address); - } else if (_type_infos[i]->type() == FieldType::OLAP_FIELD_TYPE_ARRAY) { - dynamic_cast(_type_infos[i].get()) - ->direct_copy(base, dest_address, src_address); - } else { - if (is_olap_string_type(_type_infos[i]->type())) { - auto dest_slice = reinterpret_cast(dest_address); - auto src_slice = reinterpret_cast(src_address); - dest_slice->data = reinterpret_cast(*base); - dest_slice->size = src_slice->size; - *base += src_slice->size; - } - _type_infos[i]->direct_copy(dest_address, src_address); - } - } - } - - Status from_string(void* buf, const std::string& scan_key, const int precision = 0, - const int scale = 0) const override { - return Status::Error( - "StructTypeInfo not support from_string"); - } - void set_to_max(void* buf) const override { DCHECK(false) << "set_to_max of list is not implemented."; } @@ -738,10 +598,6 @@ struct BaseFieldTypeTraits : public CppTypeTraits { memcpy(dest, src, sizeof(CppType)); } - static inline void direct_copy(void* dest, const void* src) { - memcpy(dest, src, sizeof(CppType)); - } - static inline void set_to_max(void* buf) { set_cpp_type_value(buf, type_limit::max()); } @@ -749,16 +605,6 @@ struct BaseFieldTypeTraits : public CppTypeTraits { static inline void set_to_min(void* buf) { set_cpp_type_value(buf, type_limit::min()); } - - static Status from_string(void* buf, const std::string& scan_key, const int precision, - const int scale) { - CppType value = 0; - if (scan_key.length() > 0) { - value = static_cast(strtol(scan_key.c_str(), nullptr, 10)); - } - set_cpp_type_value(buf, value); - return Status::OK(); - } }; // Using NumericFieldtypeTraits to Derived code for FieldType::OLAP_FIELD_TYPE_XXXINT, FieldType::OLAP_FIELD_TYPE_FLOAT, @@ -788,55 +634,11 @@ struct FieldTypeTraits template <> struct FieldTypeTraits : public NumericFieldTypeTraits { - static Status from_string(void* buf, const std::string& scan_key, const int precision, - const int scale) { - int128_t value = 0; - - const char* value_string = scan_key.c_str(); - char* end = nullptr; - value = strtol(value_string, &end, 10); - if (*end != 0) { - value = 0; - } else if (value > LONG_MIN && value < LONG_MAX) { - // use strtol result directly - } else { - bool is_negative = false; - if (*value_string == '-' || *value_string == '+') { - if (*(value_string++) == '-') { - is_negative = true; - } - } - - uint128_t current = 0; - uint128_t max_int128 = ~((int128_t)(1) << 127); - while (*value_string != 0) { - if (current > max_int128 / 10) { - break; - } - - current = current * 10 + (*(value_string++) - '0'); - } - if (*value_string != 0 || (!is_negative && current > max_int128) || - (is_negative && current > max_int128 + 1)) { - current = 0; - } - - value = is_negative ? -current : current; - } - - *reinterpret_cast(buf) = value; - - return Status::OK(); - } - // GCC7.3 will generate movaps instruction, which will lead to SEGV when buf is // not aligned to 16 byte static void deep_copy(void* dest, const void* src, Arena& arena) { *reinterpret_cast(dest) = *reinterpret_cast(src); } - static void direct_copy(void* dest, const void* src) { - *reinterpret_cast(dest) = *reinterpret_cast(src); - } static void set_to_max(void* buf) { *reinterpret_cast(buf) = ~((int128_t)(1) << 127); @@ -849,17 +651,6 @@ struct FieldTypeTraits template <> struct FieldTypeTraits : public BaseFieldTypeTraits { - static Status from_string(void* buf, const std::string& scan_key, const int precision, - const int scale) { - uint32_t value; - if (!IPv4Value::from_string(value, scan_key)) { - return Status::Error( - "FieldTypeTraits::from_string meet PARSE_FAILURE"); - } - *reinterpret_cast(buf) = value; - return Status::OK(); - } - static void set_to_max(void* buf) { *reinterpret_cast(buf) = 0xFFFFFFFF; // 255.255.255.255 } @@ -872,17 +663,6 @@ struct FieldTypeTraits template <> struct FieldTypeTraits : public BaseFieldTypeTraits { - static Status from_string(void* buf, const std::string& scan_key, const int precision, - const int scale) { - uint128_t value; - if (!IPv6Value::from_string(value, scan_key)) { - return Status::Error( - "FieldTypeTraits::from_string meet PARSE_FAILURE"); - } - memcpy(buf, &value, sizeof(uint128_t)); - return Status::OK(); - } - static void set_to_max(void* buf) { *reinterpret_cast(buf) = -1; // ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff } @@ -892,42 +672,9 @@ struct FieldTypeTraits } }; -template <> -struct FieldTypeTraits - : public NumericFieldTypeTraits { - static Status from_string(void* buf, const std::string& scan_key, const int precision, - const int scale) { - CppType value = 0.0f; - if (scan_key.length() > 0) { - value = static_cast(atof(scan_key.c_str())); - } - *reinterpret_cast(buf) = value; - return Status::OK(); - } -}; - -template <> -struct FieldTypeTraits - : public NumericFieldTypeTraits { - static Status from_string(void* buf, const std::string& scan_key, const int precision, - const int scale) { - CppType value = 0.0; - if (scan_key.length() > 0) { - value = atof(scan_key.c_str()); - } - *reinterpret_cast(buf) = value; - return Status::OK(); - } -}; - template <> struct FieldTypeTraits : public BaseFieldTypeTraits { - static Status from_string(void* buf, const std::string& scan_key, const int precision, - const int scale) { - CppType* data_ptr = reinterpret_cast(buf); - return data_ptr->from_string(scan_key); - } static void set_to_max(void* buf) { CppType* data = reinterpret_cast(buf); data->integer = 999999999999999999L; @@ -940,96 +687,9 @@ struct FieldTypeTraits } }; -template <> -struct FieldTypeTraits - : public BaseFieldTypeTraits { - static Status from_string(void* buf, const std::string& scan_key, const int precision, - const int scale) { - StringParser::ParseResult result = StringParser::PARSE_SUCCESS; - int32_t value = StringParser::string_to_decimal( - scan_key.c_str(), scan_key.size(), 9, scale, &result); - - if (result == StringParser::PARSE_FAILURE) { - return Status::Error( - "FieldTypeTraits::from_string meet PARSE_FAILURE"); - } - *reinterpret_cast(buf) = value; - return Status::OK(); - } -}; - -template <> -struct FieldTypeTraits - : public BaseFieldTypeTraits { - static Status from_string(void* buf, const std::string& scan_key, const int precision, - const int scale) { - StringParser::ParseResult result = StringParser::PARSE_SUCCESS; - int64_t value = StringParser::string_to_decimal( - scan_key.c_str(), scan_key.size(), 18, scale, &result); - if (result == StringParser::PARSE_FAILURE) { - return Status::Error( - "FieldTypeTraits::from_string meet PARSE_FAILURE"); - } - *reinterpret_cast(buf) = value; - return Status::OK(); - } -}; - -template <> -struct FieldTypeTraits - : public BaseFieldTypeTraits { - static Status from_string(void* buf, const std::string& scan_key, const int precision, - const int scale) { - StringParser::ParseResult result = StringParser::PARSE_SUCCESS; - int128_t value = StringParser::string_to_decimal( - scan_key.c_str(), scan_key.size(), 38, scale, &result); - if (result == StringParser::PARSE_FAILURE) { - return Status::Error( - "FieldTypeTraits::from_string meet PARSE_FAILURE"); - } - *reinterpret_cast(buf) = value; - return Status::OK(); - } -}; - -template <> -struct FieldTypeTraits - : public BaseFieldTypeTraits { - static Status from_string(void* buf, const std::string& scan_key, const int precision, - const int scale) { - StringParser::ParseResult result = StringParser::PARSE_SUCCESS; - auto value = StringParser::string_to_decimal( - scan_key.c_str(), cast_set(scan_key.size()), - BeConsts::MAX_DECIMAL256_PRECISION, scale, &result); - if (result == StringParser::PARSE_FAILURE) { - return Status::Error( - "FieldTypeTraits::from_string meet PARSE_FAILURE"); - } - *reinterpret_cast(buf) = value; - return Status::OK(); - } -}; - template <> struct FieldTypeTraits : public BaseFieldTypeTraits { - static Status from_string(void* buf, const std::string& scan_key, const int precision, - const int scale) { - tm time_tm; - char* res = strptime(scan_key.c_str(), "%Y-%m-%d", &time_tm); - - if (nullptr != res) { - int value = (time_tm.tm_year + 1900) * 16 * 32 + (time_tm.tm_mon + 1) * 32 + - time_tm.tm_mday; - *reinterpret_cast(buf) = value; - } else { - // 1400 - 01 - 01 - *reinterpret_cast(buf) = 716833; - } - - return Status::OK(); - } - static void set_to_max(void* buf) { // max is 9999 * 16 * 32 + 12 * 32 + 31; *reinterpret_cast(buf) = 5119903; @@ -1043,22 +703,6 @@ struct FieldTypeTraits template <> struct FieldTypeTraits : public BaseFieldTypeTraits { - static Status from_string(void* buf, const std::string& scan_key, const int precision, - const int scale) { - tm time_tm; - char* res = strptime(scan_key.c_str(), "%Y-%m-%d", &time_tm); - - if (nullptr != res) { - uint32_t value = - ((time_tm.tm_year + 1900) << 9) | ((time_tm.tm_mon + 1) << 5) | time_tm.tm_mday; - *reinterpret_cast(buf) = value; - } else { - *reinterpret_cast(buf) = MIN_DATE_V2; - } - - return Status::OK(); - } - static void set_to_max(void* buf) { // max is 9999 * 16 * 32 + 12 * 32 + 31; *reinterpret_cast(buf) = MAX_DATE_V2; @@ -1072,21 +716,6 @@ struct FieldTypeTraits template <> struct FieldTypeTraits : public BaseFieldTypeTraits { - static Status from_string(void* buf, const std::string& scan_key, const int precision, - const int scale) { - DateV2Value datetimev2_value; - std::string date_format = "%Y-%m-%d %H:%i:%s.%f"; - - if (datetimev2_value.from_date_format_str(date_format.data(), date_format.size(), - scan_key.data(), scan_key.size())) { - *reinterpret_cast(buf) = datetimev2_value.to_date_int_val(); - } else { - *reinterpret_cast(buf) = MIN_DATETIME_V2; - } - - return Status::OK(); - } - static void set_to_max(void* buf) { // max is 9999 * 16 * 32 + 12 * 32 + 31; *reinterpret_cast(buf) = MAX_DATETIME_V2; @@ -1100,25 +729,6 @@ struct FieldTypeTraits template <> struct FieldTypeTraits : public BaseFieldTypeTraits { - static Status from_string(void* buf, const std::string& scan_key, const int precision, - const int scale) { - tm time_tm; - char* res = strptime(scan_key.c_str(), "%Y-%m-%d %H:%M:%S", &time_tm); - - if (nullptr != res) { - CppType value = ((time_tm.tm_year + 1900) * 10000L + (time_tm.tm_mon + 1) * 100L + - time_tm.tm_mday) * - 1000000L + - time_tm.tm_hour * 10000L + time_tm.tm_min * 100L + time_tm.tm_sec; - *reinterpret_cast(buf) = value; - } else { - // 1400 - 01 - 01 - *reinterpret_cast(buf) = 14000101000000L; - } - - return Status::OK(); - } - static void set_to_max(void* buf) { // 9999-12-31 23:59:59 *reinterpret_cast(buf) = 99991231235959L; @@ -1129,20 +739,6 @@ struct FieldTypeTraits template <> struct FieldTypeTraits : public BaseFieldTypeTraits { - static Status from_string(void* buf, const std::string& scan_key, const int precision, - const int scale) { - CastParameters params; - TimestampTzValue value; - auto tz = cctz::utc_time_zone(); - if (!CastToTimstampTz::from_string(StringRef(scan_key), value, params, &tz, 6)) { - return Status::Error("parse timestamptz error, value: {}", - scan_key); - } - *reinterpret_cast(buf) = value.to_date_int_val(); - - return Status::OK(); - } - static void set_to_max(void* buf) { // max is 9999 * 16 * 32 + 12 * 32 + 31; *reinterpret_cast(buf) = MAX_DATETIME_V2; @@ -1161,31 +757,6 @@ struct FieldTypeTraits auto r_slice = reinterpret_cast(right); return l_slice->compare(*r_slice); } - static Status from_string(void* buf, const std::string& scan_key, const int precision, - const int scale) { - size_t value_len = scan_key.length(); - if (value_len > OLAP_VARCHAR_MAX_LENGTH) { - return Status::Error( - "the len of value string is too long, len={}, max_len={}", value_len, - OLAP_VARCHAR_MAX_LENGTH); - } - - auto slice = reinterpret_cast(buf); - memcpy(slice->data, scan_key.c_str(), value_len); - if (slice->size < value_len) { - /* - * CHAR type is of fixed length. Size in slice can be modified - * only if value_len is greater than the fixed length. ScanKey - * inputted by user may be greater than fixed length. - */ - slice->size = value_len; - } else { - // append \0 to the tail - memset(slice->data + value_len, 0, slice->size - value_len); - } - return Status::OK(); - } - static void deep_copy(void* dest, const void* src, Arena& arena) { auto l_slice = reinterpret_cast(dest); auto r_slice = reinterpret_cast(src); @@ -1194,13 +765,6 @@ struct FieldTypeTraits l_slice->size = r_slice->size; } - static void direct_copy(void* dest, const void* src) { - auto l_slice = reinterpret_cast(dest); - auto r_slice = reinterpret_cast(src); - memcpy(l_slice->data, r_slice->data, r_slice->size); - l_slice->size = r_slice->size; - } - // Using field.set_to_max to set varchar/char,not here. static void (*set_to_max)(void*); @@ -1213,21 +777,6 @@ struct FieldTypeTraits template <> struct FieldTypeTraits : public FieldTypeTraits { - static Status from_string(void* buf, const std::string& scan_key, const int precision, - const int scale) { - size_t value_len = scan_key.length(); - if (value_len > OLAP_VARCHAR_MAX_LENGTH) { - return Status::Error( - "the len of value string is too long, len={}, max_len={}", value_len, - OLAP_VARCHAR_MAX_LENGTH); - } - - auto slice = reinterpret_cast(buf); - memcpy(slice->data, scan_key.c_str(), value_len); - slice->size = value_len; - return Status::OK(); - } - static void set_to_min(void* buf) { auto slice = reinterpret_cast(buf); slice->size = 0; @@ -1237,21 +786,6 @@ struct FieldTypeTraits template <> struct FieldTypeTraits : public FieldTypeTraits { - static Status from_string(void* buf, const std::string& scan_key, const int precision, - const int scale) { - size_t value_len = scan_key.length(); - if (value_len > config::string_type_length_soft_limit_bytes) { - return Status::Error( - "the len of value string is too long, len={}, max_len={}", value_len, - config::string_type_length_soft_limit_bytes); - } - - auto slice = reinterpret_cast(buf); - memcpy(slice->data, scan_key.c_str(), value_len); - slice->size = value_len; - return Status::OK(); - } - static void set_to_min(void* buf) { auto slice = reinterpret_cast(buf); slice->size = 0; @@ -1266,13 +800,6 @@ struct FieldTypeTraits return -1; // always update ? } - static Status from_string(void* buf, const std::string& scan_key, const int precision, - const int scale) { - // TODO support schema change - return Status::Error( - "FieldTypeTraits not support from_string"); - } - static void set_to_min(void* buf) { auto slice = reinterpret_cast(buf); slice->size = 0; diff --git a/be/test/storage/delete/delete_bitmap_calculator_test.cpp b/be/test/storage/delete/delete_bitmap_calculator_test.cpp index 2d7d56044e4018..81b16cc8b3d01c 100644 --- a/be/test/storage/delete/delete_bitmap_calculator_test.cpp +++ b/be/test/storage/delete/delete_bitmap_calculator_test.cpp @@ -28,6 +28,7 @@ #include #include +#include "core/field.h" #include "gtest/gtest_pred_impl.h" #include "io/fs/file_reader.h" #include "io/fs/file_writer.h" @@ -37,6 +38,7 @@ #include "storage/row_cursor.h" #include "storage/segment/segment.h" #include "storage/segment/segment_writer.h" +#include "storage/segment/test_segment_writer.h" #include "storage/storage_engine.h" #include "storage/tablet/tablet_meta.h" #include "storage/tablet/tablet_schema.h" @@ -48,7 +50,7 @@ using namespace ErrorCode; static std::string kSegmentDir = "./ut_dir/delete_bitmap_calculator_test"; static RowsetId rowset_id {0}; -using Generator = std::function; +using Generator = std::function; TabletColumnPtr create_int_sequence_value(int32_t id, bool is_nullable = true, bool is_bf_column = false) { @@ -75,19 +77,18 @@ void build_segment(SegmentWriterOptions opts, TabletSchemaSPtr build_schema, siz io::FileWriterPtr file_writer; Status st = fs->create_file(path, &file_writer); EXPECT_TRUE(st.ok()) << st.to_string(); - SegmentWriter writer(file_writer.get(), segment_id, build_schema, nullptr, nullptr, opts, - nullptr); + TestSegmentWriter writer(file_writer.get(), segment_id, build_schema, nullptr, nullptr, opts, + nullptr); st = writer.init(); EXPECT_TRUE(st.ok()); RowCursor row; - auto olap_st = row._init(build_schema, build_schema->num_columns()); + auto olap_st = row.init(build_schema, build_schema->num_columns()); EXPECT_EQ(Status::OK(), olap_st); for (size_t rid = 0; rid < nrows; ++rid) { for (int cid = 0; cid < build_schema->num_columns(); ++cid) { - RowCursorCell cell = row.cell(cid); - generator(rid, cid, cell); + generator(rid, cid, row.mutable_field(cid)); } EXPECT_TRUE(writer.append_row(row).ok()); } @@ -218,9 +219,8 @@ class DeleteBitmapCalculatorTest : public testing::Test { for (size_t sid = 0; sid < num_segments; ++sid) { auto& segment = segments[sid]; std::vector row_data; - auto generator = [&](size_t rid, int cid, RowCursorCell& cell) { - cell.set_not_null(); - *(int*)cell.mutable_cell_ptr() = data_map[{sid, rid}][cid]; + auto generator = [&](size_t rid, int cid, Field& field) { + field = Field::create_field(int32_t(data_map[{sid, rid}][cid])); }; build_segment(opts, tablet_schema, sid, tablet_schema, datas[sid].size(), generator, &segment, kSegmentDir); diff --git a/be/test/storage/olap_type_test.cpp b/be/test/storage/olap_type_test.cpp index 2511b5158a06e9..8789c267097b06 100644 --- a/be/test/storage/olap_type_test.cpp +++ b/be/test/storage/olap_type_test.cpp @@ -33,7 +33,6 @@ #include "exprs/function/cast/cast_to_string.h" #include "gtest/gtest_pred_impl.h" #include "storage/olap_common.h" -#include "storage/types.h" namespace doris { @@ -101,21 +100,28 @@ TEST_F(OlapTypeTest, deser_float_old) { std::numeric_limits::quiet_NaN()}; test_input_values.insert(test_input_values.end(), special_input_values.begin(), special_input_values.end()); + auto data_type_ptr = DataTypeFactory::instance().create_data_type(TYPE_FLOAT, false); + auto data_type_serde = data_type_ptr->get_serde(); std::ifstream input_file(test_data_dir + "/ser_float_3.0.txt"); EXPECT_TRUE(input_file.is_open()); std::string line; int line_index = 0; while (std::getline(input_file, line)) { - float deser_float_value = 0.0F; - auto status = FieldTypeTraits::from_string( - &deser_float_value, line, 0, 0); + Field restored_field; + auto status = data_type_serde->from_fe_string(line, restored_field); + // from_fe_string rejects NaN/Infinity strings + if (std::isnan(test_input_values[line_index]) || + std::isinf(test_input_values[line_index])) { + EXPECT_FALSE(status.ok()); + line_index++; + continue; + } EXPECT_TRUE(status.ok()) << status.to_string(); + float deser_float_value = restored_field.get(); float diff_ratio = std::abs(deser_float_value - test_input_values[line_index]) / abs(test_input_values[line_index]); EXPECT_TRUE((test_input_values[line_index] == 0 && deser_float_value == 0) || - diff_ratio < 1e-6 || - (std::isnan(deser_float_value) && std::isnan(test_input_values[line_index])) || - (std::isinf(deser_float_value) && std::isinf(test_input_values[line_index]))) + diff_ratio < 1e-6) << "expected float value: " << fmt::format("{:.9g}", test_input_values[line_index]) << ", deser float value: " << fmt::format("{:.9g}", deser_float_value) << ", diff_ratio: " << fmt::format("{:.9g}", diff_ratio); @@ -182,24 +188,29 @@ TEST_F(OlapTypeTest, deser_double_old) { std::numeric_limits::quiet_NaN()}; test_input_values.insert(test_input_values.end(), special_input_values.begin(), special_input_values.end()); + auto data_type_ptr = DataTypeFactory::instance().create_data_type(TYPE_DOUBLE, false); + auto data_type_serde = data_type_ptr->get_serde(); std::ifstream input_file(test_data_dir + "/ser_double_3.0.txt"); EXPECT_TRUE(input_file.is_open()); std::string line; int line_index = 0; while (std::getline(input_file, line)) { - double deser_float_value = 0.0; - auto status = FieldTypeTraits::from_string( - &deser_float_value, line, 0, 0); + Field restored_field; + auto status = data_type_serde->from_fe_string(line, restored_field); + // from_fe_string rejects NaN/Infinity strings, and also rejects + // double::max()/lowest() whose string representation parses to Infinity + if (std::isnan(test_input_values[line_index]) || + std::isinf(test_input_values[line_index])) { + EXPECT_FALSE(status.ok()); + line_index++; + continue; + } EXPECT_TRUE(status.ok()) << status.to_string(); + double deser_float_value = restored_field.get(); double diff_ratio = std::abs(deser_float_value - test_input_values[line_index]) / abs(test_input_values[line_index]); EXPECT_TRUE((test_input_values[line_index] == 0 && deser_float_value == 0) || - diff_ratio < 1e-15 || - (std::isnan(deser_float_value) && std::isnan(test_input_values[line_index])) || - (std::isinf(deser_float_value) && - (std::isinf(test_input_values[line_index]) || - test_input_values[line_index] == std::numeric_limits::max() || - test_input_values[line_index] == std::numeric_limits::lowest()))) + diff_ratio < 1e-15) << "expected double value: " << fmt::format("{:.17g}", test_input_values[line_index]) << ", deser double value: " << fmt::format("{:.17g}", deser_float_value) @@ -372,14 +383,17 @@ TEST_F(OlapTypeTest, ser_deser_float) { auto field = Field::create_field(float_value); auto result_str = data_type_serde->to_olap_string(field); EXPECT_EQ(result_str, expected_str); - float deser_float_value = 0.0F; - auto status = FieldTypeTraits::from_string( - &deser_float_value, result_str, 0, 0); + Field restored_field; + auto status = data_type_serde->from_fe_string(result_str, restored_field); + // from_fe_string rejects NaN/Infinity strings + if (std::isnan(float_value) || std::isinf(float_value)) { + EXPECT_FALSE(status.ok()); + continue; + } EXPECT_TRUE(status.ok()) << status.to_string(); + float deser_float_value = restored_field.get(); float diff_ratio = std::abs(deser_float_value - float_value) / abs(float_value); - EXPECT_TRUE((float_value == 0 && deser_float_value == 0) || diff_ratio < 1e-6 || - (std::isnan(deser_float_value) && std::isnan(float_value)) || - (std::isinf(deser_float_value) && std::isinf(float_value))) + EXPECT_TRUE((float_value == 0 && deser_float_value == 0) || diff_ratio < 1e-6) << "expected float value: " << fmt::format("{:.9g}", float_value) << ", expected float str: " << expected_str << ", deser float value: " << fmt::format("{:.9g}", deser_float_value) @@ -572,17 +586,20 @@ TEST_F(OlapTypeTest, ser_deser_double) { auto field = Field::create_field(float_value); auto result_str = data_type_serde->to_olap_string(field); EXPECT_EQ(result_str, expected_str); - double deser_float_value = 0.0; - auto status = FieldTypeTraits::from_string( - &deser_float_value, result_str, 0, 0); + Field restored_field; + auto status = data_type_serde->from_fe_string(result_str, restored_field); + // from_fe_string rejects NaN/Infinity strings, and also rejects + // double::max()/lowest() whose string representation parses to Infinity + if (std::isnan(float_value) || std::isinf(float_value) || + float_value == std::numeric_limits::max() || + float_value == std::numeric_limits::lowest()) { + EXPECT_FALSE(status.ok()); + continue; + } EXPECT_TRUE(status.ok()) << status.to_string(); + double deser_float_value = restored_field.get(); double diff_ratio = std::abs(deser_float_value - float_value) / abs(float_value); - EXPECT_TRUE( - (float_value == 0 && deser_float_value == 0) || diff_ratio < 1e-15 || - (std::isnan(deser_float_value) && std::isnan(float_value)) || - (std::isinf(deser_float_value) && - (std::isinf(float_value) || float_value == std::numeric_limits::max() || - float_value == std::numeric_limits::lowest()))) + EXPECT_TRUE((float_value == 0 && deser_float_value == 0) || diff_ratio < 1e-15) << "expected double value: " << fmt::format("{:.17g}", float_value) << ", expected double str: " << expected_str << ", deser double value: " << fmt::format("{:.17g}", deser_float_value) diff --git a/be/test/storage/row_cursor_test.cpp b/be/test/storage/row_cursor_test.cpp index 358afa7de32e7f..26b1204a681868 100644 --- a/be/test/storage/row_cursor_test.cpp +++ b/be/test/storage/row_cursor_test.cpp @@ -19,186 +19,196 @@ #include +#include +#include +#include + #include "common/object_pool.h" +#include "core/decimal12.h" +#include "core/extended_types.h" +#include "core/field.h" #include "core/packed_int128.h" -#include "storage/row_cursor_cell.h" +#include "core/uint24.h" +#include "core/wide_integer.h" #include "storage/schema.h" #include "storage/tablet/tablet_schema.h" #include "storage/tablet/tablet_schema_helper.h" #include "util/debug_util.h" +#include "util/slice.h" namespace doris { -void set_tablet_schema_for_init(TabletSchemaSPtr tablet_schema) { - TabletSchemaPB tablet_schema_pb; - ColumnPB* column_1 = tablet_schema_pb.add_column(); - column_1->set_unique_id(1); - column_1->set_name("column_1"); - column_1->set_type("TINYINT"); - column_1->set_is_key(true); - column_1->set_is_nullable(true); - column_1->set_length(1); - column_1->set_index_length(1); +// Helper functions for column types not covered by tablet_schema_helper.h +static TabletColumnPtr create_float_key(int32_t id, bool is_nullable = true) { + auto column = std::make_shared(); + column->_unique_id = id; + column->_col_name = std::to_string(id); + column->_type = FieldType::OLAP_FIELD_TYPE_FLOAT; + column->_is_key = true; + column->_is_nullable = is_nullable; + column->_length = 4; + column->_index_length = 4; + return column; +} - ColumnPB* column_2 = tablet_schema_pb.add_column(); - column_2->set_unique_id(2); - column_2->set_name("column_2"); - column_2->set_type("SMALLINT"); - column_2->set_is_key(true); - column_2->set_is_nullable(true); - column_2->set_length(2); - column_2->set_index_length(2); - column_2->set_default_value("0"); +static TabletColumnPtr create_double_key(int32_t id, bool is_nullable = true) { + auto column = std::make_shared(); + column->_unique_id = id; + column->_col_name = std::to_string(id); + column->_type = FieldType::OLAP_FIELD_TYPE_DOUBLE; + column->_is_key = true; + column->_is_nullable = is_nullable; + column->_length = 8; + column->_index_length = 8; + return column; +} - ColumnPB* column_3 = tablet_schema_pb.add_column(); - column_3->set_unique_id(3); - column_3->set_name("column_3"); - column_3->set_type("INT"); - column_3->set_is_key(true); - column_3->set_is_nullable(true); - column_3->set_length(4); - column_3->set_index_length(4); +static TabletColumnPtr create_bigint_key(int32_t id, bool is_nullable = true) { + auto column = std::make_shared(); + column->_unique_id = id; + column->_col_name = std::to_string(id); + column->_type = FieldType::OLAP_FIELD_TYPE_BIGINT; + column->_is_key = true; + column->_is_nullable = is_nullable; + column->_length = 8; + column->_index_length = 8; + return column; +} - ColumnPB* column_4 = tablet_schema_pb.add_column(); - column_4->set_unique_id(4); - column_4->set_name("column_4"); - column_4->set_type("DATE"); - column_4->set_is_key(true); - column_4->set_is_nullable(true); - column_4->set_length(3); - column_4->set_index_length(3); - - ColumnPB* column_5 = tablet_schema_pb.add_column(); - column_5->set_unique_id(5); - column_5->set_name("column_5"); - column_5->set_type("DATETIME"); - column_5->set_is_key(true); - column_5->set_is_nullable(true); - column_5->set_length(8); - column_5->set_index_length(8); - - ColumnPB* column_6 = tablet_schema_pb.add_column(); - column_6->set_unique_id(5); - column_6->set_name("column_6"); - column_6->set_type("DATEV2"); - column_6->set_is_key(true); - column_6->set_is_nullable(true); - column_6->set_length(4); - column_6->set_index_length(4); - - ColumnPB* column_7 = tablet_schema_pb.add_column(); - column_7->set_unique_id(6); - column_7->set_name("column_7"); - column_7->set_type("DECIMAL"); - column_7->set_is_key(true); - column_7->set_is_nullable(true); - column_7->set_length(12); - column_7->set_index_length(12); - column_7->set_frac(3); - column_7->set_precision(6); - - ColumnPB* column_8 = tablet_schema_pb.add_column(); - column_8->set_unique_id(7); - column_8->set_name("column_8"); - column_8->set_type("CHAR"); - column_8->set_is_key(true); - column_8->set_is_nullable(true); - column_8->set_length(4); - column_8->set_index_length(4); - column_8->set_default_value("char"); - - ColumnPB* column_9 = tablet_schema_pb.add_column(); - column_9->set_unique_id(8); - column_9->set_name("column_9"); - column_9->set_type("BIGINT"); - column_9->set_is_nullable(true); - column_9->set_length(8); - column_9->set_aggregation("SUM"); - column_9->set_is_key(false); - - ColumnPB* column_10 = tablet_schema_pb.add_column(); - column_10->set_unique_id(9); - column_10->set_name("column_10"); - column_10->set_type("VARCHAR"); - column_10->set_is_nullable(true); - column_10->set_length(16 + OLAP_VARCHAR_MAX_BYTES); - column_10->set_aggregation("REPLACE"); - column_10->set_is_key(false); - - ColumnPB* column_11 = tablet_schema_pb.add_column(); - column_11->set_unique_id(10); - column_11->set_name("column_11"); - column_11->set_type("LARGEINT"); - column_11->set_is_nullable(true); - column_11->set_length(16); - column_11->set_aggregation("MAX"); - column_11->set_is_key(false); - - ColumnPB* column_12 = tablet_schema_pb.add_column(); - column_12->set_unique_id(11); - column_12->set_name("column_12"); - column_12->set_type("DECIMAL"); - column_12->set_is_nullable(true); - column_12->set_length(12); - column_12->set_aggregation("MIN"); - column_12->set_is_key(false); - - ColumnPB* column_13 = tablet_schema_pb.add_column(); - column_13->set_unique_id(12); - column_13->set_name("column_13"); - column_13->set_type("HLL"); - column_13->set_is_nullable(true); - column_13->set_length(HLL_COLUMN_DEFAULT_LEN); - column_13->set_aggregation("HLL_UNION"); - column_13->set_is_key(false); +static TabletColumnPtr create_date_key(int32_t id, bool is_nullable = true) { + auto column = std::make_shared(); + column->_unique_id = id; + column->_col_name = std::to_string(id); + column->_type = FieldType::OLAP_FIELD_TYPE_DATE; + column->_is_key = true; + column->_is_nullable = is_nullable; + column->_length = 3; + column->_index_length = 3; + return column; +} - tablet_schema->init_from_pb(tablet_schema_pb); +static TabletColumnPtr create_datev2_key(int32_t id, bool is_nullable = true) { + auto column = std::make_shared(); + column->_unique_id = id; + column->_col_name = std::to_string(id); + column->_type = FieldType::OLAP_FIELD_TYPE_DATEV2; + column->_is_key = true; + column->_is_nullable = is_nullable; + column->_length = 4; + column->_index_length = 4; + return column; } -void set_tablet_schema_for_scan_key(TabletSchemaSPtr tablet_schema) { - TabletSchemaPB tablet_schema_pb; +static TabletColumnPtr create_datetime_key(int32_t id, bool is_nullable = true) { + auto column = std::make_shared(); + column->_unique_id = id; + column->_col_name = std::to_string(id); + column->_type = FieldType::OLAP_FIELD_TYPE_DATETIME; + column->_is_key = true; + column->_is_nullable = is_nullable; + column->_length = 8; + column->_index_length = 8; + return column; +} - ColumnPB* column_1 = tablet_schema_pb.add_column(); - column_1->set_unique_id(1); - column_1->set_name("column_1"); - column_1->set_type("CHAR"); - column_1->set_is_key(true); - column_1->set_is_nullable(true); - column_1->set_length(4); - column_1->set_index_length(4); - column_1->set_default_value("char"); +static TabletColumnPtr create_decimal_key(int32_t id, bool is_nullable = true) { + auto column = std::make_shared(); + column->_unique_id = id; + column->_col_name = std::to_string(id); + column->_type = FieldType::OLAP_FIELD_TYPE_DECIMAL; + column->_is_key = true; + column->_is_nullable = is_nullable; + column->_length = 12; + column->_index_length = 12; + column->_precision = 6; + column->_frac = 3; + return column; +} - ColumnPB* column_2 = tablet_schema_pb.add_column(); - column_2->set_unique_id(2); - column_2->set_name("column_2"); - column_2->set_type("VARCHAR"); - column_2->set_is_key(true); - column_2->set_is_nullable(true); - column_2->set_length(16 + OLAP_VARCHAR_MAX_BYTES); - column_2->set_index_length(20); +static TabletColumnPtr create_datetimev2_key(int32_t id, int32_t scale = 6, + bool is_nullable = true) { + auto column = std::make_shared(); + column->_unique_id = id; + column->_col_name = std::to_string(id); + column->_type = FieldType::OLAP_FIELD_TYPE_DATETIMEV2; + column->_is_key = true; + column->_is_nullable = is_nullable; + column->_length = 8; + column->_index_length = 8; + column->_precision = 20 + scale; + column->_frac = scale; + return column; +} - ColumnPB* column_3 = tablet_schema_pb.add_column(); - column_3->set_unique_id(3); - column_3->set_name("column_3"); - column_3->set_type("LARGEINT"); - column_3->set_is_nullable(true); - column_3->set_length(16); - column_3->set_aggregation("MAX"); - column_3->set_is_key(false); +static TabletColumnPtr create_timestamptz_key(int32_t id, bool is_nullable = true) { + auto column = std::make_shared(); + column->_unique_id = id; + column->_col_name = std::to_string(id); + column->_type = FieldType::OLAP_FIELD_TYPE_TIMESTAMPTZ; + column->_is_key = true; + column->_is_nullable = is_nullable; + column->_length = 8; + column->_index_length = 8; + return column; +} - ColumnPB* column_4 = tablet_schema_pb.add_column(); - column_4->set_unique_id(9); - column_4->set_name("column_4"); - column_4->set_type("DECIMAL"); - column_4->set_is_nullable(true); - column_4->set_length(12); - column_4->set_aggregation("MIN"); - column_4->set_is_key(false); +static TabletColumnPtr create_decimal32_key(int32_t id, bool is_nullable = true) { + auto column = std::make_shared(); + column->_unique_id = id; + column->_col_name = std::to_string(id); + column->_type = FieldType::OLAP_FIELD_TYPE_DECIMAL32; + column->_is_key = true; + column->_is_nullable = is_nullable; + column->_length = 4; + column->_index_length = 4; + column->_precision = 9; + column->_frac = 3; + return column; +} - tablet_schema->init_from_pb(tablet_schema_pb); +static TabletColumnPtr create_decimal64_key(int32_t id, bool is_nullable = true) { + auto column = std::make_shared(); + column->_unique_id = id; + column->_col_name = std::to_string(id); + column->_type = FieldType::OLAP_FIELD_TYPE_DECIMAL64; + column->_is_key = true; + column->_is_nullable = is_nullable; + column->_length = 8; + column->_index_length = 8; + column->_precision = 18; + column->_frac = 6; + return column; +} + +static TabletColumnPtr create_decimal128i_key(int32_t id, bool is_nullable = true) { + auto column = std::make_shared(); + column->_unique_id = id; + column->_col_name = std::to_string(id); + column->_type = FieldType::OLAP_FIELD_TYPE_DECIMAL128I; + column->_is_key = true; + column->_is_nullable = is_nullable; + column->_length = 16; + column->_index_length = 16; + column->_precision = 38; + column->_frac = 9; + return column; +} + +static TabletColumnPtr create_decimal256_key(int32_t id, bool is_nullable = true) { + auto column = std::make_shared(); + column->_unique_id = id; + column->_col_name = std::to_string(id); + column->_type = FieldType::OLAP_FIELD_TYPE_DECIMAL256; + column->_is_key = true; + column->_is_nullable = is_nullable; + column->_length = 32; + column->_index_length = 32; + column->_precision = 76; + column->_frac = 18; + return column; } -void set_tablet_schema_for_cmp_and_aggregate(TabletSchemaSPtr tablet_schema) { +void set_tablet_schema_for_scan_key(TabletSchemaSPtr tablet_schema) { TabletSchemaPB tablet_schema_pb; ColumnPB* column_1 = tablet_schema_pb.add_column(); @@ -214,11 +224,11 @@ void set_tablet_schema_for_cmp_and_aggregate(TabletSchemaSPtr tablet_schema) { ColumnPB* column_2 = tablet_schema_pb.add_column(); column_2->set_unique_id(2); column_2->set_name("column_2"); - column_2->set_type("INT"); + column_2->set_type("VARCHAR"); column_2->set_is_key(true); column_2->set_is_nullable(true); - column_2->set_length(4); - column_2->set_index_length(4); + column_2->set_length(16 + OLAP_VARCHAR_MAX_BYTES); + column_2->set_index_length(20); ColumnPB* column_3 = tablet_schema_pb.add_column(); column_3->set_unique_id(3); @@ -226,36 +236,18 @@ void set_tablet_schema_for_cmp_and_aggregate(TabletSchemaSPtr tablet_schema) { column_3->set_type("LARGEINT"); column_3->set_is_nullable(true); column_3->set_length(16); - column_3->set_aggregation("SUM"); + column_3->set_aggregation("MAX"); column_3->set_is_key(false); ColumnPB* column_4 = tablet_schema_pb.add_column(); column_4->set_unique_id(9); column_4->set_name("column_4"); - column_4->set_type("DOUBLE"); + column_4->set_type("DECIMAL"); column_4->set_is_nullable(true); - column_4->set_length(8); + column_4->set_length(12); column_4->set_aggregation("MIN"); column_4->set_is_key(false); - ColumnPB* column_5 = tablet_schema_pb.add_column(); - column_5->set_unique_id(3); - column_5->set_name("column_5"); - column_5->set_type("DECIMAL"); - column_5->set_is_nullable(true); - column_5->set_length(12); - column_5->set_aggregation("MAX"); - column_5->set_is_key(false); - - ColumnPB* column_6 = tablet_schema_pb.add_column(); - column_6->set_unique_id(9); - column_6->set_name("column_6"); - column_6->set_type("VARCHAR"); - column_6->set_is_nullable(true); - column_6->set_length(16 + OLAP_VARCHAR_MAX_BYTES); - column_6->set_aggregation("REPLACE"); - column_6->set_is_key(false); - tablet_schema->init_from_pb(tablet_schema_pb); } @@ -264,7 +256,6 @@ class TestRowCursor : public testing::Test { TestRowCursor() { _arena.reset(new Arena()); } virtual void SetUp() {} - virtual void TearDown() {} std::unique_ptr _arena; @@ -274,22 +265,14 @@ TEST_F(TestRowCursor, InitRowCursorWithScanKey) { TabletSchemaSPtr tablet_schema = std::make_shared(); set_tablet_schema_for_scan_key(tablet_schema); - std::vector scan_keys; - scan_keys.push_back("char_exceed_length"); - scan_keys.push_back("varchar_exceed_length"); - - std::vector columns {0, 1}; - std::shared_ptr schema = std::make_shared(tablet_schema->columns(), columns); + std::vector scan_keys; + scan_keys.push_back(Field::create_field(String("char_exceed_length"))); + scan_keys.push_back(Field::create_field(String("varchar_exceed_length"))); RowCursor row; - Status res = row.init_scan_key(tablet_schema, scan_keys, schema); - EXPECT_EQ(res, Status::OK()); - EXPECT_EQ(row.get_fixed_len(), 34); - EXPECT_EQ(row.get_variable_len(), 39); - - OlapTuple tuple1(scan_keys); - res = row.from_tuple(tuple1); + Status res = row.init_scan_key(tablet_schema, scan_keys); EXPECT_EQ(res, Status::OK()); + EXPECT_EQ(row.field_count(), 2); } TEST_F(TestRowCursor, encode_key) { @@ -305,53 +288,706 @@ TEST_F(TestRowCursor, encode_key) { // test encoding with padding { RowCursor row; - static_cast(row._init(tablet_schema, 2)); + static_cast(row.init(tablet_schema, 2)); + + row.mutable_field(0) = Field::create_field(int32_t(12345)); + row.mutable_field(1) = Field::create_field(int32_t(54321)); { - // test padding - { - auto cell = row.cell(0); - cell.set_is_null(false); - *(int*)cell.mutable_cell_ptr() = 12345; - } - { - auto cell = row.cell(1); - cell.set_is_null(false); - *(int*)cell.mutable_cell_ptr() = 54321; - } std::string buf; row.encode_key_with_padding(&buf, 3, true); - // should be \x02\x80\x00\x30\x39\x02\x80\x00\xD4\x31\x00 EXPECT_STREQ("0280003039028000D43100", hexdump(buf.c_str(), buf.size()).c_str()); } + // test with null + row.mutable_field(0) = Field::create_field(int32_t(54321)); + row.mutable_field(1) = Field(PrimitiveType::TYPE_NULL); // null + { - { - auto cell = row.cell(0); - cell.set_is_null(false); - *(int*)cell.mutable_cell_ptr() = 54321; - } - { - auto cell = row.cell(1); - cell.set_is_null(true); - *(int*)cell.mutable_cell_ptr() = 54321; - } - - { - std::string buf; - row.encode_key_with_padding(&buf, 3, false); - // should be \x02\x80\x00\xD4\x31\x01\xff - EXPECT_STREQ("028000D43101FF", hexdump(buf.c_str(), buf.size()).c_str()); - } - // encode key - { - std::string buf; - row.encode_key(&buf, 2); - // should be \x02\x80\x00\xD4\x31\x01 - EXPECT_STREQ("028000D43101", hexdump(buf.c_str(), buf.size()).c_str()); - } + std::string buf; + row.encode_key_with_padding(&buf, 3, false); + EXPECT_STREQ("028000D43101FF", hexdump(buf.c_str(), buf.size()).c_str()); + } + // encode key + { + std::string buf; + row.encode_key(&buf, 2); + EXPECT_STREQ("028000D43101", hexdump(buf.c_str(), buf.size()).c_str()); } } } +// ======================================================================== +// Comprehensive encode_key tests covering multiple type combinations. +// Expected values are pre-computed string constants. After any refactoring +// of RowCursor, these tests ensure encoding behavior is preserved. +// +// Encoding rules summary: +// Signed integers: XOR sign bit, then big-endian +// Unsigned integers (DATE, DATEV2): just big-endian, no sign-bit XOR +// Float/Double: canonicalize NaN, sortable bit transform, XOR sign bit, big-endian +// DECIMAL: encode integer(int64) as BIGINT + fraction(int32) as INT +// CHAR: zero-padded to col_length; encode_ascending uses index_size bytes, +// full_encode uses all col_length bytes +// VARCHAR/STRING: encode_ascending uses min(index_size, len) bytes, +// full_encode uses all bytes +// Key markers: NORMAL=0x02, NULL_FIRST=0x01, MINIMAL=0x00, MAXIMAL=0xFF +// ======================================================================== + +// Test 1: INT + FLOAT + CHAR(8) +TEST_F(TestRowCursor, encode_key_int_float_char) { + TabletSchemaSPtr tablet_schema = std::make_shared(); + tablet_schema->_cols.push_back(create_int_key(0)); + tablet_schema->_cols.push_back(create_float_key(1)); + tablet_schema->_cols.push_back(create_char_key(2)); // CHAR(8), index_length=1 + tablet_schema->_num_columns = 3; + tablet_schema->_num_key_columns = 3; + tablet_schema->_num_short_key_columns = 3; + + // All 3 keys present: INT(12345), FLOAT(3.14f), CHAR('ab') + // _encode_field for CHAR pads to col_length=8 with \0 + { + RowCursor row; + static_cast(row.init(tablet_schema, 3)); + + row.mutable_field(0) = Field::create_field(int32_t(12345)); + row.mutable_field(1) = Field::create_field(3.14f); + row.mutable_field(2) = Field::create_field(String("ab")); + + // encode_key (encode_ascending): CHAR uses index_size=1 byte + { + std::string buf; + row.encode_key(&buf, 3); + EXPECT_STREQ("028000303902C048F5C30261", hexdump(buf.c_str(), buf.size()).c_str()); + } + + // encode_key (full_encode_ascending): CHAR uses all 8 bytes (padded) + { + std::string buf; + row.encode_key(&buf, 3); + EXPECT_STREQ("028000303902C048F5C3026162000000000000", + hexdump(buf.c_str(), buf.size()).c_str()); + } + } + + // Padding: only 2 keys initialized, 3rd padded as KEY_MINIMAL_MARKER(0x00) + { + RowCursor row; + static_cast(row.init(tablet_schema, 2)); + + row.mutable_field(0) = Field::create_field(int32_t(12345)); + row.mutable_field(1) = Field::create_field(3.14f); + + { + std::string buf; + row.encode_key_with_padding(&buf, 3, true); + EXPECT_STREQ("028000303902C048F5C300", hexdump(buf.c_str(), buf.size()).c_str()); + } + + // Padding maximal marker (0xFF) + { + std::string buf; + row.encode_key_with_padding(&buf, 3, false); + EXPECT_STREQ("028000303902C048F5C3FF", hexdump(buf.c_str(), buf.size()).c_str()); + } + } + + // Null: INT(12345) present, FLOAT null, CHAR('ab') present + { + RowCursor row; + static_cast(row.init(tablet_schema, 3)); + + row.mutable_field(0) = Field::create_field(int32_t(12345)); + // field(1) stays null (default from init) + row.mutable_field(2) = Field::create_field(String("ab")); + + { + std::string buf; + row.encode_key(&buf, 3); + EXPECT_STREQ("0280003039010261", hexdump(buf.c_str(), buf.size()).c_str()); + } + } +} + +// Test 2: INT + DATE + VARCHAR +TEST_F(TestRowCursor, encode_key_int_date_varchar) { + TabletSchemaSPtr tablet_schema = std::make_shared(); + tablet_schema->_cols.push_back(create_int_key(0)); + tablet_schema->_cols.push_back(create_date_key(1)); + tablet_schema->_cols.push_back(create_varchar_key(2)); // index_length=4 + tablet_schema->_num_columns = 3; + tablet_schema->_num_key_columns = 3; + tablet_schema->_num_short_key_columns = 3; + + RowCursor row; + static_cast(row.init(tablet_schema, 3)); + + row.mutable_field(0) = Field::create_field(int32_t(12345)); + // DATE(2020-01-01): raw uint24_t = 2020*512 + 1*32 + 1 = 1034273 + row.mutable_field(1) = + Field::create_field_from_olap_value(uint24_t(2020 * 512 + 1 * 32 + 1)); + row.mutable_field(2) = Field::create_field(String("hello")); + + // encode_key: VARCHAR('hello') truncated to 4 bytes + { + std::string buf; + row.encode_key(&buf, 3); + EXPECT_STREQ("0280003039020FC8210268656C6C", hexdump(buf.c_str(), buf.size()).c_str()); + } + + // full_encode: VARCHAR('hello') all 5 bytes + { + std::string buf; + row.encode_key(&buf, 3); + EXPECT_STREQ("0280003039020FC8210268656C6C6F", hexdump(buf.c_str(), buf.size()).c_str()); + } +} + +// Test 3: DOUBLE(NaN) + DECIMAL + STRING +TEST_F(TestRowCursor, encode_key_double_nan_decimal_string) { + TabletSchemaSPtr tablet_schema = std::make_shared(); + tablet_schema->_cols.push_back(create_double_key(0)); + tablet_schema->_cols.push_back(create_decimal_key(1)); + tablet_schema->_cols.push_back(create_string_key(2)); // index_length=4 + tablet_schema->_num_columns = 3; + tablet_schema->_num_key_columns = 3; + tablet_schema->_num_short_key_columns = 3; + + RowCursor row; + static_cast(row.init(tablet_schema, 3)); + + row.mutable_field(0) = + Field::create_field(std::numeric_limits::quiet_NaN()); + // DECIMAL(123.456000000): stored as decimal12_t{123, 456000000} + row.mutable_field(1) = + Field::create_field_from_olap_value(decimal12_t {123, 456000000}); + row.mutable_field(2) = Field::create_field(String("hello world")); + + // encode_key: STRING truncated to 4 bytes + { + std::string buf; + row.encode_key(&buf, 3); + EXPECT_STREQ("02FFF800000000000002800000000000007B9B2E02000268656C6C", + hexdump(buf.c_str(), buf.size()).c_str()); + } + + // full_encode: STRING all 11 bytes + { + std::string buf; + row.encode_key(&buf, 3); + EXPECT_STREQ("02FFF800000000000002800000000000007B9B2E02000268656C6C6F20776F726C64", + hexdump(buf.c_str(), buf.size()).c_str()); + } +} + +// Test 4: BIGINT + DATEV2 + CHAR(8, full) +TEST_F(TestRowCursor, encode_key_bigint_datev2_char) { + TabletSchemaSPtr tablet_schema = std::make_shared(); + tablet_schema->_cols.push_back(create_bigint_key(0)); + tablet_schema->_cols.push_back(create_datev2_key(1)); + tablet_schema->_cols.push_back(create_char_key(2)); // CHAR(8), index_length=1 + tablet_schema->_num_columns = 3; + tablet_schema->_num_key_columns = 3; + tablet_schema->_num_short_key_columns = 3; + + RowCursor row; + static_cast(row.init(tablet_schema, 3)); + + row.mutable_field(0) = Field::create_field(int64_t(9999999999LL)); + // DATEV2(2024-12-31): packed = (2024<<9)|(12<<5)|31 = 1036703 + { + uint32_t packed = (2024 << 9) | (12 << 5) | 31; + row.mutable_field(1) = Field::create_field(packed); + } + row.mutable_field(2) = Field::create_field(String("abcdefgh")); + + // encode_key: CHAR index_size=1 -> only 'a' (0x61) + { + std::string buf; + row.encode_key(&buf, 3); + EXPECT_STREQ("0280000002540BE3FF02000FD19F0261", hexdump(buf.c_str(), buf.size()).c_str()); + } + + // full_encode: CHAR all 8 bytes + { + std::string buf; + row.encode_key(&buf, 3); + EXPECT_STREQ("0280000002540BE3FF02000FD19F026162636465666768", + hexdump(buf.c_str(), buf.size()).c_str()); + } +} + +// Test 5: FLOAT(inf) + DATETIME + VARCHAR +TEST_F(TestRowCursor, encode_key_float_inf_datetime_varchar) { + TabletSchemaSPtr tablet_schema = std::make_shared(); + tablet_schema->_cols.push_back(create_float_key(0)); + tablet_schema->_cols.push_back(create_datetime_key(1)); + tablet_schema->_cols.push_back(create_varchar_key(2)); // index_length=4 + tablet_schema->_num_columns = 3; + tablet_schema->_num_key_columns = 3; + tablet_schema->_num_short_key_columns = 3; + + RowCursor row; + static_cast(row.init(tablet_schema, 3)); + + row.mutable_field(0) = Field::create_field(std::numeric_limits::infinity()); + // DATETIME(2020-01-01 12:00:00) = 20200101120000 in olap datetime format + row.mutable_field(1) = + Field::create_field_from_olap_value(uint64_t(20200101120000ULL)); + row.mutable_field(2) = Field::create_field(String("ab")); + + // encode_key: VARCHAR('ab') only 2 bytes (< index_size=4, not truncated) + { + std::string buf; + row.encode_key(&buf, 3); + EXPECT_STREQ("02FF800000028000125F33DA0800026162", + hexdump(buf.c_str(), buf.size()).c_str()); + } + + // full_encode: same result since VARCHAR('ab') is only 2 bytes + { + std::string buf; + row.encode_key(&buf, 3); + EXPECT_STREQ("02FF800000028000125F33DA0800026162", + hexdump(buf.c_str(), buf.size()).c_str()); + } +} + +// Test: encode_key_with_padding for MOW mode (is_mow=true) +TEST_F(TestRowCursor, encode_key_with_padding_mow) { + TabletSchemaSPtr tablet_schema = std::make_shared(); + tablet_schema->_cols.push_back(create_int_key(0)); + tablet_schema->_cols.push_back(create_int_key(1)); + tablet_schema->_cols.push_back(create_int_key(2)); + tablet_schema->_num_columns = 3; + tablet_schema->_num_key_columns = 3; + tablet_schema->_num_short_key_columns = 3; + + RowCursor row; + static_cast(row.init(tablet_schema, 2)); + + row.mutable_field(0) = Field::create_field(int32_t(12345)); + row.mutable_field(1) = Field::create_field(int32_t(54321)); + + // MOW padding_minimal=false -> KEY_NORMAL_NEXT_MARKER(0x03) instead of 0xFF + { + std::string buf; + row.encode_key_with_padding(&buf, 3, false); + EXPECT_STREQ("0280003039028000D43103", hexdump(buf.c_str(), buf.size()).c_str()); + } + + // MOW padding_minimal=true -> KEY_MINIMAL_MARKER(0x00) + { + std::string buf; + row.encode_key_with_padding(&buf, 3, true); + EXPECT_STREQ("0280003039028000D43100", hexdump(buf.c_str(), buf.size()).c_str()); + } +} + +// Test: negative values for various types +TEST_F(TestRowCursor, encode_key_negative_values) { + TabletSchemaSPtr tablet_schema = std::make_shared(); + tablet_schema->_cols.push_back(create_int_key(0)); + tablet_schema->_cols.push_back(create_double_key(1)); + tablet_schema->_cols.push_back(create_decimal_key(2)); + tablet_schema->_num_columns = 3; + tablet_schema->_num_key_columns = 3; + tablet_schema->_num_short_key_columns = 3; + + RowCursor row; + static_cast(row.init(tablet_schema, 3)); + + row.mutable_field(0) = Field::create_field(int32_t(-12345)); + row.mutable_field(1) = Field::create_field(-1.0); + // DECIMAL(-123, -456000000) representing -123.456000000 + row.mutable_field(2) = + Field::create_field_from_olap_value(decimal12_t {-123, -456000000}); + + { + std::string buf; + row.encode_key(&buf, 3); + EXPECT_STREQ("027FFFCFC702400FFFFFFFFFFFFF027FFFFFFFFFFFFF8564D1FE00", + hexdump(buf.c_str(), buf.size()).c_str()); + } +} + +// Test: float special values (-inf, NaN) and boundary integers +TEST_F(TestRowCursor, encode_key_float_special_values) { + TabletSchemaSPtr tablet_schema = std::make_shared(); + tablet_schema->_cols.push_back(create_float_key(0)); + tablet_schema->_cols.push_back(create_int_key(1)); + tablet_schema->_num_columns = 2; + tablet_schema->_num_key_columns = 2; + tablet_schema->_num_short_key_columns = 2; + + // Test -infinity + { + RowCursor row; + static_cast(row.init(tablet_schema, 2)); + row.mutable_field(0) = + Field::create_field(-std::numeric_limits::infinity()); + row.mutable_field(1) = Field::create_field(int32_t(0)); + + std::string buf; + row.encode_key(&buf, 2); + EXPECT_STREQ("02007FFFFF0280000000", hexdump(buf.c_str(), buf.size()).c_str()); + } + + // Test INT boundary values + { + RowCursor row; + static_cast(row.init(tablet_schema, 2)); + row.mutable_field(0) = + Field::create_field(std::numeric_limits::quiet_NaN()); + row.mutable_field(1) = Field::create_field(std::numeric_limits::max()); + + std::string buf; + row.encode_key(&buf, 2); + EXPECT_STREQ("02FFC0000002FFFFFFFF", hexdump(buf.c_str(), buf.size()).c_str()); + } + + // Test INT_MIN + { + TabletSchemaSPtr schema2 = std::make_shared(); + schema2->_cols.push_back(create_int_key(0)); + schema2->_num_columns = 1; + schema2->_num_key_columns = 1; + schema2->_num_short_key_columns = 1; + + RowCursor row2; + static_cast(row2.init(schema2, 1)); + row2.mutable_field(0) = Field::create_field(std::numeric_limits::min()); + + std::string buf; + row2.encode_key(&buf, 1); + EXPECT_STREQ("0200000000", hexdump(buf.c_str(), buf.size()).c_str()); + } +} + +// Test: all null keys +TEST_F(TestRowCursor, encode_key_all_null) { + TabletSchemaSPtr tablet_schema = std::make_shared(); + tablet_schema->_cols.push_back(create_int_key(0)); + tablet_schema->_cols.push_back(create_float_key(1)); + tablet_schema->_cols.push_back(create_varchar_key(2)); + tablet_schema->_num_columns = 3; + tablet_schema->_num_key_columns = 3; + tablet_schema->_num_short_key_columns = 3; + + RowCursor row; + static_cast(row.init(tablet_schema, 3)); + // All fields stay null (default from init) + + { + std::string buf; + row.encode_key(&buf, 3); + // All null: three 0x01 markers + EXPECT_STREQ("010101", hexdump(buf.c_str(), buf.size()).c_str()); + } +} + +// DATETIME v1 encoding +TEST_F(TestRowCursor, encode_key_datetime_v1) { + TabletSchemaSPtr tablet_schema = std::make_shared(); + tablet_schema->_cols.push_back(create_datetime_key(0)); + tablet_schema->_num_columns = 1; + tablet_schema->_num_key_columns = 1; + tablet_schema->_num_short_key_columns = 1; + + { + RowCursor row; + static_cast(row.init(tablet_schema, 1)); + row.mutable_field(0) = + Field::create_field_from_olap_value(uint64_t(20241231123045ULL)); + + std::string buf; + row.encode_key(&buf, 1); + EXPECT_STREQ("0280001268C7641265", hexdump(buf.c_str(), buf.size()).c_str()); + } +} + +// DATETIMEV2 encoding at various precisions +TEST_F(TestRowCursor, encode_key_datetimev2) { + // --- DATETIMEV2(0) --- + { + TabletSchemaSPtr tablet_schema = std::make_shared(); + tablet_schema->_cols.push_back(create_datetimev2_key(0, 0)); + tablet_schema->_num_columns = 1; + tablet_schema->_num_key_columns = 1; + tablet_schema->_num_short_key_columns = 1; + + RowCursor row; + static_cast(row.init(tablet_schema, 1)); + uint64_t packed = (uint64_t(2024) << 46) | (uint64_t(12) << 42) | (uint64_t(31) << 37) | + (uint64_t(12) << 32) | (uint64_t(30) << 26) | (uint64_t(45) << 20) | + uint64_t(0); + row.mutable_field(0) = Field::create_field(packed); + + std::string buf; + row.encode_key(&buf, 1); + EXPECT_STREQ("0201FA33EC7AD00000", hexdump(buf.c_str(), buf.size()).c_str()); + } + + // --- DATETIMEV2(3) --- + { + TabletSchemaSPtr tablet_schema = std::make_shared(); + tablet_schema->_cols.push_back(create_datetimev2_key(0, 3)); + tablet_schema->_num_columns = 1; + tablet_schema->_num_key_columns = 1; + tablet_schema->_num_short_key_columns = 1; + + RowCursor row; + static_cast(row.init(tablet_schema, 1)); + uint64_t packed = (uint64_t(2024) << 46) | (uint64_t(12) << 42) | (uint64_t(31) << 37) | + (uint64_t(12) << 32) | (uint64_t(30) << 26) | (uint64_t(45) << 20) | + uint64_t(123000); + row.mutable_field(0) = Field::create_field(packed); + + std::string buf; + row.encode_key(&buf, 1); + EXPECT_STREQ("0201FA33EC7AD1E078", hexdump(buf.c_str(), buf.size()).c_str()); + } + + // --- DATETIMEV2(6) --- + { + TabletSchemaSPtr tablet_schema = std::make_shared(); + tablet_schema->_cols.push_back(create_datetimev2_key(0, 6)); + tablet_schema->_num_columns = 1; + tablet_schema->_num_key_columns = 1; + tablet_schema->_num_short_key_columns = 1; + + RowCursor row; + static_cast(row.init(tablet_schema, 1)); + uint64_t packed = (uint64_t(2024) << 46) | (uint64_t(12) << 42) | (uint64_t(31) << 37) | + (uint64_t(12) << 32) | (uint64_t(30) << 26) | (uint64_t(45) << 20) | + uint64_t(123456); + row.mutable_field(0) = Field::create_field(packed); + + std::string buf; + row.encode_key(&buf, 1); + EXPECT_STREQ("0201FA33EC7AD1E240", hexdump(buf.c_str(), buf.size()).c_str()); + } + + // Null DATETIMEV2 + { + TabletSchemaSPtr tablet_schema = std::make_shared(); + tablet_schema->_cols.push_back(create_datetimev2_key(0, 6)); + tablet_schema->_num_columns = 1; + tablet_schema->_num_key_columns = 1; + tablet_schema->_num_short_key_columns = 1; + + RowCursor row; + static_cast(row.init(tablet_schema, 1)); + // field(0) stays null + + std::string buf; + row.encode_key(&buf, 1); + EXPECT_STREQ("01", hexdump(buf.c_str(), buf.size()).c_str()); + } +} + +// TIMESTAMPTZ encoding +TEST_F(TestRowCursor, encode_key_timestamptz) { + TabletSchemaSPtr tablet_schema = std::make_shared(); + tablet_schema->_cols.push_back(create_timestamptz_key(0)); + tablet_schema->_num_columns = 1; + tablet_schema->_num_key_columns = 1; + tablet_schema->_num_short_key_columns = 1; + + { + RowCursor row; + static_cast(row.init(tablet_schema, 1)); + // TimestampTzValue wraps DateV2Value internally. + // The uint64_t is the packed DateTimeV2 representation. + row.mutable_field(0) = + Field::create_field(TimestampTzValue(1704067200000000ULL)); + + std::string buf; + row.encode_key(&buf, 1); + EXPECT_STREQ("0200060DD710212000", hexdump(buf.c_str(), buf.size()).c_str()); + } +} + +// DECIMAL32 encoding +TEST_F(TestRowCursor, encode_key_decimal32) { + TabletSchemaSPtr tablet_schema = std::make_shared(); + tablet_schema->_cols.push_back(create_decimal32_key(0)); + tablet_schema->_num_columns = 1; + tablet_schema->_num_key_columns = 1; + tablet_schema->_num_short_key_columns = 1; + + // Positive: 12345 + { + RowCursor row; + static_cast(row.init(tablet_schema, 1)); + row.mutable_field(0) = Field::create_field(Decimal32(int32_t(12345))); + + std::string buf; + row.encode_key(&buf, 1); + EXPECT_STREQ("0280003039", hexdump(buf.c_str(), buf.size()).c_str()); + } + + // Negative: -12345 + { + RowCursor row; + static_cast(row.init(tablet_schema, 1)); + row.mutable_field(0) = Field::create_field(Decimal32(int32_t(-12345))); + + std::string buf; + row.encode_key(&buf, 1); + EXPECT_STREQ("027FFFCFC7", hexdump(buf.c_str(), buf.size()).c_str()); + } + + // Zero + { + RowCursor row; + static_cast(row.init(tablet_schema, 1)); + row.mutable_field(0) = Field::create_field(Decimal32(int32_t(0))); + + std::string buf; + row.encode_key(&buf, 1); + EXPECT_STREQ("0280000000", hexdump(buf.c_str(), buf.size()).c_str()); + } +} + +// DECIMAL64 encoding +TEST_F(TestRowCursor, encode_key_decimal64) { + TabletSchemaSPtr tablet_schema = std::make_shared(); + tablet_schema->_cols.push_back(create_decimal64_key(0)); + tablet_schema->_num_columns = 1; + tablet_schema->_num_key_columns = 1; + tablet_schema->_num_short_key_columns = 1; + + // Positive: 9999999999 + { + RowCursor row; + static_cast(row.init(tablet_schema, 1)); + row.mutable_field(0) = + Field::create_field(Decimal64(int64_t(9999999999LL))); + + std::string buf; + row.encode_key(&buf, 1); + EXPECT_STREQ("0280000002540BE3FF", hexdump(buf.c_str(), buf.size()).c_str()); + } + + // Negative: -123456789 + { + RowCursor row; + static_cast(row.init(tablet_schema, 1)); + row.mutable_field(0) = + Field::create_field(Decimal64(int64_t(-123456789LL))); + + std::string buf; + row.encode_key(&buf, 1); + EXPECT_STREQ("027FFFFFFFF8A432EB", hexdump(buf.c_str(), buf.size()).c_str()); + } +} + +// DECIMAL128I encoding +TEST_F(TestRowCursor, encode_key_decimal128i) { + TabletSchemaSPtr tablet_schema = std::make_shared(); + tablet_schema->_cols.push_back(create_decimal128i_key(0)); + tablet_schema->_num_columns = 1; + tablet_schema->_num_key_columns = 1; + tablet_schema->_num_short_key_columns = 1; + + // Positive: 123456789012345678 + { + RowCursor row; + static_cast(row.init(tablet_schema, 1)); + int128_t val = static_cast(123456789012345678LL); + row.mutable_field(0) = Field::create_field(Decimal128V3(val)); + + std::string buf; + row.encode_key(&buf, 1); + EXPECT_STREQ("02800000000000000001B69B4BA630F34E", + hexdump(buf.c_str(), buf.size()).c_str()); + } + + // Negative: -123456789012345678 + { + RowCursor row; + static_cast(row.init(tablet_schema, 1)); + int128_t val = static_cast(-123456789012345678LL); + row.mutable_field(0) = Field::create_field(Decimal128V3(val)); + + std::string buf; + row.encode_key(&buf, 1); + EXPECT_STREQ("027FFFFFFFFFFFFFFFFE4964B459CF0CB2", + hexdump(buf.c_str(), buf.size()).c_str()); + } +} + +// DECIMAL256 encoding +TEST_F(TestRowCursor, encode_key_decimal256) { + TabletSchemaSPtr tablet_schema = std::make_shared(); + tablet_schema->_cols.push_back(create_decimal256_key(0)); + tablet_schema->_num_columns = 1; + tablet_schema->_num_key_columns = 1; + tablet_schema->_num_short_key_columns = 1; + + // Positive: 123456789012345678901234567890 + { + RowCursor row; + static_cast(row.init(tablet_schema, 1)); + wide::Int256 val({0xC373E0EE4E3F0AD2ULL, 0x000000018EE90FF6ULL, 0ULL, 0ULL}); + row.mutable_field(0) = Field::create_field(Decimal256(val)); + + std::string buf; + row.encode_key(&buf, 1); + EXPECT_STREQ("0280000000000000000000000000000000000000018EE90FF6C373E0EE4E3F0AD2", + hexdump(buf.c_str(), buf.size()).c_str()); + } + + // Negative: -123456789012345678901234567890 + { + RowCursor row; + static_cast(row.init(tablet_schema, 1)); + wide::Int256 val({0x3C8C1F11B1C0F52EULL, 0xFFFFFFFE7116F009ULL, 0xFFFFFFFFFFFFFFFFULL, + 0xFFFFFFFFFFFFFFFFULL}); + row.mutable_field(0) = Field::create_field(Decimal256(val)); + + std::string buf; + row.encode_key(&buf, 1); + EXPECT_STREQ("027FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFE7116F0093C8C1F11B1C0F52E", + hexdump(buf.c_str(), buf.size()).c_str()); + } +} + +// Combined test: DATETIMEV2 + DECIMAL128I + DECIMAL32 +TEST_F(TestRowCursor, encode_key_datetimev2_decimal128i_decimal32) { + TabletSchemaSPtr tablet_schema = std::make_shared(); + tablet_schema->_cols.push_back(create_datetimev2_key(0, 3)); + tablet_schema->_cols.push_back(create_decimal128i_key(1)); + tablet_schema->_cols.push_back(create_decimal32_key(2)); + tablet_schema->_num_columns = 3; + tablet_schema->_num_key_columns = 3; + tablet_schema->_num_short_key_columns = 3; + + RowCursor row; + static_cast(row.init(tablet_schema, 3)); + + // DATETIMEV2(3): 2024-12-31 12:30:45.123 + { + uint64_t packed = (uint64_t(2024) << 46) | (uint64_t(12) << 42) | (uint64_t(31) << 37) | + (uint64_t(12) << 32) | (uint64_t(30) << 26) | (uint64_t(45) << 20) | + uint64_t(123000); + row.mutable_field(0) = Field::create_field(packed); + } + // DECIMAL128I: 123456789012345678 + { + int128_t val = static_cast(123456789012345678LL); + row.mutable_field(1) = Field::create_field(Decimal128V3(val)); + } + // DECIMAL32: -12345 + row.mutable_field(2) = Field::create_field(Decimal32(int32_t(-12345))); + + { + std::string buf; + row.encode_key(&buf, 3); + EXPECT_STREQ("0201FA33EC7AD1E07802800000000000000001B69B4BA630F34E027FFFCFC7", + hexdump(buf.c_str(), buf.size()).c_str()); + } +} + } // namespace doris diff --git a/be/test/storage/segment/column_meta_accessor_test.cpp b/be/test/storage/segment/column_meta_accessor_test.cpp index ace736aabca51e..25443db7bc07cf 100644 --- a/be/test/storage/segment/column_meta_accessor_test.cpp +++ b/be/test/storage/segment/column_meta_accessor_test.cpp @@ -24,6 +24,7 @@ #include #include +#include "core/field.h" #include "io/fs/local_file_system.h" #include "storage/segment/segment.h" #include "storage/segment/segment_writer.h" @@ -94,7 +95,7 @@ Status read_footer_from_file(const io::FileReaderSPtr& fr, SegmentFooterPB* foot namespace doris { -using Generator = std::function; +using Generator = std::function; // Helper declarations are defined in tablet_schema_helper.{h,cpp} and // delete_bitmap_calculator_test.cpp. @@ -630,10 +631,9 @@ TEST(ColumnMetaAccessorTest, FooterSizeWithManyColumnsExternalVsInline) { SegmentWriterOptions opts; opts.enable_unique_key_merge_on_write = false; - auto generator = [](size_t rid, int cid, RowCursorCell& cell) { - cell.set_not_null(); + auto generator = [](size_t rid, int cid, Field& field) { // deterministic int payload: value = rid * 10 + cid - *reinterpret_cast(cell.mutable_cell_ptr()) = static_cast(rid * 10 + cid); + field = Field::create_field(int32_t(rid * 10 + cid)); }; // 3. Build inline segment (V2 footer, inline ColumnMetaPB). diff --git a/be/test/storage/segment/external_col_meta_util_test.cpp b/be/test/storage/segment/external_col_meta_util_test.cpp index 29a68c027a032a..e2c2f285b45100 100644 --- a/be/test/storage/segment/external_col_meta_util_test.cpp +++ b/be/test/storage/segment/external_col_meta_util_test.cpp @@ -24,6 +24,7 @@ #include #include +#include "core/field.h" #include "gtest/gtest.h" #include "io/fs/local_file_system.h" #include "storage/segment/segment.h" @@ -115,7 +116,7 @@ Status read_footer_from_file(const io::FileReaderSPtr& fr, SegmentFooterPB* foot namespace doris { -using Generator = std::function; +using Generator = std::function; // Helper declarations are defined in tablet_schema_helper.{h,cpp} and // delete_bitmap_calculator_test.cpp. @@ -500,10 +501,9 @@ TEST(ExternalColMetaUtilTest, BuildSegmentAndVerifyDataAndFooterMeta) { opts.enable_unique_key_merge_on_write = true; const size_t nrows = 16; - auto generator = [](size_t rid, int cid, RowCursorCell& cell) { - cell.set_not_null(); + auto generator = [](size_t rid, int cid, Field& field) { // deterministic int payload: value = rid * 10 + cid - *reinterpret_cast(cell.mutable_cell_ptr()) = static_cast(rid * 10 + cid); + field = Field::create_field(int32_t(rid * 10 + cid)); }; std::shared_ptr segment; diff --git a/be/test/storage/segment/segment_corruption_test.cpp b/be/test/storage/segment/segment_corruption_test.cpp index 0bc03d2c36af49..affeac8fc3770a 100644 --- a/be/test/storage/segment/segment_corruption_test.cpp +++ b/be/test/storage/segment/segment_corruption_test.cpp @@ -24,6 +24,7 @@ #include #include "common/status.h" +#include "core/field.h" #include "cpp/sync_point.h" #include "io/cache/block_file_cache.h" #include "io/cache/block_file_cache_factory.h" @@ -39,6 +40,7 @@ #include "storage/rowset/rowset_id_generator.h" #include "storage/segment/segment.h" #include "storage/segment/segment_writer.h" +#include "storage/segment/test_segment_writer.h" #include "storage/storage_engine.h" #include "storage/tablet/tablet_schema.h" #include "storage/tablet/tablet_schema_helper.h" @@ -217,27 +219,20 @@ class SegmentCorruptionTest : public testing::Test { InvertedIndexStorageFormatPB::V2, std::move(idx_file_writer)); SegmentWriterOptions opts; - SegmentWriter writer(file_writer.get(), segment_id, schema, nullptr, nullptr, opts, - index_file_writer.get()); + TestSegmentWriter writer(file_writer.get(), segment_id, schema, nullptr, nullptr, opts, + index_file_writer.get()); st = writer.init(); EXPECT_TRUE(st.ok()); // Write rows RowCursor row; - auto olap_st = row._init(schema, schema->num_columns()); + auto olap_st = row.init(schema, schema->num_columns()); EXPECT_EQ(Status::OK(), olap_st); // Write one row: (1, "hello") { - RowCursorCell cell0 = row.cell(0); - *(int32_t*)cell0.mutable_cell_ptr() = 1; - cell0.set_not_null(); - - RowCursorCell cell1 = row.cell(1); - Slice value("hello"); - reinterpret_cast(cell1.mutable_cell_ptr())->data = value.data; - reinterpret_cast(cell1.mutable_cell_ptr())->size = value.size; - cell1.set_not_null(); + row.mutable_field(0) = Field::create_field(int32_t(1)); + row.mutable_field(1) = Field::create_field(String("hello")); st = writer.append_row(row); EXPECT_TRUE(st.ok()); diff --git a/be/test/storage/segment/segment_footer_cache_test.cpp b/be/test/storage/segment/segment_footer_cache_test.cpp index e30aeaedb8099a..0e75b3945f3ba8 100644 --- a/be/test/storage/segment/segment_footer_cache_test.cpp +++ b/be/test/storage/segment/segment_footer_cache_test.cpp @@ -19,6 +19,7 @@ #include +#include "core/field.h" #include "storage/cache/page_cache.h" #include "storage/segment/segment.h" #include "storage/segment/segment_writer.h" @@ -33,7 +34,7 @@ TabletColumnPtr create_int_sequence_value(int32_t id, bool is_nullable = true, TabletSchemaSPtr create_schema(const std::vector& columns, KeysType keys_type = UNIQUE_KEYS); -using Generator = std::function; +using Generator = std::function; void build_segment(SegmentWriterOptions opts, TabletSchemaSPtr build_schema, size_t segment_id, TabletSchemaSPtr query_schema, size_t nrows, Generator generator, @@ -116,9 +117,8 @@ class SegmentFooterCacheTest : public ::testing::Test { for (size_t sid = 0; sid < num_segments; ++sid) { auto& segment = segments[sid]; std::vector row_data; - auto generator = [&](size_t rid, int cid, RowCursorCell& cell) { - cell.set_not_null(); - *(int*)cell.mutable_cell_ptr() = data_map[{sid, rid}][cid]; + auto generator = [&](size_t rid, int cid, Field& field) { + field = Field::create_field(int32_t(data_map[{sid, rid}][cid])); }; build_segment(opts, tablet_schema, sid, tablet_schema, datas[sid].size(), generator, &segment, segment_footer_cache_test_segment_dir); diff --git a/be/test/storage/segment/test_segment_writer.h b/be/test/storage/segment/test_segment_writer.h new file mode 100644 index 00000000000000..fcb385e255456c --- /dev/null +++ b/be/test/storage/segment/test_segment_writer.h @@ -0,0 +1,136 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include "common/consts.h" +#include "core/data_type/primitive_type.h" +#include "core/field.h" +#include "core/value/vdatetime_value.h" +#include "storage/index/primary_key_index.h" +#include "storage/index/short_key_index.h" +#include "storage/olap_common.h" +#include "storage/row_cursor.h" +#include "storage/segment/column_writer.h" +#include "storage/segment/segment_writer.h" +#include "util/slice.h" + +namespace doris::segment_v2 { + +// Test-only subclass that provides RowCursor-based append_row. +// Production code should only use SegmentWriter::append_block. +// SegmentWriter declares TestSegmentWriter as a friend so we can access private members. +class TestSegmentWriter : public SegmentWriter { +public: + using SegmentWriter::SegmentWriter; + + Status append_row(const RowCursor& row) { + for (size_t cid = 0; cid < _column_writers.size(); ++cid) { + const auto& f = row.field(cid); + if (f.is_null()) { + RETURN_IF_ERROR(_column_writers[cid]->append(true, nullptr)); + continue; + } + auto ft = row.schema()->column(cid)->type(); + // Convert Field to storage format that ColumnWriter expects + alignas(16) char buf[sizeof(__int128)]; + const void* ptr = nullptr; + switch (ft) { + case FieldType::OLAP_FIELD_TYPE_CHAR: + case FieldType::OLAP_FIELD_TYPE_VARCHAR: + case FieldType::OLAP_FIELD_TYPE_STRING: { + const auto& s = f.get(); + Slice slice(s.data(), s.size()); + RETURN_IF_ERROR(_column_writers[cid]->append(false, &slice)); + continue; + } + case FieldType::OLAP_FIELD_TYPE_DATE: { + auto v = f.get().to_olap_date(); + memcpy(buf, &v, sizeof(v)); + ptr = buf; + break; + } + case FieldType::OLAP_FIELD_TYPE_DATETIME: { + auto v = f.get().to_olap_datetime(); + memcpy(buf, &v, sizeof(v)); + ptr = buf; + break; + } + case FieldType::OLAP_FIELD_TYPE_DECIMAL: { + auto v = PrimitiveTypeConvertor::to_storage_field_type( + f.get()); + memcpy(buf, &v, sizeof(v)); + ptr = buf; + break; + } +#define FIXED_LEN_CASE(OLAP_TYPE, PTYPE) \ + case FieldType::OLAP_TYPE: \ + ptr = reinterpret_cast(&f.get()); \ + break; + FIXED_LEN_CASE(OLAP_FIELD_TYPE_BOOL, TYPE_BOOLEAN) + FIXED_LEN_CASE(OLAP_FIELD_TYPE_TINYINT, TYPE_TINYINT) + FIXED_LEN_CASE(OLAP_FIELD_TYPE_SMALLINT, TYPE_SMALLINT) + FIXED_LEN_CASE(OLAP_FIELD_TYPE_INT, TYPE_INT) + FIXED_LEN_CASE(OLAP_FIELD_TYPE_BIGINT, TYPE_BIGINT) + FIXED_LEN_CASE(OLAP_FIELD_TYPE_LARGEINT, TYPE_LARGEINT) + FIXED_LEN_CASE(OLAP_FIELD_TYPE_FLOAT, TYPE_FLOAT) + FIXED_LEN_CASE(OLAP_FIELD_TYPE_DOUBLE, TYPE_DOUBLE) + FIXED_LEN_CASE(OLAP_FIELD_TYPE_DATEV2, TYPE_DATEV2) + FIXED_LEN_CASE(OLAP_FIELD_TYPE_DATETIMEV2, TYPE_DATETIMEV2) + FIXED_LEN_CASE(OLAP_FIELD_TYPE_TIMESTAMPTZ, TYPE_TIMESTAMPTZ) + FIXED_LEN_CASE(OLAP_FIELD_TYPE_DECIMAL32, TYPE_DECIMAL32) + FIXED_LEN_CASE(OLAP_FIELD_TYPE_DECIMAL64, TYPE_DECIMAL64) + FIXED_LEN_CASE(OLAP_FIELD_TYPE_DECIMAL128I, TYPE_DECIMAL128I) + FIXED_LEN_CASE(OLAP_FIELD_TYPE_DECIMAL256, TYPE_DECIMAL256) + FIXED_LEN_CASE(OLAP_FIELD_TYPE_IPV4, TYPE_IPV4) + FIXED_LEN_CASE(OLAP_FIELD_TYPE_IPV6, TYPE_IPV6) +#undef FIXED_LEN_CASE + default: + return Status::InternalError("Unsupported field type in append_row: {}", int(ft)); + } + RETURN_IF_ERROR(_column_writers[cid]->append(false, const_cast(ptr))); + } + std::string full_encoded_key; + row.encode_key(&full_encoded_key, _num_sort_key_columns); + if (_tablet_schema->has_sequence_col()) { + full_encoded_key.push_back(KeyConsts::KEY_NORMAL_MARKER); + auto cid = _tablet_schema->sequence_col_idx(); + row.encode_single_field(cid, &full_encoded_key, true /*full_encode*/); + } + + if (_is_mow_with_cluster_key()) { + return Status::InternalError( + "TestSegmentWriter::append_row does not support mow tables with cluster key"); + } else if (_is_mow()) { + RETURN_IF_ERROR(_primary_key_index_builder->add_item(full_encoded_key)); + } else { + // At the beginning of one block, so add a short key index entry + if ((_num_rows_written % _opts.num_rows_per_block) == 0) { + std::string encoded_key; + row.encode_key(&encoded_key, _num_short_key_columns); + RETURN_IF_ERROR(_short_key_index_builder->add_item(encoded_key)); + } + set_min_max_key(full_encoded_key); + } + ++_num_rows_written; + return Status::OK(); + } +}; + +} // namespace doris::segment_v2 diff --git a/be/test/storage/storage_types_test.cpp b/be/test/storage/storage_types_test.cpp index d95b500219daaf..131f5a900212e5 100644 --- a/be/test/storage/storage_types_test.cpp +++ b/be/test/storage/storage_types_test.cpp @@ -52,11 +52,6 @@ void common_test(typename TypeTraits::CppType src_val) { type->deep_copy((char*)&dst_val, (char*)&src_val, pool); EXPECT_EQ(0, type->cmp((char*)&src_val, (char*)&dst_val)); } - { - typename TypeTraits::CppType dst_val; - type->direct_copy((char*)&dst_val, (char*)&src_val); - EXPECT_EQ(0, type->cmp((char*)&src_val, (char*)&dst_val)); - } // test min { typename TypeTraits::CppType dst_val; @@ -88,12 +83,6 @@ void test_char(Slice src_val) { type->deep_copy((char*)&dst_val, (char*)&src_val, pool); EXPECT_EQ(0, type->cmp((char*)&src_val, (char*)&dst_val)); } - { - char buf[64]; - Slice dst_val(buf, sizeof(buf)); - type->direct_copy((char*)&dst_val, (char*)&src_val); - EXPECT_EQ(0, type->cmp((char*)&src_val, (char*)&dst_val)); - } // test min { char buf[64]; @@ -170,13 +159,6 @@ void common_test_array(CollectionValue src_val) { array_type->deep_copy((char*)&dst_val, (char*)&src_val, pool); EXPECT_EQ(0, array_type->cmp((char*)&src_val, (char*)&dst_val)); } - { // test direct copy - bool null_signs[50]; - uint8_t data[50]; - CollectionValue dst_val(data, sizeof(null_signs), null_signs); - array_type->direct_copy((char*)&dst_val, (char*)&src_val); - EXPECT_EQ(0, array_type->cmp((char*)&src_val, (char*)&dst_val)); - } } TEST(ArrayTypeTest, copy_and_equal) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/PointQueryExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/PointQueryExecutor.java index 8ddf2487313817..ea6471b8243382 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/PointQueryExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/PointQueryExecutor.java @@ -19,11 +19,13 @@ import org.apache.doris.analysis.BinaryPredicate; import org.apache.doris.analysis.Expr; +import org.apache.doris.analysis.ExprToThriftVisitor; import org.apache.doris.analysis.LiteralExpr; import org.apache.doris.analysis.SlotRef; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.Env; import org.apache.doris.catalog.OlapTable; +import org.apache.doris.catalog.Type; import org.apache.doris.cloud.catalog.CloudPartition; import org.apache.doris.common.Config; import org.apache.doris.common.Status; @@ -41,6 +43,8 @@ import org.apache.doris.rpc.RpcException; import org.apache.doris.rpc.TCustomProtocolFactory; import org.apache.doris.system.Backend; +import org.apache.doris.thrift.TExpr; +import org.apache.doris.thrift.TExprNode; import org.apache.doris.thrift.TNetworkAddress; import org.apache.doris.thrift.TResultBatch; import org.apache.doris.thrift.TScanRangeLocations; @@ -54,6 +58,7 @@ import org.apache.logging.log4j.Logger; import org.apache.thrift.TDeserializer; import org.apache.thrift.TException; +import org.apache.thrift.TSerializer; import java.util.ArrayList; import java.util.Collections; @@ -191,7 +196,7 @@ public void setTimeout(long timeoutMs) { } void addKeyTuples( - InternalService.PTabletKeyLookupRequest.Builder requestBuilder) { + InternalService.PTabletKeyLookupRequest.Builder requestBuilder) throws TException { // TODO handle IN predicates Map columnExpr = Maps.newHashMap(); KeyTuple.Builder kBuilder = KeyTuple.newBuilder(); @@ -202,9 +207,39 @@ void addKeyTuples( SlotRef columnSlot = left.unwrapSlotRef(); columnExpr.put(columnSlot.getColumnName(), right); } - // add key tuple in keys order + // Serialize each literal expr as TExprNode bytes for typed value transfer. + // BE deserializes the TExprNode and uses DataType::get_field() to extract + // typed Field values directly, avoiding string parsing. + TSerializer serializer = new TSerializer(); for (Column column : shortCircuitQueryContext.scanNode.getOlapTable().getBaseSchemaKeyColumns()) { - kBuilder.addKeyColumnRep(columnExpr.get(column.getName()).getStringValue()); + Expr literalExpr = columnExpr.get(column.getName()); + // Ensure the literal type matches the column type for proper TExprNode + // deserialization on BE side. Prepared statement parameters may have + // mismatched types (e.g., setBigDecimal for INT column produces a + // DecimalLiteral, but BE expects INT_LITERAL for INT columns). + if (literalExpr instanceof LiteralExpr) { + Type colType = column.getType(); + if (!colType.equals(literalExpr.getType()) + && !colType.matchesType(literalExpr.getType())) { + try { + literalExpr = LiteralExpr.create( + ((LiteralExpr) literalExpr).getStringValue(), colType); + } catch (org.apache.doris.common.AnalysisException e) { + throw new TException("Failed to re-type literal for key column " + + column.getName() + ": " + e.getMessage(), e); + } + } + } + TExpr texpr = ExprToThriftVisitor.treeToThrift(literalExpr); + // For point queries, key column values are always simple literals + // (CastExpr no-ops are already stripped by treeToThrift). + Preconditions.checkState(texpr.getNodesSize() == 1, + "Expected single TExprNode for key column literal of " + column.getName() + + ", got " + texpr.getNodesSize()); + TExprNode exprNode = texpr.getNodes().get(0); + byte[] serialized = serializer.serialize(exprNode); + kBuilder.addKeyColumnLiterals( + com.google.protobuf.ByteString.copyFrom(serialized)); } requestBuilder.addKeyTuples(kBuilder); } diff --git a/gensrc/proto/internal_service.proto b/gensrc/proto/internal_service.proto index e047e5ca8c9e57..a5f684e6d3c295 100644 --- a/gensrc/proto/internal_service.proto +++ b/gensrc/proto/internal_service.proto @@ -314,6 +314,11 @@ message PFetchArrowFlightSchemaResult { message KeyTuple { repeated string key_column_rep = 1; + // Each entry is a serialized TExprNode (Thrift binary protocol) representing + // a literal value for the corresponding key column. When present, BE uses + // DataType::get_field(TExprNode) to extract typed Field values directly, + // avoiding the string-parsing path used by key_column_rep. + repeated bytes key_column_literals = 2; } message UUID { From 6377c5f763121565f2bdd428fbf1cae42f9a958d Mon Sep 17 00:00:00 2001 From: yiguolei Date: Thu, 26 Mar 2026 13:52:34 +0800 Subject: [PATCH 2/4] f --- be/src/exec/operator/scan_operator.cpp | 42 +++++++++++--------------- 1 file changed, 18 insertions(+), 24 deletions(-) diff --git a/be/src/exec/operator/scan_operator.cpp b/be/src/exec/operator/scan_operator.cpp index 111cae9af0196e..36cf949a637027 100644 --- a/be/src/exec/operator/scan_operator.cpp +++ b/be/src/exec/operator/scan_operator.cpp @@ -229,30 +229,24 @@ static void init_slot_value_range( slot_id_to_value_range[slot->id()] = std::move(range); \ break; \ } -#define APPLY_FOR_SCALAR_TYPE(M) \ - M(TINYINT) \ - M(SMALLINT) \ - M(INT) \ - M(BIGINT) \ - M(LARGEINT) \ - M(FLOAT) \ - M(DOUBLE) \ - M(CHAR) \ - M(DATE) \ - M(DATETIME) \ - M(DATEV2) \ - M(DATETIMEV2) \ - M(TIMESTAMPTZ) \ - M(VARCHAR) \ - M(STRING) \ - M(DECIMAL32) \ - M(DECIMAL64) \ - M(DECIMAL128I) \ - M(DECIMAL256) \ - M(DECIMALV2) \ - M(BOOLEAN) \ - M(IPV4) \ - M(IPV6) +#define APPLY_FOR_SCALAR_TYPE(M) \ + M(TINYINT) \ + M(SMALLINT) \ + M(INT) \ + M(BIGINT) \ + M(LARGEINT) \ + M(FLOAT) \ + M(DOUBLE) \ + M(CHAR) \ + M(DATE) \ + M(DATETIME) \ + M(DATEV2) \ + M(DATETIMEV2) \ + M(TIMESTAMPTZ) \ + M(VARCHAR) \ + M(VARCHAR) \ + M(STRING) M(DECIMAL32) M(DECIMAL64) M(DECIMAL128I) M(DECIMAL256) M(DECIMALV2) M(BOOLEAN) \ + M(IPV4) M(IPV6) APPLY_FOR_PRIMITIVE_TYPE(M) #undef M default: { From b60814e0bb89dd680c2a8e490815687a583d73ae Mon Sep 17 00:00:00 2001 From: yiguolei Date: Thu, 26 Mar 2026 15:35:39 +0800 Subject: [PATCH 3/4] f --- be/src/exec/operator/scan_operator.cpp | 46 ++++++++++++++------------ 1 file changed, 25 insertions(+), 21 deletions(-) diff --git a/be/src/exec/operator/scan_operator.cpp b/be/src/exec/operator/scan_operator.cpp index 36cf949a637027..3864fbf63fc05b 100644 --- a/be/src/exec/operator/scan_operator.cpp +++ b/be/src/exec/operator/scan_operator.cpp @@ -216,7 +216,6 @@ static std::string predicates_to_string( } return fmt::to_string(debug_string_buffer); } - static void init_slot_value_range( phmap::flat_hash_map& slot_id_to_value_range, SlotDescriptor* slot, const DataTypePtr type_desc) { @@ -229,28 +228,33 @@ static void init_slot_value_range( slot_id_to_value_range[slot->id()] = std::move(range); \ break; \ } -#define APPLY_FOR_SCALAR_TYPE(M) \ - M(TINYINT) \ - M(SMALLINT) \ - M(INT) \ - M(BIGINT) \ - M(LARGEINT) \ - M(FLOAT) \ - M(DOUBLE) \ - M(CHAR) \ - M(DATE) \ - M(DATETIME) \ - M(DATEV2) \ - M(DATETIMEV2) \ - M(TIMESTAMPTZ) \ - M(VARCHAR) \ - M(VARCHAR) \ - M(STRING) M(DECIMAL32) M(DECIMAL64) M(DECIMAL128I) M(DECIMAL256) M(DECIMALV2) M(BOOLEAN) \ - M(IPV4) M(IPV6) - APPLY_FOR_PRIMITIVE_TYPE(M) +#define APPLY_FOR_SCALAR_TYPE(M) \ + M(TINYINT) \ + M(SMALLINT) \ + M(INT) \ + M(BIGINT) \ + M(LARGEINT) \ + M(FLOAT) \ + M(DOUBLE) \ + M(CHAR) \ + M(DATE) \ + M(DATETIME) \ + M(DATEV2) \ + M(DATETIMEV2) \ + M(TIMESTAMPTZ) \ + M(VARCHAR) \ + M(STRING) \ + M(DECIMAL32) \ + M(DECIMAL64) \ + M(DECIMAL128I) \ + M(DECIMAL256) \ + M(DECIMALV2) \ + M(BOOLEAN) \ + M(IPV4) \ + M(IPV6) + APPLY_FOR_SCALAR_TYPE(M) #undef M default: { - VLOG_CRITICAL << "Unsupported Normalize Slot [ColName=" << slot->col_name() << "]"; break; } } From 75c8451736e9b1c01c166abf33228f67ab842738 Mon Sep 17 00:00:00 2001 From: yiguolei Date: Thu, 26 Mar 2026 18:56:03 +0800 Subject: [PATCH 4/4] f --- .../src/main/java/org/apache/doris/qe/PointQueryExecutor.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/PointQueryExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/PointQueryExecutor.java index ea6471b8243382..15c529ef140f0d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/PointQueryExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/PointQueryExecutor.java @@ -19,7 +19,6 @@ import org.apache.doris.analysis.BinaryPredicate; import org.apache.doris.analysis.Expr; -import org.apache.doris.analysis.ExprToThriftVisitor; import org.apache.doris.analysis.LiteralExpr; import org.apache.doris.analysis.SlotRef; import org.apache.doris.catalog.Column; @@ -230,7 +229,7 @@ void addKeyTuples( } } } - TExpr texpr = ExprToThriftVisitor.treeToThrift(literalExpr); + TExpr texpr = literalExpr.treeToThrift(); // For point queries, key column values are always simple literals // (CastExpr no-ops are already stripped by treeToThrift). Preconditions.checkState(texpr.getNodesSize() == 1,