|
| 1 | +#include <timeplus/blocking_queue.h> |
| 2 | +#include <timeplus/timeplus.h> |
| 3 | + |
| 4 | +#include <atomic> |
| 5 | +#include <chrono> |
| 6 | +#include <iomanip> |
| 7 | +#include <iostream> |
| 8 | +#include <thread> |
| 9 | + |
| 10 | +using namespace timeplus; |
| 11 | + |
| 12 | +const size_t INSERT_BLOCKS = 100'000; |
| 13 | +const size_t BLOCKS_PER_BATCH = 1000; |
| 14 | + |
| 15 | +const std::vector<std::pair<std::string, uint16_t>> HOST_PORTS = { |
| 16 | + /// Single instance |
| 17 | + {"localhost", 8463}, |
| 18 | + /// Cluster nodes |
| 19 | + {"localhost", 18463}, |
| 20 | + {"localhost", 28463}, |
| 21 | + {"localhost", 38463}, |
| 22 | +}; |
| 23 | + |
| 24 | +void prepareTable() { |
| 25 | + ClientOptions options; |
| 26 | + for (const auto& [host, port] : HOST_PORTS) { |
| 27 | + options.endpoints.push_back({host, port}); |
| 28 | + } |
| 29 | + |
| 30 | + Client client{options}; |
| 31 | + client.Execute("DROP STREAM IF EXISTS insert_async_test;"); |
| 32 | + client.Execute("CREATE STREAM insert_async_test (i uint64, s string);"); |
| 33 | +} |
| 34 | + |
| 35 | +auto timestamp() { |
| 36 | + auto now = std::chrono::system_clock::now(); |
| 37 | + std::time_t now_time = std::chrono::system_clock::to_time_t(now); |
| 38 | + std::tm* local_time = std::localtime(&now_time); |
| 39 | + return std::put_time(local_time, "%Y-%m-%d %H:%M:%S"); |
| 40 | +} |
| 41 | + |
| 42 | +int main() { |
| 43 | + prepareTable(); |
| 44 | + |
| 45 | + TimeplusConfig config; |
| 46 | + for (const auto& [host, port] : HOST_PORTS) { |
| 47 | + config.client_options.endpoints.push_back({host, port}); |
| 48 | + } |
| 49 | + config.max_connections = 4; |
| 50 | + config.max_retries = 5; |
| 51 | + config.task_executors = 4; |
| 52 | + config.task_queue_capacity = BLOCKS_PER_BATCH; /// use large input queue to avoid deadlock on retry failure |
| 53 | + |
| 54 | + Timeplus tp{std::move(config)}; |
| 55 | + |
| 56 | + auto block = std::make_shared<Block>(); |
| 57 | + auto col_i = std::make_shared<ColumnUInt64>(std::vector<uint64_t>{5, 7, 4, 8}); |
| 58 | + auto col_s = std::make_shared<ColumnString>( |
| 59 | + std::vector<std::string>{"Before my bed, the moon is bright,", "I think that it is frost on the ground.", |
| 60 | + "I raise my head to gaze at the bright moon,", "And lower it to think of my hometown."}); |
| 61 | + block->AppendColumn("i", col_i); |
| 62 | + block->AppendColumn("s", col_s); |
| 63 | + |
| 64 | + /// Queue to store failed inserts which need to be resent. |
| 65 | + BlockingQueue<std::pair<size_t, BlockPtr>> insert_failure(BLOCKS_PER_BATCH); |
| 66 | + std::atomic<size_t> insert_success_count{0}; |
| 67 | + |
| 68 | + auto handle_insert_result = [&insert_failure, &insert_success_count](size_t block_id, const InsertResult& result) { |
| 69 | + if (result.ok()) { |
| 70 | + insert_success_count.fetch_add(1); |
| 71 | + } else { |
| 72 | + std::cout << "[" << timestamp() << "]\t Failed to insert block: insert_id=" << block_id << " err=" << result.err_msg |
| 73 | + << std::endl; |
| 74 | + insert_failure.emplace(block_id, result.block); |
| 75 | + } |
| 76 | + }; |
| 77 | + |
| 78 | + auto async_insert_block = [&tp, &handle_insert_result](size_t block_id, BlockPtr block) { |
| 79 | + tp.InsertAsync(/*table_name=*/"insert_async_test", block, [block_id, &handle_insert_result](const BaseResult& result) { |
| 80 | + const auto& insert_result = static_cast<const InsertResult&>(result); |
| 81 | + handle_insert_result(block_id, insert_result); |
| 82 | + }); |
| 83 | + }; |
| 84 | + |
| 85 | + auto start_time = std::chrono::high_resolution_clock::now(); |
| 86 | + auto last_time = start_time; |
| 87 | + for (size_t batch = 0; batch < INSERT_BLOCKS / BLOCKS_PER_BATCH; ++batch) { |
| 88 | + insert_success_count = 0; |
| 89 | + /// Insert blocks asynchronously. |
| 90 | + for (size_t i = 0; i < BLOCKS_PER_BATCH; ++i) { |
| 91 | + async_insert_block(i, block); |
| 92 | + } |
| 93 | + |
| 94 | + /// Wait for all blocks of the batch are inserted. |
| 95 | + while (insert_success_count.load() != BLOCKS_PER_BATCH) { |
| 96 | + if (!insert_failure.empty()) { |
| 97 | + /// Re-insert the failed blocks |
| 98 | + auto blocks = insert_failure.drain(); |
| 99 | + for (auto &[i, b] : blocks) { |
| 100 | + async_insert_block(i, b); |
| 101 | + } |
| 102 | + } |
| 103 | + |
| 104 | + std::this_thread::yield(); |
| 105 | + } |
| 106 | + |
| 107 | + /// Print insert statistics of the batch. |
| 108 | + auto current_time = std::chrono::high_resolution_clock::now(); |
| 109 | + std::chrono::duration<double> elapsed = current_time - last_time; |
| 110 | + last_time = current_time; |
| 111 | + std::cout << "[" << timestamp() << "]\t" << (batch + 1) * BLOCKS_PER_BATCH << " blocks inserted\telapsed = " << elapsed.count() |
| 112 | + << " sec\teps = " << static_cast<double>(BLOCKS_PER_BATCH * block->GetRowCount()) / elapsed.count() << std::endl; |
| 113 | + } |
| 114 | + |
| 115 | + /// Print summary. |
| 116 | + auto current_time = std::chrono::high_resolution_clock::now(); |
| 117 | + std::chrono::duration<double> elapsed = current_time - start_time; |
| 118 | + std::cout << "\nInsert Done. Total Events = " << INSERT_BLOCKS * block->GetRowCount() << " Total Time = " << elapsed.count() << " sec" |
| 119 | + << std::endl; |
| 120 | + |
| 121 | + return 0; |
| 122 | +} |
0 commit comments