diff --git a/Makefile.am b/Makefile.am index 921d52ea..bd4ccb34 100644 --- a/Makefile.am +++ b/Makefile.am @@ -37,6 +37,7 @@ src_libbitcoin_server_la_LIBADD = ${bitcoin_node_LIBS} src_libbitcoin_server_la_SOURCES = \ src/configuration.cpp \ src/error.cpp \ + src/estimator.cpp \ src/parser.cpp \ src/server_node.cpp \ src/settings.cpp \ @@ -65,6 +66,7 @@ test_libbitcoin_server_test_LDADD = src/libbitcoin-server.la ${boost_unit_test_f test_libbitcoin_server_test_SOURCES = \ test/configuration.cpp \ test/error.cpp \ + test/estimator.cpp \ test/main.cpp \ test/settings.cpp \ test/test.cpp \ @@ -128,6 +130,7 @@ include_bitcoin_server_HEADERS = \ include/bitcoin/server/configuration.hpp \ include/bitcoin/server/define.hpp \ include/bitcoin/server/error.hpp \ + include/bitcoin/server/estimator.hpp \ include/bitcoin/server/parser.hpp \ include/bitcoin/server/server_node.hpp \ include/bitcoin/server/settings.hpp \ diff --git a/builds/cmake/CMakeLists.txt b/builds/cmake/CMakeLists.txt index 29f614fa..33c1907f 100644 --- a/builds/cmake/CMakeLists.txt +++ b/builds/cmake/CMakeLists.txt @@ -236,6 +236,7 @@ add_definitions( add_library( ${CANONICAL_LIB_NAME} "../../src/configuration.cpp" "../../src/error.cpp" + "../../src/estimator.cpp" "../../src/parser.cpp" "../../src/server_node.cpp" "../../src/settings.cpp" @@ -288,10 +289,17 @@ if (with-tests) add_executable( libbitcoin-server-test "../../test/configuration.cpp" "../../test/error.cpp" + "../../test/estimator.cpp" "../../test/main.cpp" "../../test/settings.cpp" "../../test/test.cpp" "../../test/test.hpp" + "../../test/endpoints/README.md" + "../../test/endpoints/bitcoind_rpc.py" + "../../test/endpoints/configuration.py" + "../../test/endpoints/electrum.py" + "../../test/endpoints/native.py" + "../../test/endpoints/utils.py" "../../test/parsers/bitcoind_query.cpp" "../../test/parsers/bitcoind_target.cpp" "../../test/parsers/native_query.cpp" diff --git a/builds/msvc/vs2022/libbitcoin-server-test/libbitcoin-server-test.vcxproj b/builds/msvc/vs2022/libbitcoin-server-test/libbitcoin-server-test.vcxproj index e160f516..e5c83e43 100644 --- a/builds/msvc/vs2022/libbitcoin-server-test/libbitcoin-server-test.vcxproj +++ b/builds/msvc/vs2022/libbitcoin-server-test/libbitcoin-server-test.vcxproj @@ -118,8 +118,11 @@ - + + $(IntDir)test_configuration.obj + + diff --git a/builds/msvc/vs2022/libbitcoin-server-test/libbitcoin-server-test.vcxproj.filters b/builds/msvc/vs2022/libbitcoin-server-test/libbitcoin-server-test.vcxproj.filters index ff807ff3..9c888f35 100644 --- a/builds/msvc/vs2022/libbitcoin-server-test/libbitcoin-server-test.vcxproj.filters +++ b/builds/msvc/vs2022/libbitcoin-server-test/libbitcoin-server-test.vcxproj.filters @@ -10,9 +10,12 @@ {66A0E586-2E3A-448F-0000-000000000000} - + {66A0E586-2E3A-448F-0000-000000000001} + + {66A0E586-2E3A-448F-0000-000000000002} + @@ -21,6 +24,9 @@ src + + src + src diff --git a/builds/msvc/vs2022/libbitcoin-server/libbitcoin-server.vcxproj b/builds/msvc/vs2022/libbitcoin-server/libbitcoin-server.vcxproj index d83044cf..8a53b6b9 100644 --- a/builds/msvc/vs2022/libbitcoin-server/libbitcoin-server.vcxproj +++ b/builds/msvc/vs2022/libbitcoin-server/libbitcoin-server.vcxproj @@ -123,6 +123,7 @@ + @@ -151,6 +152,7 @@ + diff --git a/builds/msvc/vs2022/libbitcoin-server/libbitcoin-server.vcxproj.filters b/builds/msvc/vs2022/libbitcoin-server/libbitcoin-server.vcxproj.filters index 7989780a..de1d323e 100644 --- a/builds/msvc/vs2022/libbitcoin-server/libbitcoin-server.vcxproj.filters +++ b/builds/msvc/vs2022/libbitcoin-server/libbitcoin-server.vcxproj.filters @@ -57,6 +57,9 @@ src + + src + src @@ -137,6 +140,9 @@ include\bitcoin\server + + include\bitcoin\server + include\bitcoin\server\interfaces diff --git a/include/bitcoin/server.hpp b/include/bitcoin/server.hpp index 9772848e..69f03294 100644 --- a/include/bitcoin/server.hpp +++ b/include/bitcoin/server.hpp @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include diff --git a/include/bitcoin/server/estimator.hpp b/include/bitcoin/server/estimator.hpp new file mode 100644 index 00000000..41e86103 --- /dev/null +++ b/include/bitcoin/server/estimator.hpp @@ -0,0 +1,148 @@ +/** + * Copyright (c) 2011-2025 libbitcoin developers (see AUTHORS) + * + * This file is part of libbitcoin. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#ifndef LIBBITCOIN_SERVER_ESTIMATOR_HPP +#define LIBBITCOIN_SERVER_ESTIMATOR_HPP + +#include +#include + +namespace libbitcoin { +namespace server { + +class BCS_API estimator +{ +public: + DELETE_COPY_MOVE_DESTRUCT(estimator); + + /// Estimation modes. + enum class mode + { + basic, + markov, + economical, + conservative + }; + + estimator() NOEXCEPT {}; + + /// Fee estimation in satoshis / transaction virtual size. + /// Pass zero to target next block for confirmation, range:0..1007. + uint64_t estimate(size_t target, mode mode) const NOEXCEPT; + + /// Populate accumulator. + bool initialize(std::atomic_bool& cancel, const node::query& query, + size_t top, size_t count) NOEXCEPT; + + /// Update accumulator. + bool push(const node::query& query) NOEXCEPT; + bool pop(const node::query& query) NOEXCEPT; + + /// Top height of accumulator. + size_t top_height() const NOEXCEPT; + +protected: + using rates = database::fee_rates; + using rate_sets = database::fee_rate_sets; + + /// Bucket count sizing parameters. + enum horizon : size_t + { + small = 12, + medium = 48, + large = 1008 + }; + + /// Estimation confidences. + struct confidence + { + static constexpr double low = 0.60; + static constexpr double mid = 0.85; + static constexpr double high = 0.95; + }; + + /// Bucket count sizing parameters. + struct sizing + { + static constexpr double min = 0.1; + static constexpr double max = 100'000.0; + static constexpr double step = 1.05; + + /// Derived from min/max/step above. + static constexpr size_t count = 283; + }; + + /// Accumulator (persistent, decay-weighted counters). + struct accumulator + { + template + struct bucket + { + /// Total scaled txs in bucket. + std::atomic total{}; + + /// confirmed[n]: scaled txs confirmed in > n blocks. + std::array, Depth> confirmed; + }; + + /// Current block height of accumulated state. + size_t top_height{}; + + /// Accumulated scaled fee in decayed buckets by horizon. + /// Array count is the half life of the decay it implies. + std::array, sizing::count> small{}; + std::array, sizing::count> medium{}; + std::array, sizing::count> large{}; + }; + + // C++23: make consteval (std::pow). + static inline double decay_rate() NOEXCEPT + { + static const auto rate = std::pow(0.5, 1.0 / sizing::count); + return rate; + } + + static inline double to_scale_term(size_t age) NOEXCEPT + { + return std::pow(decay_rate(), age); + } + + static inline double to_scale_factor(bool push) NOEXCEPT + { + return std::pow(decay_rate(), push ? +1.0 : -1.0); + } + + const accumulator& history() const NOEXCEPT; + bool initialize(const rate_sets& blocks) NOEXCEPT; + bool push(const rates& block) NOEXCEPT; + bool pop(const rates& block) NOEXCEPT; + uint64_t compute(size_t target, double confidence, + bool markov=false) const NOEXCEPT; + +private: + bool update(const rates& block, size_t height, bool push) NOEXCEPT; + void decay(auto& buckets, double factor) NOEXCEPT; + void decay(bool push) NOEXCEPT; + + accumulator fees_{}; +}; + +} // namespace server +} // namespace libbitcoin + +#endif diff --git a/src/estimator.cpp b/src/estimator.cpp new file mode 100644 index 00000000..ed67d114 --- /dev/null +++ b/src/estimator.cpp @@ -0,0 +1,298 @@ +/** + * Copyright (c) 2011-2025 libbitcoin developers (see AUTHORS) + * + * This file is part of libbitcoin. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace libbitcoin { +namespace server { + +using namespace system; +constexpr auto relaxed = std::memory_order_relaxed; + +// public +// ---------------------------------------------------------------------------- + +uint64_t estimator::estimate(size_t target, mode mode) const NOEXCEPT +{ + // max_uint64 is failure sentinel (and unachievable/invalid as a fee). + auto estimate = max_uint64; + constexpr size_t large = horizon::large; + if (target >= large) + return estimate; + + // Valid results are effectively limited to at least 1 sat/vb. + // threshold_fee is thread safe but values are affected during update. + switch (mode) + { + case mode::basic: + { + estimate = compute(target, confidence::high); + break; + } + case mode::markov: + { + estimate = compute(target, confidence::high, true); + break; + } + case mode::economical: + { + const auto target1 = to_half(target); + const auto target2 = std::min(one, target); + const auto target3 = std::min(large, two * target); + const auto fee1 = compute(target1, confidence::low); + const auto fee2 = compute(target2, confidence::mid); + const auto fee3 = compute(target3, confidence::high); + estimate = std::max({ fee1, fee2, fee3 }); + break; + } + case mode::conservative: + { + const auto target1 = to_half(target); + const auto target2 = std::min(one, target); + const auto target3 = std::min(large, two * target); + const auto fee1 = compute(target1, confidence::low); + const auto fee2 = compute(target2, confidence::mid); + const auto fee3 = compute(target3, confidence::high); + estimate = std::max({ fee1, fee2, fee3 }); + break; + } + } + + return estimate; +} + +bool estimator::initialize(std::atomic_bool& cancel, const node::query& query, + size_t top, size_t count) NOEXCEPT +{ + rate_sets blocks{}; + return query.get_block_fees(cancel, blocks, top, count) && + initialize(blocks); +} + +bool estimator::push(const node::query& query) NOEXCEPT +{ + if (is_add_overflow(top_height(), one)) + return false; + + rates block{}; + const auto link = query.to_confirmed(add1(top_height())); + return query.get_block_fees(block, link) && push(block); +} + +bool estimator::pop(const node::query& query) NOEXCEPT +{ + if (is_subtract_overflow(top_height(), one)) + return false; + + rates block{}; + const auto link = query.to_confirmed(sub1(top_height())); + return query.get_block_fees(block, link) && pop(block); +} + +size_t estimator::top_height() const NOEXCEPT +{ + return fees_.top_height; +} + +// protected +// ---------------------------------------------------------------------------- + +const estimator::accumulator& estimator::history() const NOEXCEPT +{ + return fees_; +} + +bool estimator::initialize(const rate_sets& blocks) NOEXCEPT +{ + const auto count = blocks.size(); + if (is_zero(count)) + return true; + + const auto top = top_height(); + auto height = top - sub1(count); + if (system::is_subtract_overflow(top, sub1(count))) + return false; + + // TODO: could be parallel by block. + for (const auto& block: blocks) + if (!update(block, height++, true)) + return false; + + return true; +} + +// Blocks must be pushed in order (but independent of chain index). +bool estimator::push(const rates& block) NOEXCEPT +{ + decay(true); + return update(block, ++fees_.top_height, true); +} + +// Blocks must be pushed in order (but independent of chain index). +bool estimator::pop(const rates& block) NOEXCEPT +{ + const auto result = update(block, fees_.top_height--, false); + decay(false); + return result; +} + +uint64_t estimator::compute(size_t target, double confidence, + bool markov) const NOEXCEPT +{ + const auto threshold = [](double part, double total, size_t) NOEXCEPT + { + return part / total; + }; + + // Geometric distribution approximation, not a full Markov process. + const auto geometric = [](double part, double total, size_t target) NOEXCEPT + { + return system::power(part / total, target); + }; + + const auto call = [&](const auto& buckets) NOEXCEPT + { + constexpr auto magic_number = 2u; + const auto at_least_four = magic_number * add1(target); + const auto& contribution = markov ? geometric : threshold; + + double total{}, part{}; + auto index = buckets.size(); + auto found = index; + for (const auto& bucket: std::views::reverse(buckets)) + { + --index; + total += to_floating(bucket.total.load(relaxed)); + part += to_floating(bucket.confirmed[target].load(relaxed)); + if (total < at_least_four) + continue; + + if (contribution(part, total, target) > (1.0 - confidence)) + break; + + found = index; + } + + if (found == buckets.size()) + return max_uint64; + + const auto minimum = sizing::min * std::pow(sizing::step, found); + return to_ceilinged_integer(minimum); + }; + + if (target < horizon::small) return call(fees_.small); + if (target < horizon::medium) return call(fees_.medium); + if (target < horizon::large) return call(fees_.large); + return max_uint64; +} + +// private +// ---------------------------------------------------------------------------- + +void estimator::decay(bool push) NOEXCEPT +{ + // Not thread safe (use sequentially by block). + const auto factor = to_scale_factor(push); + decay(fees_.large, factor); + decay(fees_.medium, factor); + decay(fees_.small, factor); +} + +void estimator::decay(auto& buckets, double factor) NOEXCEPT +{ + // Not thread safe (apply sequentially by block). + const auto call = [factor](auto& count) NOEXCEPT + { + const auto value = count.load(relaxed); + count.store(system::to_floored_integer(value * factor), relaxed); + }; + + for (auto& bucket: buckets) + { + call(bucket.total); + for (auto& count: bucket.confirmed) + call(count); + } +} + +bool estimator::update(const rates& block, size_t height, bool push) NOEXCEPT +{ + // std::log (replace static with constexpr in c++26). + static const auto growth = std::log(sizing::step); + std::array counts{}; + + for (const auto& tx: block) + { + if (is_zero(tx.bytes)) + return false; + + if (is_zero(tx.fee)) + continue; + + const auto rate = to_floating(tx.fee) / tx.bytes; + if (rate < sizing::min) + continue; + + // Clamp overflow to last bin. + const auto bin = std::log(rate / sizing::min) / growth; + ++counts[std::min(to_floored_integer(bin), sub1(sizing::count))]; + } + + // At age zero scale term is one. + const auto age = top_height() - height; + const auto scale = to_scale_term(age); + const auto call = [&](auto& buckets) NOEXCEPT + { + // The array count of the buckets element type. + const auto depth = buckets.front().confirmed.size(); + + size_t bin{}; + for (const auto count: counts) + { + if (is_zero(count)) + { + ++bin; + continue; + } + + auto& bucket = buckets[bin++]; + const auto scaled = to_floored_integer(count * scale); + const auto signed_term = push ? scaled : twos_complement(scaled); + + bucket.total.fetch_add(signed_term, relaxed); + for (auto target = age; target < depth; ++target) + bucket.confirmed[target].fetch_add(signed_term, relaxed); + } + }; + + call(fees_.large); + call(fees_.medium); + call(fees_.small); + return true; +} + +} // namespace server +} // namespace libbitcoin diff --git a/test/estimator.cpp b/test/estimator.cpp new file mode 100644 index 00000000..f076e927 --- /dev/null +++ b/test/estimator.cpp @@ -0,0 +1,28 @@ +/** + * Copyright (c) 2011-2025 libbitcoin developers (see AUTHORS) + * + * This file is part of libbitcoin. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#include "test.hpp" + +BOOST_AUTO_TEST_SUITE(estimator_tests) + +BOOST_AUTO_TEST_CASE(estimator_test) +{ + BOOST_REQUIRE(true); +} + +BOOST_AUTO_TEST_SUITE_END()