diff --git a/.gitignore b/.gitignore index 30e6548..9f04897 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,5 @@ heaptrack.*.zst context-snapshot.yaml check-quality.sh .claude/ +logs/ +env.sh diff --git a/Cargo.lock b/Cargo.lock index d525d9d..4736db2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1014,9 +1014,9 @@ dependencies = [ [[package]] name = "arrow" -version = "56.1.0" +version = "57.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c26b57282a08ae92f727497805122fec964c6245cfa0e13f0e75452eaf3bc41f" +checksum = "e4754a624e5ae42081f464514be454b39711daae0458906dacde5f4c632f33a8" dependencies = [ "arrow-arith", "arrow-array", @@ -1035,23 +1035,23 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "56.1.0" +version = "57.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cebf38ca279120ff522f4954b81a39527425b6e9f615e6b72842f4de1ffe02b8" +checksum = "f7b3141e0ec5145a22d8694ea8b6d6f69305971c4fa1c1a13ef0195aef2d678b" dependencies = [ "arrow-array", "arrow-buffer", "arrow-data", "arrow-schema", "chrono", - "num", + "num-traits", ] [[package]] name = "arrow-array" -version = "56.1.0" +version = "57.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "744109142cdf8e7b02795e240e20756c2a782ac9180d4992802954a8f871c0de" +checksum = "4c8955af33b25f3b175ee10af580577280b4bd01f7e823d94c7cdef7cf8c9aef" dependencies = [ "ahash", "arrow-buffer", @@ -1059,30 +1059,34 @@ dependencies = [ "arrow-schema", "chrono", "half", - "hashbrown 0.15.5", - "num", + "hashbrown 0.16.0", + "num-complex", + "num-integer", + "num-traits", ] [[package]] name = "arrow-buffer" -version = "56.1.0" +version = "57.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "601bb103c4c374bcd1f62c66bcea67b42a2ee91a690486c37d4c180236f11ccc" +checksum = "c697ddca96183182f35b3a18e50b9110b11e916d7b7799cbfd4d34662f2c56c2" dependencies = [ "bytes", "half", - "num", + "num-bigint", + "num-traits", ] [[package]] name = "arrow-cast" -version = "56.1.0" +version = "57.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eed61d9d73eda8df9e3014843def37af3050b5080a9acbe108f045a316d5a0be" +checksum = "646bbb821e86fd57189c10b4fcdaa941deaf4181924917b0daa92735baa6ada5" dependencies = [ "arrow-array", "arrow-buffer", "arrow-data", + "arrow-ord", "arrow-schema", "arrow-select", "atoi", @@ -1090,15 +1094,15 @@ dependencies = [ "chrono", "half", "lexical-core", - "num", + "num-traits", "ryu", ] [[package]] name = "arrow-csv" -version = "56.1.0" +version = "57.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa95b96ce0c06b4d33ac958370db8c0d31e88e54f9d6e08b0353d18374d9f991" +checksum = "8da746f4180004e3ce7b83c977daf6394d768332349d3d913998b10a120b790a" dependencies = [ "arrow-array", "arrow-cast", @@ -1111,21 +1115,22 @@ dependencies = [ [[package]] name = "arrow-data" -version = "56.1.0" +version = "57.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43407f2c6ba2367f64d85d4603d6fb9c4b92ed79d2ffd21021b37efa96523e12" +checksum = "1fdd994a9d28e6365aa78e15da3f3950c0fdcea6b963a12fa1c391afb637b304" dependencies = [ "arrow-buffer", "arrow-schema", "half", - "num", + "num-integer", + "num-traits", ] [[package]] name = "arrow-flight" -version = "56.1.0" +version = "57.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7c66c5e4a7aedc2bfebffeabc2116d76adb22e08d230b968b995da97f8b11ca" +checksum = "58c5b083668e6230eae3eab2fc4b5fb989974c845d0aa538dde61a4327c78675" dependencies = [ "arrow-array", "arrow-buffer", @@ -1138,13 +1143,14 @@ dependencies = [ "prost", "prost-types", "tonic", + "tonic-prost", ] [[package]] name = "arrow-ipc" -version = "56.1.0" +version = "57.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4b0487c4d2ad121cbc42c4db204f1509f8618e589bc77e635e9c40b502e3b90" +checksum = "abf7df950701ab528bf7c0cf7eeadc0445d03ef5d6ffc151eaae6b38a58feff1" dependencies = [ "arrow-array", "arrow-buffer", @@ -1156,9 +1162,9 @@ dependencies = [ [[package]] name = "arrow-json" -version = "56.1.0" +version = "57.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26d747573390905905a2dc4c5a61a96163fe2750457f90a04ee2a88680758c79" +checksum = "0ff8357658bedc49792b13e2e862b80df908171275f8e6e075c460da5ee4bf86" dependencies = [ "arrow-array", "arrow-buffer", @@ -1168,19 +1174,21 @@ dependencies = [ "chrono", "half", "indexmap 2.11.4", + "itoa", "lexical-core", "memchr", - "num", - "serde", + "num-traits", + "ryu", + "serde_core", "serde_json", "simdutf8", ] [[package]] name = "arrow-ord" -version = "56.1.0" +version = "57.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c142a147dceb59d057bad82400f1693847c80dca870d008bf7b91caf902810ae" +checksum = "f7d8f1870e03d4cbed632959498bcc84083b5a24bded52905ae1695bd29da45b" dependencies = [ "arrow-array", "arrow-buffer", @@ -1191,9 +1199,9 @@ dependencies = [ [[package]] name = "arrow-row" -version = "56.1.0" +version = "57.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dac6620667fccdab4204689ca173bd84a15de6bb6b756c3a8764d4d7d0c2fc04" +checksum = "18228633bad92bff92a95746bbeb16e5fc318e8382b75619dec26db79e4de4c0" dependencies = [ "arrow-array", "arrow-buffer", @@ -1204,29 +1212,29 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "56.1.0" +version = "57.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfa93af9ff2bb80de539e6eb2c1c8764abd0f4b73ffb0d7c82bf1f9868785e66" +checksum = "8c872d36b7bf2a6a6a2b40de9156265f0242910791db366a2c17476ba8330d68" [[package]] name = "arrow-select" -version = "56.1.0" +version = "57.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be8b2e0052cd20d36d64f32640b68a5ab54d805d24a473baee5d52017c85536c" +checksum = "68bf3e3efbd1278f770d67e5dc410257300b161b93baedb3aae836144edcaf4b" dependencies = [ "ahash", "arrow-array", "arrow-buffer", "arrow-data", "arrow-schema", - "num", + "num-traits", ] [[package]] name = "arrow-string" -version = "56.1.0" +version = "57.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2155e26e17f053c8975c546fc70cf19c00542f9abf43c23a88a46ef7204204f" +checksum = "85e968097061b3c0e9fe3079cf2e703e487890700546b5b0647f60fca1b5a8d8" dependencies = [ "arrow-array", "arrow-buffer", @@ -1234,7 +1242,7 @@ dependencies = [ "arrow-schema", "arrow-select", "memchr", - "num", + "num-traits", "regex", "regex-syntax", ] @@ -1650,6 +1658,21 @@ dependencies = [ "zeroize", ] +[[package]] +name = "bridge-test" +version = "0.1.0" +dependencies = [ + "arrow", + "clap", + "futures", + "phaser-client", + "phaser-types", + "serde_json", + "tokio", + "tracing", + "tracing-subscriber", +] + [[package]] name = "brotli" version = "8.0.2" @@ -1968,6 +1991,16 @@ dependencies = [ "wat-cpu 0.1.0 (git+https://github.com/dwerner/core-executor)", ] +[[package]] +name = "core-foundation" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "core-foundation-sys" version = "0.8.7" @@ -2385,8 +2418,8 @@ dependencies = [ "hyper-util", "lazy_static", "num_cpus", - "phaser-bridge", "phaser-metrics", + "phaser-server", "prometheus", "prost", "prost-types", @@ -2396,7 +2429,8 @@ dependencies = [ "tokio", "tokio-stream", "tonic", - "tonic-build", + "tonic-prost", + "tonic-prost-build", "tower 0.5.2", "tracing", "tracing-subscriber", @@ -2601,15 +2635,16 @@ checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" [[package]] name = "fusio" -version = "0.4.2" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42fe478d86cfabaeaa4f4d7353593b3738ca4ec449898ef34c3fe8370c5b8b5c" +checksum = "c9d1040b5fd481e1e09c184150d8ecb1bb3f69acf8b268001c34250aca0c1ecd" dependencies = [ "async-lock", "async-stream", "bytes", "cfg-if", "fusio-core", + "futures-channel", "futures-core", "futures-executor", "futures-util", @@ -2619,13 +2654,15 @@ dependencies = [ "thiserror 2.0.16", "tokio", "url", + "web-time", + "windows-sys 0.59.0", ] [[package]] name = "fusio-core" -version = "0.4.2" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ae0e43ded4f23b055f4f8c3c132f574e15bd0d0f9e6693af7d1f84d8650a144" +checksum = "d2fbefd51de0865685dc028694d492496a7dd941f10bdae49b7f942b7499594c" dependencies = [ "bytes", "thiserror 2.0.16", @@ -2633,9 +2670,9 @@ dependencies = [ [[package]] name = "fusio-parquet" -version = "0.4.2" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c80ee4f8c6cece380996d6f2a087ce09c0694ad8a0e21c8599b766c9c5c68c94" +checksum = "fa1818ee46541f02704ee732ae9de48f87c77af6a8afa5eb1cb9af9099bf837d" dependencies = [ "bytes", "cfg-if", @@ -3070,7 +3107,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2 0.6.0", + "socket2", "tokio", "tower-service", "tracing", @@ -3392,6 +3429,7 @@ dependencies = [ "arrow-schema", "async-stream", "async-trait", + "axum 0.7.9", "clap", "evm-common", "futures", @@ -3399,7 +3437,9 @@ dependencies = [ "hex", "hyper-util", "num_cpus", - "phaser-bridge", + "phaser-metrics", + "phaser-server", + "prometheus", "serde", "serde_json", "thiserror 1.0.69", @@ -3720,9 +3760,9 @@ dependencies = [ [[package]] name = "lz4_flex" -version = "0.11.5" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08ab2867e3eeeca90e844d1940eab391c9dc5228783db2ed999acbc0a9ed375a" +checksum = "ab6473172471198271ff72e9379150e9dfd70d8e533e0752a27e515b48dd375e" dependencies = [ "twox-hash", ] @@ -3822,20 +3862,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "num" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" -dependencies = [ - "num-bigint", - "num-complex", - "num-integer", - "num-iter", - "num-rational", - "num-traits", -] - [[package]] name = "num-bigint" version = "0.4.6" @@ -3870,28 +3896,6 @@ dependencies = [ "num-traits", ] -[[package]] -name = "num-iter" -version = "0.1.45" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf" -dependencies = [ - "autocfg", - "num-integer", - "num-traits", -] - -[[package]] -name = "num-rational" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" -dependencies = [ - "num-bigint", - "num-integer", - "num-traits", -] - [[package]] name = "num-traits" version = "0.2.19" @@ -3974,6 +3978,12 @@ version = "11.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" +[[package]] +name = "openssl-probe" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" + [[package]] name = "ordered-float" version = "2.10.1" @@ -4042,9 +4052,9 @@ dependencies = [ [[package]] name = "parquet" -version = "56.1.0" +version = "57.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89b56b41d1bd36aae415e42f91cae70ee75cf6cba74416b14dce3e958d5990ec" +checksum = "6ee96b29972a257b855ff2341b37e61af5f12d6af1158b6dcdb5b31ea07bb3cb" dependencies = [ "ahash", "arrow-array", @@ -4061,10 +4071,11 @@ dependencies = [ "flate2", "futures", "half", - "hashbrown 0.15.5", + "hashbrown 0.16.0", "lz4_flex", - "num", "num-bigint", + "num-integer", + "num-traits", "paste", "seq-macro", "simdutf8", @@ -4163,11 +4174,12 @@ dependencies = [ [[package]] name = "petgraph" -version = "0.7.1" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772" +checksum = "8701b58ea97060d5e5b155d383a69952a60943f0e6dfe30b04c287beb0b27455" dependencies = [ "fixedbitset", + "hashbrown 0.15.5", "indexmap 2.11.4", ] @@ -4182,20 +4194,19 @@ dependencies = [ ] [[package]] -name = "phaser-bridge" +name = "phaser-client" version = "0.1.0" dependencies = [ - "anyhow", "arrow", "arrow-array", "arrow-flight", "arrow-ipc", "arrow-schema", "async-stream", - "async-trait", - "bincode", "futures", + "http", "hyper-util", + "phaser-types", "prost", "serde", "serde_json", @@ -4232,7 +4243,6 @@ name = "phaser-metrics" version = "0.1.0" dependencies = [ "once_cell", - "phaser-bridge", "prometheus", "tracing", "tracing-subscriber", @@ -4274,7 +4284,7 @@ dependencies = [ "jsonrpsee", "num_cpus", "parquet", - "phaser-bridge", + "phaser-client", "phaser-metrics", "phaser-parquet-metadata", "prost", @@ -4289,7 +4299,8 @@ dependencies = [ "tokio", "tokio-stream", "tonic", - "tonic-build", + "tonic-prost", + "tonic-prost-build", "tracing", "tracing-subscriber", "typed-arrow", @@ -4297,6 +4308,39 @@ dependencies = [ "uuid", ] +[[package]] +name = "phaser-server" +version = "0.1.0" +dependencies = [ + "arrow", + "arrow-array", + "arrow-flight", + "arrow-schema", + "async-trait", + "futures", + "phaser-types", + "serde", + "serde_json", + "thiserror 1.0.69", + "tokio", + "tonic", + "tracing", +] + +[[package]] +name = "phaser-types" +version = "0.1.0" +dependencies = [ + "anyhow", + "arrow-array", + "arrow-flight", + "bincode", + "futures", + "serde", + "serde_json", + "thiserror 1.0.69", +] + [[package]] name = "pin-project" version = "1.1.10" @@ -4495,9 +4539,9 @@ dependencies = [ [[package]] name = "prost" -version = "0.13.5" +version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5" +checksum = "d2ea70524a2f82d518bce41317d0fae74151505651af45faf1ffbd6fd33f0568" dependencies = [ "bytes", "prost-derive", @@ -4505,19 +4549,20 @@ dependencies = [ [[package]] name = "prost-build" -version = "0.13.5" +version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" +checksum = "343d3bd7056eda839b03204e68deff7d1b13aba7af2b2fd16890697274262ee7" dependencies = [ "heck", "itertools 0.14.0", "log", "multimap", - "once_cell", "petgraph", "prettyplease", "prost", "prost-types", + "pulldown-cmark", + "pulldown-cmark-to-cmark", "regex", "syn 2.0.106", "tempfile", @@ -4525,9 +4570,9 @@ dependencies = [ [[package]] name = "prost-derive" -version = "0.13.5" +version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" +checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b" dependencies = [ "anyhow", "itertools 0.14.0", @@ -4538,9 +4583,9 @@ dependencies = [ [[package]] name = "prost-types" -version = "0.13.5" +version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52c2c1bf36ddb1a1c396b3601a3cec27c2462e45f07c386894ec3ccf5332bd16" +checksum = "8991c4cbdb8bc5b11f0b074ffe286c30e523de90fee5ba8132f1399f23cb3dd7" dependencies = [ "prost", ] @@ -4551,6 +4596,26 @@ version = "2.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "106dd99e98437432fed6519dedecfade6a06a73bb7b2a1e019fdd2bee5778d94" +[[package]] +name = "pulldown-cmark" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83c41efbf8f90ac44de7f3a868f0867851d261b56291732d0cbf7cceaaeb55a6" +dependencies = [ + "bitflags 2.9.4", + "memchr", + "unicase", +] + +[[package]] +name = "pulldown-cmark-to-cmark" +version = "22.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50793def1b900256624a709439404384204a5dc3a6ec580281bfaac35e882e90" +dependencies = [ + "pulldown-cmark", +] + [[package]] name = "quick-error" version = "1.2.3" @@ -4570,7 +4635,7 @@ dependencies = [ "quinn-udp", "rustc-hash 2.1.1", "rustls", - "socket2 0.6.0", + "socket2", "thiserror 2.0.16", "tokio", "tracing", @@ -4607,7 +4672,7 @@ dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2 0.6.0", + "socket2", "tracing", "windows-sys 0.60.2", ] @@ -4979,6 +5044,18 @@ dependencies = [ "zeroize", ] +[[package]] +name = "rustls-native-certs" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "612460d5f7bea540c490b2b6395d8e34a953e52b491accd6c86c8164c5932a63" +dependencies = [ + "openssl-probe", + "rustls-pki-types", + "schannel", + "security-framework", +] + [[package]] name = "rustls-pemfile" version = "2.2.0" @@ -5042,6 +5119,15 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "schannel" +version = "0.1.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "891d81b926048e76efe18581bf793546b4c0eaf8448d72be8de2bbee5fd166e1" +dependencies = [ + "windows-sys 0.61.1", +] + [[package]] name = "schemars" version = "0.9.0" @@ -5108,6 +5194,29 @@ dependencies = [ "cc", ] +[[package]] +name = "security-framework" +version = "3.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d17b898a6d6948c3a8ee4372c17cb384f90d2e6e912ef00895b14fd7ab54ec38" +dependencies = [ + "bitflags 2.9.4", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "semver" version = "0.11.0" @@ -5368,16 +5477,6 @@ version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b6b67fb9a61334225b5b790716f609cd58395f895b3fe8b328786812a40bc3b" -[[package]] -name = "socket2" -version = "0.5.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678" -dependencies = [ - "libc", - "windows-sys 0.52.0", -] - [[package]] name = "socket2" version = "0.6.0" @@ -5702,7 +5801,7 @@ dependencies = [ "pin-project-lite", "signal-hook-registry", "slab", - "socket2 0.6.0", + "socket2", "tokio-macros", "windows-sys 0.59.0", ] @@ -5802,9 +5901,9 @@ dependencies = [ [[package]] name = "tonic" -version = "0.13.1" +version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e581ba15a835f4d9ea06c55ab1bd4dce26fc53752c69a04aac00703bfb49ba9" +checksum = "fec7c61a0695dc1887c1b53952990f3ad2e3a31453e1f49f10e75424943a93ec" dependencies = [ "async-trait", "axum 0.8.4", @@ -5820,9 +5919,11 @@ dependencies = [ "hyper-util", "percent-encoding", "pin-project", - "prost", - "socket2 0.5.10", + "rustls-native-certs", + "socket2", + "sync_wrapper", "tokio", + "tokio-rustls", "tokio-stream", "tower 0.5.2", "tower-layer", @@ -5833,9 +5934,32 @@ dependencies = [ [[package]] name = "tonic-build" -version = "0.13.1" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1882ac3bf5ef12877d7ed57aad87e75154c11931c2ba7e6cde5e22d63522c734" +dependencies = [ + "prettyplease", + "proc-macro2", + "quote", + "syn 2.0.106", +] + +[[package]] +name = "tonic-prost" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a55376a0bbaa4975a3f10d009ad763d8f4108f067c7c2e74f3001fb49778d309" +dependencies = [ + "bytes", + "prost", + "tonic", +] + +[[package]] +name = "tonic-prost-build" +version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eac6f67be712d12f0b41328db3137e0d0757645d8904b4cb7d51cd9c2279e847" +checksum = "f3144df636917574672e93d0f56d7edec49f90305749c668df5101751bb8f95a" dependencies = [ "prettyplease", "proc-macro2", @@ -5843,6 +5967,8 @@ dependencies = [ "prost-types", "quote", "syn 2.0.106", + "tempfile", + "tonic-build", ] [[package]] @@ -6004,8 +6130,8 @@ checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c" [[package]] name = "typed-arrow" -version = "0.4.1" -source = "git+https://github.com/tonbo-io/typed-arrow?rev=30f2507147e8b48d51a26285bf8c84fdd58afa84#30f2507147e8b48d51a26285bf8c84fdd58afa84" +version = "0.6.1" +source = "git+https://github.com/tonbo-io/typed-arrow?rev=bb351a14915641d96d1888f5d8e946ed2bcca5ef#bb351a14915641d96d1888f5d8e946ed2bcca5ef" dependencies = [ "arrow-array", "arrow-buffer", @@ -6018,8 +6144,8 @@ dependencies = [ [[package]] name = "typed-arrow-derive" -version = "0.4.1" -source = "git+https://github.com/tonbo-io/typed-arrow?rev=30f2507147e8b48d51a26285bf8c84fdd58afa84#30f2507147e8b48d51a26285bf8c84fdd58afa84" +version = "0.6.1" +source = "git+https://github.com/tonbo-io/typed-arrow?rev=bb351a14915641d96d1888f5d8e946ed2bcca5ef#bb351a14915641d96d1888f5d8e946ed2bcca5ef" dependencies = [ "proc-macro2", "quote", @@ -6056,6 +6182,12 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" +[[package]] +name = "unicase" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbc4bc3a9f746d862c45cb89d705aa10f187bb96c76001afab07a0d35ce60142" + [[package]] name = "unicode-ident" version = "1.0.19" diff --git a/Cargo.toml b/Cargo.toml index db34912..2a46928 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,7 +2,9 @@ resolver = "2" members = [ "crates/phaser-query", - "crates/phaser-bridge", + "crates/phaser-types", + "crates/phaser-client", + "crates/phaser-server", "crates/phaser-metrics", "crates/bridges/evm/erigon-bridge", "crates/bridges/evm/jsonrpc-bridge", @@ -15,13 +17,16 @@ members = [ "crates/parquet-index-rocksdb", "crates/parquet-meta", "crates/phaser-parquet-metadata", + "crates/tools/bridge-test", ] [workspace.dependencies] # Internal crates erigon-bridge = { path = "crates/bridges/evm/erigon-bridge" } jsonrpc-bridge = { path = "crates/bridges/evm/jsonrpc-bridge" } -phaser-bridge = { path = "crates/phaser-bridge" } +phaser-types = { path = "crates/phaser-types" } +phaser-client = { path = "crates/phaser-client" } +phaser-server = { path = "crates/phaser-server" } phaser-metrics = { path = "crates/phaser-metrics" } evm-common = { path = "crates/schemas/evm/common", features = ["views"] } validators-evm = { path = "crates/validators/evm" } @@ -35,11 +40,13 @@ phaser-parquet-metadata = { path = "crates/phaser-parquet-metadata" } # External dependencies tokio = { version = "1.41", features = ["full"] } async-trait = "0.1" -datafusion = "49.0" -arrow = "56" -arrow-flight = "56" -arrow-array = "56" -typed-arrow = { git = "https://github.com/tonbo-io/typed-arrow", rev = "30f2507147e8b48d51a26285bf8c84fdd58afa84" } +datafusion = "52" +arrow = "57" +arrow-flight = "57" +arrow-array = "57" +arrow-schema = "57" +arrow-ipc = "57" +typed-arrow = { git = "https://github.com/tonbo-io/typed-arrow", rev = "bb351a14915641d96d1888f5d8e946ed2bcca5ef", default-features = false, features = ["derive", "views", "arrow-57"] } alloy-primitives = "1.3" alloy-consensus = "1.0" alloy-rlp = "0.3" @@ -71,12 +78,24 @@ axum = "0.7" jsonrpsee = { version = "0.24", features = ["server"] } rayon = "1.10" num_cpus = "1.16" -parquet = "56" -fusio = { version = "0.4", features = ["tokio"] } -fusio-parquet = "0.4" +parquet = "57" +fusio = { version = "0.6", features = ["tokio"] } +fusio-parquet = "0.6" futures = "0.3" # gRPC dependencies -tonic = { version = "0.13", features = ["gzip", "zstd"] } -prost = "0.13" -prost-types = "0.13" +tonic = { version = "0.14", features = ["transport", "gzip", "zstd", "tls-native-roots"] } +prost = "0.14" +prost-types = "0.14" + +# Release profile with debug info for profiling +[profile.release] +debug = true # Include debug symbols +strip = false # Don't strip symbols + +# Optimized release without debug info (for production) +[profile.release-prod] +inherits = "release" +debug = false +strip = true +lto = "thin" diff --git a/README.md b/README.md index 7e1071f..6558fc0 100644 --- a/README.md +++ b/README.md @@ -67,7 +67,7 @@ The core library defining the Arrow Flight protocol abstraction: - **`FlightBridge` trait**: Interface that bridge implementations must satisfy - **`BlockchainDescriptor`**: Specifies what data to stream (type, range, filters) - **`StreamType`**: Blocks, Transactions, Logs, State, etc. -- **`FlightBridgeClient`**: Client for connecting to any bridge implementation +- **`PhaserClient`**: Client for connecting to any bridge implementation - **`FlightBridgeServer`**: Server wrapper for exposing a bridge via Flight **Compression Support:** diff --git a/crates/bridges/evm/erigon-bridge/Cargo.toml b/crates/bridges/evm/erigon-bridge/Cargo.toml index 064f3ec..f2a1e66 100644 --- a/crates/bridges/evm/erigon-bridge/Cargo.toml +++ b/crates/bridges/evm/erigon-bridge/Cargo.toml @@ -13,7 +13,7 @@ path = "src/main.rs" [dependencies] # Local dependencies -phaser-bridge = { workspace = true } +phaser-server = { workspace = true } phaser-metrics = { workspace = true } evm-common = { workspace = true } validators-evm = { workspace = true } @@ -24,13 +24,14 @@ typed-arrow = { workspace = true } # Arrow and Flight dependencies arrow = { workspace = true } arrow-flight = { workspace = true } -arrow-array = "56.1" -arrow-schema = "56.1" -arrow-ipc = "56.1" +arrow-array = { workspace = true } +arrow-schema = { workspace = true } +arrow-ipc = { workspace = true } # Async and gRPC tokio = { workspace = true } tonic = { workspace = true } +tonic-prost = "0.14" async-trait = { workspace = true } futures = "0.3" tokio-stream = "0.1" @@ -71,4 +72,4 @@ alloy-primitives = { workspace = true } derive_more = { version = "1.0", features = ["display"] } [build-dependencies] -tonic-build = "0.13" \ No newline at end of file +tonic-prost-build = "0.14" \ No newline at end of file diff --git a/crates/bridges/evm/erigon-bridge/build.rs b/crates/bridges/evm/erigon-bridge/build.rs index cee5830..256edc9 100644 --- a/crates/bridges/evm/erigon-bridge/build.rs +++ b/crates/bridges/evm/erigon-bridge/build.rs @@ -1,5 +1,5 @@ fn main() -> Result<(), Box> { - tonic_build::configure() + tonic_prost_build::configure() .build_server(false) .build_client(true) .out_dir("src/generated") diff --git a/crates/bridges/evm/erigon-bridge/src/blockdata_client.rs b/crates/bridges/evm/erigon-bridge/src/blockdata_client.rs index fb4b0b5..88336cf 100644 --- a/crates/bridges/evm/erigon-bridge/src/blockdata_client.rs +++ b/crates/bridges/evm/erigon-bridge/src/blockdata_client.rs @@ -14,8 +14,8 @@ pub struct BlockDataClient { endpoint: String, } -// Configure message size limits (128MB to handle large transaction batches) -const MAX_MESSAGE_SIZE: usize = 128 * 1024 * 1024; +// Configure message size limits (512MB to handle large log batches from dense blocks) +const MAX_MESSAGE_SIZE: usize = 512 * 1024 * 1024; impl BlockDataClient { /// Connect to Erigon's BlockDataBackend service diff --git a/crates/bridges/evm/erigon-bridge/src/bridge.rs b/crates/bridges/evm/erigon-bridge/src/bridge.rs index 9675554..969999d 100644 --- a/crates/bridges/evm/erigon-bridge/src/bridge.rs +++ b/crates/bridges/evm/erigon-bridge/src/bridge.rs @@ -5,10 +5,11 @@ use arrow_flight::{ use arrow_ipc::writer::IpcWriteOptions; use async_trait::async_trait; use futures::{stream, Stream, StreamExt as FuturesStreamExt}; -use phaser_bridge::{ - bridge::{BridgeCapabilities, FlightBridge}, - descriptors::{BridgeInfo, StreamType}, +use phaser_server::{ + BridgeCapabilities, BridgeInfo, DiscoveryCapabilities, FlightBridge, GenericQuery, + GenericQueryMode, StreamType, TableDescriptor, }; +use prost::Message as ProstMessage; use std::pin::Pin; use tonic::{Request, Response, Status, Streaming}; use tracing::{debug, error, info}; @@ -152,22 +153,28 @@ impl ErigonFlightBridge { } } - /// Parse a FlightDescriptor to extract the BlockchainDescriptor - fn parse_descriptor( - descriptor: &FlightDescriptor, - ) -> Result> { - if let Some(first) = descriptor.path.first() { - serde_json::from_str::(first).map_err( - |e| { - Box::new(TonicStatus::invalid_argument(format!( - "Invalid descriptor: {e}" - ))) - }, - ) - } else { - Err(Box::new(TonicStatus::invalid_argument( - "Empty descriptor path", - ))) + /// Parse a GenericQuery from a Flight Ticket + fn parse_ticket(ticket: &Ticket) -> Result { + GenericQuery::from_ticket(ticket) + .map_err(|e| TonicStatus::invalid_argument(format!("Invalid query in ticket: {e}"))) + } + + /// Parse a GenericQuery from a FlightDescriptor + fn parse_descriptor(descriptor: &FlightDescriptor) -> Result { + GenericQuery::from_flight_descriptor(descriptor) + .map_err(|e| TonicStatus::invalid_argument(format!("Invalid query in descriptor: {e}"))) + } + + /// Map table name to StreamType + fn table_to_stream_type(table: &str) -> Result { + match table { + "blocks" => Ok(StreamType::Blocks), + "transactions" => Ok(StreamType::Transactions), + "logs" => Ok(StreamType::Logs), + "trie" => Ok(StreamType::Trie), + other => Err(TonicStatus::invalid_argument(format!( + "Unknown table: {other}. Available: blocks, transactions, logs, trie" + ))), } } @@ -235,7 +242,7 @@ impl ErigonFlightBridge { start: u64, end: u64, validate: bool, - ) -> impl Stream> + ) -> impl Stream> + Send + 'static { let max_concurrent = config.max_concurrent_segments; @@ -364,7 +371,7 @@ impl ErigonFlightBridge { start: u64, end: u64, validate: bool, - ) -> impl Stream> + ) -> impl Stream> + Send + 'static { let max_concurrent = config.max_concurrent_segments; @@ -495,7 +502,7 @@ impl ErigonFlightBridge { Box< dyn Stream< Item = Result< - phaser_bridge::BatchWithRange, + phaser_server::BatchWithRange, arrow_flight::error::FlightError, >, > + Send @@ -618,12 +625,15 @@ impl ErigonFlightBridge { match batch_result { Ok(block_batch) => { batch_count += 1; - debug!("Received batch {} from BlockDataBackend with {} blocks", batch_count, block_batch.blocks.len()); + // Track gRPC message size for capacity planning + let msg_size = block_batch.encoded_len(); + metrics.grpc_message_size("blocks", msg_size); + debug!("Received batch {} from BlockDataBackend with {} blocks ({} bytes)", batch_count, block_batch.blocks.len(), msg_size); match BlockDataConverter::blocks_to_arrow(block_batch) { Ok(record_batch) => { debug!("Converted block batch {} with {} rows", batch_count, record_batch.num_rows()); // Wrap batch with responsibility range - let wrapped = phaser_bridge::BatchWithRange::new(record_batch, start, end); + let wrapped = phaser_server::BatchWithRange::new(record_batch, start, end); yield Ok(wrapped); } Err(e) => { @@ -710,6 +720,56 @@ impl FlightBridge for ErigonFlightBridge { }) } + async fn get_discovery_capabilities(&self) -> Result { + // Query current state from Erigon + let (current_block, oldest_block) = { + let mut client = self.client.lock().await; + let current = client.get_latest_block().await.unwrap_or(0); + // Erigon always has data from genesis (could be pruned, but we report 0) + (current, 0u64) + }; + + // Define available tables + let mut tables = vec![ + TableDescriptor::new("blocks", "_block_num") + .with_modes(vec!["historical", "live"]) + .with_sorted_by(vec!["_block_num"]), + TableDescriptor::new("transactions", "_block_num") + .with_modes(vec!["historical", "live"]) + .with_sorted_by(vec!["_block_num", "_tx_idx"]), + TableDescriptor::new("logs", "_block_num") + .with_modes(vec!["historical", "live"]) + .with_sorted_by(vec!["_block_num", "_tx_idx", "_log_idx"]), + ]; + + // Only advertise trie if available + if self.trie_client.is_some() { + tables.push( + TableDescriptor::new("trie", "_block_num") + .with_modes(vec!["live"]) + .with_sorted_by(vec!["_block_num"]), + ); + } + + // Add chain_id to metadata + let mut metadata = std::collections::HashMap::new(); + metadata.insert( + "chain_id".to_string(), + serde_json::Value::Number(self.chain_id.into()), + ); + + Ok(DiscoveryCapabilities { + name: "erigon-bridge".to_string(), + version: env!("CARGO_PKG_VERSION").to_string(), + protocol: "evm".to_string(), + position_label: "block_number".to_string(), + current_position: current_block, + oldest_position: oldest_block, + tables, + metadata, + }) + } + async fn handshake( &self, _request: Request>, @@ -757,9 +817,10 @@ impl FlightBridge for ErigonFlightBridge { request: Request, ) -> std::result::Result, Status> { let descriptor = request.into_inner(); - let blockchain_desc = Self::parse_descriptor(&descriptor).map_err(|e| *e)?; + let query = Self::parse_descriptor(&descriptor)?; + let stream_type = Self::table_to_stream_type(&query.table)?; - let info = create_flight_info(blockchain_desc.stream_type)?; + let info = create_flight_info(stream_type)?; Ok(Response::new(info)) } @@ -769,8 +830,9 @@ impl FlightBridge for ErigonFlightBridge { request: Request, ) -> Result, Status> { let descriptor = request.into_inner(); - let blockchain_desc = Self::parse_descriptor(&descriptor).map_err(|e| *e)?; - let schema = Self::get_schema_for_type(blockchain_desc.stream_type); + let query = Self::parse_descriptor(&descriptor)?; + let stream_type = Self::table_to_stream_type(&query.table)?; + let schema = Self::get_schema_for_type(stream_type); // Convert Arrow schema to IPC format for Flight // SchemaResult expects raw bytes - we need to encode the schema properly @@ -810,23 +872,20 @@ impl FlightBridge for ErigonFlightBridge { String::from_utf8_lossy(&ticket.ticket) ); - // Parse ticket to determine what data to stream - // The ticket contains a JSON-serialized BlockchainDescriptor - let blockchain_desc = String::from_utf8(ticket.ticket.to_vec()) - .map_err(|_| Status::invalid_argument("Ticket is not valid UTF-8")) - .and_then(|s| { - serde_json::from_str::(&s) - .map_err(|e| { - Status::invalid_argument(format!("Invalid descriptor in ticket: {e}")) - }) - })?; - let stream_type = blockchain_desc.stream_type; - let query_mode = blockchain_desc.query_mode.clone(); - let preferences = blockchain_desc.get_preferences(); + // Parse the GenericQuery from the ticket + let query = Self::parse_ticket(&ticket)?; + let stream_type = Self::table_to_stream_type(&query.table)?; + + // Extract enable_traces from filters if present + let enable_traces = query + .filters + .get("enable_traces") + .and_then(|v| v.as_bool()) + .unwrap_or(false); info!( - "Processing do_get for {:?} with mode {:?}, preferences: max_msg={} bytes, compression={:?}, batch_hint={}", - stream_type, query_mode, preferences.max_message_bytes, preferences.compression, preferences.batch_size_hint + "Processing do_get for table '{}' ({:?}) with mode {:?}", + query.table, stream_type, query.mode ); // Handle trie streaming separately @@ -847,43 +906,39 @@ impl FlightBridge for ErigonFlightBridge { return Ok(Response::new(Box::pin(flight_stream))); } - // Handle based on query mode - use phaser_bridge::subscription::QueryMode; - use phaser_bridge::ValidationStage; - - // Determine if we should do ingestion validation - let should_validate_ingestion = matches!( - blockchain_desc.validation, - ValidationStage::Ingestion | ValidationStage::Both - ); - + // Build the batch stream based on query mode let batch_stream: Pin< Box< dyn Stream< Item = Result< - phaser_bridge::BatchWithRange, + phaser_server::BatchWithRange, arrow_flight::error::FlightError, >, > + Send, >, - > = match query_mode { - QueryMode::Historical { start, end } => { - info!( - "Creating historical stream for blocks {}-{} (validation: {:?})", - start, end, blockchain_desc.validation - ); + > = match query.mode { + GenericQueryMode::Range { start, end } => { + info!("Creating historical stream for positions {}-{}", start, end); Box::pin( self.create_historical_stream( stream_type, start, end, - should_validate_ingestion, - blockchain_desc.enable_traces, + false, // No validation via generic query (can add filter later) + enable_traces, ) .await?, ) } - QueryMode::Live => { + GenericQueryMode::Snapshot { at } => { + // Snapshot is just a single-position range + info!("Creating snapshot at position {}", at); + Box::pin( + self.create_historical_stream(stream_type, at, at, false, enable_traces) + .await?, + ) + } + GenericQueryMode::Live => { info!("Creating live stream from current head"); let receiver = match stream_type { StreamType::Blocks => self.streaming_service.subscribe_blocks(), @@ -897,7 +952,7 @@ impl FlightBridge for ErigonFlightBridge { while let Ok(batch) = rx.recv().await { // For live mode, wrap batch with placeholder range // TODO: Extract actual block numbers from batch data - let wrapped = phaser_bridge::BatchWithRange::new(batch, 0, 0); + let wrapped = phaser_server::BatchWithRange::new(batch, 0, 0); yield Ok(wrapped); } }) @@ -1011,7 +1066,7 @@ impl FlightBridge for ErigonFlightBridge { let stream_type = if let Some(desc) = first.flight_descriptor { Self::parse_descriptor(&desc) - .map(|bd| bd.stream_type) + .and_then(|q| Self::table_to_stream_type(&q.table)) .unwrap_or(StreamType::Blocks) } else { StreamType::Blocks diff --git a/crates/bridges/evm/erigon-bridge/src/error.rs b/crates/bridges/evm/erigon-bridge/src/error.rs index 26cb4e1..5ee3f2a 100644 --- a/crates/bridges/evm/erigon-bridge/src/error.rs +++ b/crates/bridges/evm/erigon-bridge/src/error.rs @@ -42,7 +42,7 @@ pub enum ErigonBridgeError { ConnectionFailed(String), #[error("Stream protocol error: {0}")] - StreamProtocol(#[from] phaser_bridge::StreamError), + StreamProtocol(#[from] phaser_server::StreamError), } /// Implements conversion from tonic::Status to ErigonBridgeError diff --git a/crates/bridges/evm/erigon-bridge/src/generated/custom.rs b/crates/bridges/evm/erigon-bridge/src/generated/custom.rs index 5173cd4..85ddf86 100644 --- a/crates/bridges/evm/erigon-bridge/src/generated/custom.rs +++ b/crates/bridges/evm/erigon-bridge/src/generated/custom.rs @@ -2,7 +2,7 @@ /// Request state root for a block #[allow(dead_code)] #[allow(clippy::enum_variant_names)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct GetStateRootRequest { #[prost(oneof = "get_state_root_request::BlockId", tags = "1, 2, 3")] pub block_id: ::core::option::Option, @@ -11,7 +11,7 @@ pub struct GetStateRootRequest { pub mod get_state_root_request { #[allow(dead_code)] #[allow(clippy::enum_variant_names)] - #[derive(Clone, PartialEq, ::prost::Oneof)] + #[derive(Clone, PartialEq, Eq, Hash, ::prost::Oneof)] pub enum BlockId { #[prost(uint64, tag = "1")] BlockNumber(u64), @@ -24,7 +24,7 @@ pub mod get_state_root_request { } #[allow(dead_code)] #[allow(clippy::enum_variant_names)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct GetStateRootReply { /// 32-byte state root hash #[prost(bytes = "vec", tag = "1")] @@ -40,7 +40,7 @@ pub struct GetStateRootReply { /// Stream commitment nodes for full trie reconstruction #[allow(dead_code)] #[allow(clippy::enum_variant_names)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct StreamCommitmentRequest { /// Target state root (empty for latest) #[prost(bytes = "vec", tag = "1")] @@ -92,7 +92,7 @@ pub struct CommitmentNodeBatch { /// Individual commitment node from domain - minimal version #[allow(dead_code)] #[allow(clippy::enum_variant_names)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct CommitmentNode { /// Trie path/key from CommitmentDomain (variable length) #[prost(bytes = "vec", tag = "1")] @@ -110,7 +110,7 @@ pub struct CommitmentNode { /// Get specific nodes (for gap filling) #[allow(dead_code)] #[allow(clippy::enum_variant_names)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct GetNodesByHashRequest { /// List of 32-byte node hashes #[prost(bytes = "vec", repeated, tag = "1")] @@ -136,10 +136,10 @@ pub mod trie_backend_client { dead_code, missing_docs, clippy::wildcard_imports, - clippy::let_unit_value, + clippy::let_unit_value )] - use tonic::codegen::*; use tonic::codegen::http::Uri; + use tonic::codegen::*; /// TrieBackend service provides access to Merkle Patricia Trie data from CommitmentDomain #[derive(Debug, Clone)] pub struct TrieBackendClient { @@ -184,9 +184,8 @@ pub mod trie_backend_client { >::ResponseBody, >, >, - , - >>::Error: Into + std::marker::Send + std::marker::Sync, + >>::Error: + Into + std::marker::Send + std::marker::Sync, { TrieBackendClient::new(InterceptedService::new(inner, interceptor)) } @@ -225,22 +224,12 @@ pub mod trie_backend_client { pub async fn get_state_root( &mut self, request: impl tonic::IntoRequest, - ) -> std::result::Result< - tonic::Response, - tonic::Status, - > { - self.inner - .ready() - .await - .map_err(|e| { - tonic::Status::unknown( - format!("Service was not ready: {}", e.into()), - ) - })?; - let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static( - "/custom.TrieBackend/GetStateRoot", - ); + ) -> std::result::Result, tonic::Status> { + self.inner.ready().await.map_err(|e| { + tonic::Status::unknown(format!("Service was not ready: {}", e.into())) + })?; + let codec = tonic_prost::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static("/custom.TrieBackend/GetStateRoot"); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("custom.TrieBackend", "GetStateRoot")); @@ -254,43 +243,30 @@ pub mod trie_backend_client { tonic::Response>, tonic::Status, > { - self.inner - .ready() - .await - .map_err(|e| { - tonic::Status::unknown( - format!("Service was not ready: {}", e.into()), - ) - })?; - let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static( - "/custom.TrieBackend/StreamCommitmentNodes", - ); + self.inner.ready().await.map_err(|e| { + tonic::Status::unknown(format!("Service was not ready: {}", e.into())) + })?; + let codec = tonic_prost::ProstCodec::default(); + let path = + http::uri::PathAndQuery::from_static("/custom.TrieBackend/StreamCommitmentNodes"); let mut req = request.into_request(); - req.extensions_mut() - .insert(GrpcMethod::new("custom.TrieBackend", "StreamCommitmentNodes")); + req.extensions_mut().insert(GrpcMethod::new( + "custom.TrieBackend", + "StreamCommitmentNodes", + )); self.inner.server_streaming(req, path, codec).await } /// Get specific nodes by hash (for filling gaps) pub async fn get_nodes_by_hash( &mut self, request: impl tonic::IntoRequest, - ) -> std::result::Result< - tonic::Response, - tonic::Status, - > { - self.inner - .ready() - .await - .map_err(|e| { - tonic::Status::unknown( - format!("Service was not ready: {}", e.into()), - ) - })?; - let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static( - "/custom.TrieBackend/GetNodesByHash", - ); + ) -> std::result::Result, tonic::Status> + { + self.inner.ready().await.map_err(|e| { + tonic::Status::unknown(format!("Service was not ready: {}", e.into())) + })?; + let codec = tonic_prost::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static("/custom.TrieBackend/GetNodesByHash"); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("custom.TrieBackend", "GetNodesByHash")); @@ -301,7 +277,7 @@ pub mod trie_backend_client { /// Request to stream data for a block range #[allow(dead_code)] #[allow(clippy::enum_variant_names)] -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct BlockRangeRequest { /// Starting block number (inclusive) #[prost(uint64, tag = "1")] @@ -338,7 +314,7 @@ pub struct BlockBatch { /// RLP-encoded block header data #[allow(dead_code)] #[allow(clippy::enum_variant_names)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct BlockData { #[prost(uint64, tag = "1")] pub block_number: u64, @@ -371,7 +347,7 @@ pub struct TransactionBatch { /// RLP-encoded transaction data (no receipts - use ExecuteBlocks for receipts) #[allow(dead_code)] #[allow(clippy::enum_variant_names)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct TransactionData { #[prost(uint64, tag = "1")] pub block_number: u64, @@ -413,7 +389,7 @@ pub struct ReceiptBatch { /// RLP-encoded receipt data with optional trace data #[allow(dead_code)] #[allow(clippy::enum_variant_names)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct ReceiptData { #[prost(uint64, tag = "1")] pub block_number: u64, @@ -440,10 +416,10 @@ pub mod block_data_backend_client { dead_code, missing_docs, clippy::wildcard_imports, - clippy::let_unit_value, + clippy::let_unit_value )] - use tonic::codegen::*; use tonic::codegen::http::Uri; + use tonic::codegen::*; /// BlockDataBackend service provides blockchain data as RLP-encoded bytes /// This service streams data from both snapshots (frozen blocks) and live database #[derive(Debug, Clone)] @@ -489,9 +465,8 @@ pub mod block_data_backend_client { >::ResponseBody, >, >, - , - >>::Error: Into + std::marker::Send + std::marker::Sync, + >>::Error: + Into + std::marker::Send + std::marker::Sync, { BlockDataBackendClient::new(InterceptedService::new(inner, interceptor)) } @@ -534,18 +509,12 @@ pub mod block_data_backend_client { tonic::Response>, tonic::Status, > { - self.inner - .ready() - .await - .map_err(|e| { - tonic::Status::unknown( - format!("Service was not ready: {}", e.into()), - ) - })?; - let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static( - "/custom.BlockDataBackend/StreamBlocks", - ); + self.inner.ready().await.map_err(|e| { + tonic::Status::unknown(format!("Service was not ready: {}", e.into())) + })?; + let codec = tonic_prost::ProstCodec::default(); + let path = + http::uri::PathAndQuery::from_static("/custom.BlockDataBackend/StreamBlocks"); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("custom.BlockDataBackend", "StreamBlocks")); @@ -559,23 +528,17 @@ pub mod block_data_backend_client { tonic::Response>, tonic::Status, > { - self.inner - .ready() - .await - .map_err(|e| { - tonic::Status::unknown( - format!("Service was not ready: {}", e.into()), - ) - })?; - let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static( - "/custom.BlockDataBackend/StreamTransactions", - ); + self.inner.ready().await.map_err(|e| { + tonic::Status::unknown(format!("Service was not ready: {}", e.into())) + })?; + let codec = tonic_prost::ProstCodec::default(); + let path = + http::uri::PathAndQuery::from_static("/custom.BlockDataBackend/StreamTransactions"); let mut req = request.into_request(); - req.extensions_mut() - .insert( - GrpcMethod::new("custom.BlockDataBackend", "StreamTransactions"), - ); + req.extensions_mut().insert(GrpcMethod::new( + "custom.BlockDataBackend", + "StreamTransactions", + )); self.inner.server_streaming(req, path, codec).await } /// Execute blocks in range and stream RLP receipts (slow: executes blocks, generates receipts) @@ -587,18 +550,12 @@ pub mod block_data_backend_client { tonic::Response>, tonic::Status, > { - self.inner - .ready() - .await - .map_err(|e| { - tonic::Status::unknown( - format!("Service was not ready: {}", e.into()), - ) - })?; - let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static( - "/custom.BlockDataBackend/ExecuteBlocks", - ); + self.inner.ready().await.map_err(|e| { + tonic::Status::unknown(format!("Service was not ready: {}", e.into())) + })?; + let codec = tonic_prost::ProstCodec::default(); + let path = + http::uri::PathAndQuery::from_static("/custom.BlockDataBackend/ExecuteBlocks"); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("custom.BlockDataBackend", "ExecuteBlocks")); diff --git a/crates/bridges/evm/erigon-bridge/src/generated/mod.rs b/crates/bridges/evm/erigon-bridge/src/generated/mod.rs index 61713e7..a62ed96 100644 --- a/crates/bridges/evm/erigon-bridge/src/generated/mod.rs +++ b/crates/bridges/evm/erigon-bridge/src/generated/mod.rs @@ -1,11 +1,4 @@ -// Generated protobuf modules -// These are generated by build.rs from .proto files - -#[allow(clippy::all, dead_code)] +// Generated module exports pub mod custom; - -#[allow(clippy::all, dead_code)] pub mod remote; - -#[allow(clippy::all, dead_code)] pub mod types; diff --git a/crates/bridges/evm/erigon-bridge/src/generated/remote.rs b/crates/bridges/evm/erigon-bridge/src/generated/remote.rs index 89822e1..b47dc7b 100644 --- a/crates/bridges/evm/erigon-bridge/src/generated/remote.rs +++ b/crates/bridges/evm/erigon-bridge/src/generated/remote.rs @@ -1,14 +1,14 @@ // This file is @generated by prost-build. #[allow(dead_code)] #[allow(clippy::enum_variant_names)] -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct BorTxnLookupRequest { #[prost(message, optional, tag = "1")] pub bor_tx_hash: ::core::option::Option, } #[allow(dead_code)] #[allow(clippy::enum_variant_names)] -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct BorTxnLookupReply { #[prost(bool, tag = "1")] pub present: bool, @@ -17,7 +17,7 @@ pub struct BorTxnLookupReply { } #[allow(dead_code)] #[allow(clippy::enum_variant_names)] -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct BorEventsRequest { #[prost(uint64, tag = "1")] pub block_num: u64, @@ -26,7 +26,7 @@ pub struct BorEventsRequest { } #[allow(dead_code)] #[allow(clippy::enum_variant_names)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct BorEventsReply { #[prost(string, tag = "1")] pub state_receiver_contract_address: ::prost::alloc::string::String, @@ -35,7 +35,7 @@ pub struct BorEventsReply { } #[allow(dead_code)] #[allow(clippy::enum_variant_names)] -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct BorProducersRequest { #[prost(uint64, tag = "1")] pub block_num: u64, @@ -51,7 +51,7 @@ pub struct BorProducersResponse { } #[allow(dead_code)] #[allow(clippy::enum_variant_names)] -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct Validator { #[prost(uint64, tag = "1")] pub id: u64, @@ -69,10 +69,10 @@ pub mod bridge_backend_client { dead_code, missing_docs, clippy::wildcard_imports, - clippy::let_unit_value, + clippy::let_unit_value )] - use tonic::codegen::*; use tonic::codegen::http::Uri; + use tonic::codegen::*; #[derive(Debug, Clone)] pub struct BridgeBackendClient { inner: tonic::client::Grpc, @@ -116,9 +116,8 @@ pub mod bridge_backend_client { >::ResponseBody, >, >, - , - >>::Error: Into + std::marker::Send + std::marker::Sync, + >>::Error: + Into + std::marker::Send + std::marker::Sync, { BridgeBackendClient::new(InterceptedService::new(inner, interceptor)) } @@ -157,22 +156,13 @@ pub mod bridge_backend_client { pub async fn version( &mut self, request: impl tonic::IntoRequest<()>, - ) -> std::result::Result< - tonic::Response, - tonic::Status, - > { - self.inner - .ready() - .await - .map_err(|e| { - tonic::Status::unknown( - format!("Service was not ready: {}", e.into()), - ) - })?; - let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static( - "/remote.BridgeBackend/Version", - ); + ) -> std::result::Result, tonic::Status> + { + self.inner.ready().await.map_err(|e| { + tonic::Status::unknown(format!("Service was not ready: {}", e.into())) + })?; + let codec = tonic_prost::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static("/remote.BridgeBackend/Version"); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("remote.BridgeBackend", "Version")); @@ -181,22 +171,12 @@ pub mod bridge_backend_client { pub async fn bor_txn_lookup( &mut self, request: impl tonic::IntoRequest, - ) -> std::result::Result< - tonic::Response, - tonic::Status, - > { - self.inner - .ready() - .await - .map_err(|e| { - tonic::Status::unknown( - format!("Service was not ready: {}", e.into()), - ) - })?; - let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static( - "/remote.BridgeBackend/BorTxnLookup", - ); + ) -> std::result::Result, tonic::Status> { + self.inner.ready().await.map_err(|e| { + tonic::Status::unknown(format!("Service was not ready: {}", e.into())) + })?; + let codec = tonic_prost::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static("/remote.BridgeBackend/BorTxnLookup"); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("remote.BridgeBackend", "BorTxnLookup")); @@ -206,18 +186,11 @@ pub mod bridge_backend_client { &mut self, request: impl tonic::IntoRequest, ) -> std::result::Result, tonic::Status> { - self.inner - .ready() - .await - .map_err(|e| { - tonic::Status::unknown( - format!("Service was not ready: {}", e.into()), - ) - })?; - let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static( - "/remote.BridgeBackend/BorEvents", - ); + self.inner.ready().await.map_err(|e| { + tonic::Status::unknown(format!("Service was not ready: {}", e.into())) + })?; + let codec = tonic_prost::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static("/remote.BridgeBackend/BorEvents"); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("remote.BridgeBackend", "BorEvents")); @@ -232,10 +205,10 @@ pub mod heimdall_backend_client { dead_code, missing_docs, clippy::wildcard_imports, - clippy::let_unit_value, + clippy::let_unit_value )] - use tonic::codegen::*; use tonic::codegen::http::Uri; + use tonic::codegen::*; #[derive(Debug, Clone)] pub struct HeimdallBackendClient { inner: tonic::client::Grpc, @@ -279,9 +252,8 @@ pub mod heimdall_backend_client { >::ResponseBody, >, >, - , - >>::Error: Into + std::marker::Send + std::marker::Sync, + >>::Error: + Into + std::marker::Send + std::marker::Sync, { HeimdallBackendClient::new(InterceptedService::new(inner, interceptor)) } @@ -320,22 +292,13 @@ pub mod heimdall_backend_client { pub async fn version( &mut self, request: impl tonic::IntoRequest<()>, - ) -> std::result::Result< - tonic::Response, - tonic::Status, - > { - self.inner - .ready() - .await - .map_err(|e| { - tonic::Status::unknown( - format!("Service was not ready: {}", e.into()), - ) - })?; - let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static( - "/remote.HeimdallBackend/Version", - ); + ) -> std::result::Result, tonic::Status> + { + self.inner.ready().await.map_err(|e| { + tonic::Status::unknown(format!("Service was not ready: {}", e.into())) + })?; + let codec = tonic_prost::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static("/remote.HeimdallBackend/Version"); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("remote.HeimdallBackend", "Version")); @@ -344,22 +307,13 @@ pub mod heimdall_backend_client { pub async fn producers( &mut self, request: impl tonic::IntoRequest, - ) -> std::result::Result< - tonic::Response, - tonic::Status, - > { - self.inner - .ready() - .await - .map_err(|e| { - tonic::Status::unknown( - format!("Service was not ready: {}", e.into()), - ) - })?; - let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static( - "/remote.HeimdallBackend/Producers", - ); + ) -> std::result::Result, tonic::Status> + { + self.inner.ready().await.map_err(|e| { + tonic::Status::unknown(format!("Service was not ready: {}", e.into())) + })?; + let codec = tonic_prost::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static("/remote.HeimdallBackend/Producers"); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("remote.HeimdallBackend", "Producers")); @@ -369,22 +323,22 @@ pub mod heimdall_backend_client { } #[allow(dead_code)] #[allow(clippy::enum_variant_names)] -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct EtherbaseRequest {} #[allow(dead_code)] #[allow(clippy::enum_variant_names)] -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct EtherbaseReply { #[prost(message, optional, tag = "1")] pub address: ::core::option::Option, } #[allow(dead_code)] #[allow(clippy::enum_variant_names)] -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct NetVersionRequest {} #[allow(dead_code)] #[allow(clippy::enum_variant_names)] -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct NetVersionReply { #[prost(uint64, tag = "1")] pub id: u64, @@ -408,7 +362,7 @@ pub struct SyncingReply { pub mod syncing_reply { #[allow(dead_code)] #[allow(clippy::enum_variant_names)] - #[derive(Clone, PartialEq, ::prost::Message)] + #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct StageProgress { #[prost(string, tag = "1")] pub stage_name: ::prost::alloc::string::String, @@ -418,93 +372,93 @@ pub mod syncing_reply { } #[allow(dead_code)] #[allow(clippy::enum_variant_names)] -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct NetPeerCountRequest {} #[allow(dead_code)] #[allow(clippy::enum_variant_names)] -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct NetPeerCountReply { #[prost(uint64, tag = "1")] pub count: u64, } #[allow(dead_code)] #[allow(clippy::enum_variant_names)] -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct ProtocolVersionRequest {} #[allow(dead_code)] #[allow(clippy::enum_variant_names)] -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct ProtocolVersionReply { #[prost(uint64, tag = "1")] pub id: u64, } #[allow(dead_code)] #[allow(clippy::enum_variant_names)] -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct ClientVersionRequest {} #[allow(dead_code)] #[allow(clippy::enum_variant_names)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct ClientVersionReply { #[prost(string, tag = "1")] pub node_name: ::prost::alloc::string::String, } #[allow(dead_code)] #[allow(clippy::enum_variant_names)] -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct CanonicalHashRequest { #[prost(uint64, tag = "1")] pub block_number: u64, } #[allow(dead_code)] #[allow(clippy::enum_variant_names)] -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct CanonicalHashReply { #[prost(message, optional, tag = "1")] pub hash: ::core::option::Option, } #[allow(dead_code)] #[allow(clippy::enum_variant_names)] -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct HeaderNumberRequest { #[prost(message, optional, tag = "1")] pub hash: ::core::option::Option, } #[allow(dead_code)] #[allow(clippy::enum_variant_names)] -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct HeaderNumberReply { #[prost(uint64, optional, tag = "1")] pub number: ::core::option::Option, } #[allow(dead_code)] #[allow(clippy::enum_variant_names)] -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct CanonicalBodyForStorageRequest { #[prost(uint64, tag = "1")] pub block_number: u64, } #[allow(dead_code)] #[allow(clippy::enum_variant_names)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct CanonicalBodyForStorageReply { #[prost(bytes = "vec", tag = "1")] pub body: ::prost::alloc::vec::Vec, } #[allow(dead_code)] #[allow(clippy::enum_variant_names)] -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct SubscribeRequest { #[prost(enumeration = "Event", tag = "1")] pub r#type: i32, } #[allow(dead_code)] #[allow(clippy::enum_variant_names)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct SubscribeReply { #[prost(enumeration = "Event", tag = "1")] pub r#type: i32, - /// serialized data + /// serialized data #[prost(bytes = "vec", tag = "2")] pub data: ::prost::alloc::vec::Vec, } @@ -546,7 +500,7 @@ pub struct SubscribeLogsReply { } #[allow(dead_code)] #[allow(clippy::enum_variant_names)] -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct BlockRequest { #[prost(uint64, tag = "2")] pub block_height: u64, @@ -555,7 +509,7 @@ pub struct BlockRequest { } #[allow(dead_code)] #[allow(clippy::enum_variant_names)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct BlockReply { #[prost(bytes = "vec", tag = "1")] pub block_rlp: ::prost::alloc::vec::Vec, @@ -564,14 +518,14 @@ pub struct BlockReply { } #[allow(dead_code)] #[allow(clippy::enum_variant_names)] -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct TxnLookupRequest { #[prost(message, optional, tag = "1")] pub txn_hash: ::core::option::Option, } #[allow(dead_code)] #[allow(clippy::enum_variant_names)] -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct TxnLookupReply { #[prost(uint64, tag = "1")] pub block_number: u64, @@ -580,21 +534,21 @@ pub struct TxnLookupReply { } #[allow(dead_code)] #[allow(clippy::enum_variant_names)] -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct NodesInfoRequest { #[prost(uint32, tag = "1")] pub limit: u32, } #[allow(dead_code)] #[allow(clippy::enum_variant_names)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct AddPeerRequest { #[prost(string, tag = "1")] pub url: ::prost::alloc::string::String, } #[allow(dead_code)] #[allow(clippy::enum_variant_names)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct RemovePeerRequest { #[prost(string, tag = "1")] pub url: ::prost::alloc::string::String, @@ -615,21 +569,21 @@ pub struct PeersReply { } #[allow(dead_code)] #[allow(clippy::enum_variant_names)] -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct AddPeerReply { #[prost(bool, tag = "1")] pub success: bool, } #[allow(dead_code)] #[allow(clippy::enum_variant_names)] -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct RemovePeerReply { #[prost(bool, tag = "1")] pub success: bool, } #[allow(dead_code)] #[allow(clippy::enum_variant_names)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct PendingBlockReply { #[prost(bytes = "vec", tag = "1")] pub block_rlp: ::prost::alloc::vec::Vec, @@ -643,7 +597,7 @@ pub struct EngineGetPayloadBodiesByHashV1Request { } #[allow(dead_code)] #[allow(clippy::enum_variant_names)] -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct EngineGetPayloadBodiesByRangeV1Request { #[prost(uint64, tag = "1")] pub start: u64, @@ -659,21 +613,21 @@ pub struct AaValidationRequest { } #[allow(dead_code)] #[allow(clippy::enum_variant_names)] -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct AaValidationReply { #[prost(bool, tag = "1")] pub valid: bool, } #[allow(dead_code)] #[allow(clippy::enum_variant_names)] -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct BlockForTxNumRequest { #[prost(uint64, tag = "1")] pub txnum: u64, } #[allow(dead_code)] #[allow(clippy::enum_variant_names)] -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct BlockForTxNumResponse { #[prost(uint64, tag = "1")] pub block_number: u64, @@ -682,7 +636,7 @@ pub struct BlockForTxNumResponse { } #[allow(dead_code)] #[allow(clippy::enum_variant_names)] -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct MinimumBlockAvailableReply { #[prost(uint64, tag = "1")] pub block_num: u64, @@ -731,10 +685,10 @@ pub mod ethbackend_client { dead_code, missing_docs, clippy::wildcard_imports, - clippy::let_unit_value, + clippy::let_unit_value )] - use tonic::codegen::*; use tonic::codegen::http::Uri; + use tonic::codegen::*; #[derive(Debug, Clone)] pub struct EthbackendClient { inner: tonic::client::Grpc, @@ -778,9 +732,8 @@ pub mod ethbackend_client { >::ResponseBody, >, >, - , - >>::Error: Into + std::marker::Send + std::marker::Sync, + >>::Error: + Into + std::marker::Send + std::marker::Sync, { EthbackendClient::new(InterceptedService::new(inner, interceptor)) } @@ -819,18 +772,11 @@ pub mod ethbackend_client { &mut self, request: impl tonic::IntoRequest, ) -> std::result::Result, tonic::Status> { - self.inner - .ready() - .await - .map_err(|e| { - tonic::Status::unknown( - format!("Service was not ready: {}", e.into()), - ) - })?; - let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static( - "/remote.ETHBACKEND/Etherbase", - ); + self.inner.ready().await.map_err(|e| { + tonic::Status::unknown(format!("Service was not ready: {}", e.into())) + })?; + let codec = tonic_prost::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static("/remote.ETHBACKEND/Etherbase"); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("remote.ETHBACKEND", "Etherbase")); @@ -839,22 +785,12 @@ pub mod ethbackend_client { pub async fn net_version( &mut self, request: impl tonic::IntoRequest, - ) -> std::result::Result< - tonic::Response, - tonic::Status, - > { - self.inner - .ready() - .await - .map_err(|e| { - tonic::Status::unknown( - format!("Service was not ready: {}", e.into()), - ) - })?; - let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static( - "/remote.ETHBACKEND/NetVersion", - ); + ) -> std::result::Result, tonic::Status> { + self.inner.ready().await.map_err(|e| { + tonic::Status::unknown(format!("Service was not ready: {}", e.into())) + })?; + let codec = tonic_prost::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static("/remote.ETHBACKEND/NetVersion"); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("remote.ETHBACKEND", "NetVersion")); @@ -863,22 +799,12 @@ pub mod ethbackend_client { pub async fn net_peer_count( &mut self, request: impl tonic::IntoRequest, - ) -> std::result::Result< - tonic::Response, - tonic::Status, - > { - self.inner - .ready() - .await - .map_err(|e| { - tonic::Status::unknown( - format!("Service was not ready: {}", e.into()), - ) - })?; - let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static( - "/remote.ETHBACKEND/NetPeerCount", - ); + ) -> std::result::Result, tonic::Status> { + self.inner.ready().await.map_err(|e| { + tonic::Status::unknown(format!("Service was not ready: {}", e.into())) + })?; + let codec = tonic_prost::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static("/remote.ETHBACKEND/NetPeerCount"); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("remote.ETHBACKEND", "NetPeerCount")); @@ -888,24 +814,16 @@ pub mod ethbackend_client { pub async fn version( &mut self, request: impl tonic::IntoRequest<()>, - ) -> std::result::Result< - tonic::Response, - tonic::Status, - > { - self.inner - .ready() - .await - .map_err(|e| { - tonic::Status::unknown( - format!("Service was not ready: {}", e.into()), - ) - })?; - let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static( - "/remote.ETHBACKEND/Version", - ); + ) -> std::result::Result, tonic::Status> + { + self.inner.ready().await.map_err(|e| { + tonic::Status::unknown(format!("Service was not ready: {}", e.into())) + })?; + let codec = tonic_prost::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static("/remote.ETHBACKEND/Version"); let mut req = request.into_request(); - req.extensions_mut().insert(GrpcMethod::new("remote.ETHBACKEND", "Version")); + req.extensions_mut() + .insert(GrpcMethod::new("remote.ETHBACKEND", "Version")); self.inner.unary(req, path, codec).await } /// Syncing returns a data object detailing the status of the sync process @@ -913,42 +831,27 @@ pub mod ethbackend_client { &mut self, request: impl tonic::IntoRequest<()>, ) -> std::result::Result, tonic::Status> { - self.inner - .ready() - .await - .map_err(|e| { - tonic::Status::unknown( - format!("Service was not ready: {}", e.into()), - ) - })?; - let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static( - "/remote.ETHBACKEND/Syncing", - ); + self.inner.ready().await.map_err(|e| { + tonic::Status::unknown(format!("Service was not ready: {}", e.into())) + })?; + let codec = tonic_prost::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static("/remote.ETHBACKEND/Syncing"); let mut req = request.into_request(); - req.extensions_mut().insert(GrpcMethod::new("remote.ETHBACKEND", "Syncing")); + req.extensions_mut() + .insert(GrpcMethod::new("remote.ETHBACKEND", "Syncing")); self.inner.unary(req, path, codec).await } /// ProtocolVersion returns the Ethereum protocol version number (e.g. 66 for ETH66). pub async fn protocol_version( &mut self, request: impl tonic::IntoRequest, - ) -> std::result::Result< - tonic::Response, - tonic::Status, - > { - self.inner - .ready() - .await - .map_err(|e| { - tonic::Status::unknown( - format!("Service was not ready: {}", e.into()), - ) - })?; - let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static( - "/remote.ETHBACKEND/ProtocolVersion", - ); + ) -> std::result::Result, tonic::Status> + { + self.inner.ready().await.map_err(|e| { + tonic::Status::unknown(format!("Service was not ready: {}", e.into())) + })?; + let codec = tonic_prost::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static("/remote.ETHBACKEND/ProtocolVersion"); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("remote.ETHBACKEND", "ProtocolVersion")); @@ -958,22 +861,13 @@ pub mod ethbackend_client { pub async fn client_version( &mut self, request: impl tonic::IntoRequest, - ) -> std::result::Result< - tonic::Response, - tonic::Status, - > { - self.inner - .ready() - .await - .map_err(|e| { - tonic::Status::unknown( - format!("Service was not ready: {}", e.into()), - ) - })?; - let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static( - "/remote.ETHBACKEND/ClientVersion", - ); + ) -> std::result::Result, tonic::Status> + { + self.inner.ready().await.map_err(|e| { + tonic::Status::unknown(format!("Service was not ready: {}", e.into())) + })?; + let codec = tonic_prost::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static("/remote.ETHBACKEND/ClientVersion"); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("remote.ETHBACKEND", "ClientVersion")); @@ -986,18 +880,11 @@ pub mod ethbackend_client { tonic::Response>, tonic::Status, > { - self.inner - .ready() - .await - .map_err(|e| { - tonic::Status::unknown( - format!("Service was not ready: {}", e.into()), - ) - })?; - let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static( - "/remote.ETHBACKEND/Subscribe", - ); + self.inner.ready().await.map_err(|e| { + tonic::Status::unknown(format!("Service was not ready: {}", e.into())) + })?; + let codec = tonic_prost::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static("/remote.ETHBACKEND/Subscribe"); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("remote.ETHBACKEND", "Subscribe")); @@ -1011,18 +898,11 @@ pub mod ethbackend_client { tonic::Response>, tonic::Status, > { - self.inner - .ready() - .await - .map_err(|e| { - tonic::Status::unknown( - format!("Service was not ready: {}", e.into()), - ) - })?; - let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static( - "/remote.ETHBACKEND/SubscribeLogs", - ); + self.inner.ready().await.map_err(|e| { + tonic::Status::unknown(format!("Service was not ready: {}", e.into())) + })?; + let codec = tonic_prost::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static("/remote.ETHBACKEND/SubscribeLogs"); let mut req = request.into_streaming_request(); req.extensions_mut() .insert(GrpcMethod::new("remote.ETHBACKEND", "SubscribeLogs")); @@ -1035,65 +915,46 @@ pub mod ethbackend_client { &mut self, request: impl tonic::IntoRequest, ) -> std::result::Result, tonic::Status> { - self.inner - .ready() - .await - .map_err(|e| { - tonic::Status::unknown( - format!("Service was not ready: {}", e.into()), - ) - })?; - let codec = tonic::codec::ProstCodec::default(); + self.inner.ready().await.map_err(|e| { + tonic::Status::unknown(format!("Service was not ready: {}", e.into())) + })?; + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static("/remote.ETHBACKEND/Block"); let mut req = request.into_request(); - req.extensions_mut().insert(GrpcMethod::new("remote.ETHBACKEND", "Block")); + req.extensions_mut() + .insert(GrpcMethod::new("remote.ETHBACKEND", "Block")); self.inner.unary(req, path, codec).await } /// High-level method - can read block body (only storage metadata) from db, snapshots or apply any other logic pub async fn canonical_body_for_storage( &mut self, request: impl tonic::IntoRequest, - ) -> std::result::Result< - tonic::Response, - tonic::Status, - > { - self.inner - .ready() - .await - .map_err(|e| { - tonic::Status::unknown( - format!("Service was not ready: {}", e.into()), - ) - })?; - let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static( - "/remote.ETHBACKEND/CanonicalBodyForStorage", - ); + ) -> std::result::Result, tonic::Status> + { + self.inner.ready().await.map_err(|e| { + tonic::Status::unknown(format!("Service was not ready: {}", e.into())) + })?; + let codec = tonic_prost::ProstCodec::default(); + let path = + http::uri::PathAndQuery::from_static("/remote.ETHBACKEND/CanonicalBodyForStorage"); let mut req = request.into_request(); - req.extensions_mut() - .insert(GrpcMethod::new("remote.ETHBACKEND", "CanonicalBodyForStorage")); + req.extensions_mut().insert(GrpcMethod::new( + "remote.ETHBACKEND", + "CanonicalBodyForStorage", + )); self.inner.unary(req, path, codec).await } /// High-level method - can find block hash by block number pub async fn canonical_hash( &mut self, request: impl tonic::IntoRequest, - ) -> std::result::Result< - tonic::Response, - tonic::Status, - > { - self.inner - .ready() - .await - .map_err(|e| { - tonic::Status::unknown( - format!("Service was not ready: {}", e.into()), - ) - })?; - let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static( - "/remote.ETHBACKEND/CanonicalHash", - ); + ) -> std::result::Result, tonic::Status> + { + self.inner.ready().await.map_err(|e| { + tonic::Status::unknown(format!("Service was not ready: {}", e.into())) + })?; + let codec = tonic_prost::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static("/remote.ETHBACKEND/CanonicalHash"); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("remote.ETHBACKEND", "CanonicalHash")); @@ -1103,22 +964,12 @@ pub mod ethbackend_client { pub async fn header_number( &mut self, request: impl tonic::IntoRequest, - ) -> std::result::Result< - tonic::Response, - tonic::Status, - > { - self.inner - .ready() - .await - .map_err(|e| { - tonic::Status::unknown( - format!("Service was not ready: {}", e.into()), - ) - })?; - let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static( - "/remote.ETHBACKEND/HeaderNumber", - ); + ) -> std::result::Result, tonic::Status> { + self.inner.ready().await.map_err(|e| { + tonic::Status::unknown(format!("Service was not ready: {}", e.into())) + })?; + let codec = tonic_prost::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static("/remote.ETHBACKEND/HeaderNumber"); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("remote.ETHBACKEND", "HeaderNumber")); @@ -1130,18 +981,11 @@ pub mod ethbackend_client { &mut self, request: impl tonic::IntoRequest, ) -> std::result::Result, tonic::Status> { - self.inner - .ready() - .await - .map_err(|e| { - tonic::Status::unknown( - format!("Service was not ready: {}", e.into()), - ) - })?; - let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static( - "/remote.ETHBACKEND/TxnLookup", - ); + self.inner.ready().await.map_err(|e| { + tonic::Status::unknown(format!("Service was not ready: {}", e.into())) + })?; + let codec = tonic_prost::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static("/remote.ETHBACKEND/TxnLookup"); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("remote.ETHBACKEND", "TxnLookup")); @@ -1152,18 +996,11 @@ pub mod ethbackend_client { &mut self, request: impl tonic::IntoRequest, ) -> std::result::Result, tonic::Status> { - self.inner - .ready() - .await - .map_err(|e| { - tonic::Status::unknown( - format!("Service was not ready: {}", e.into()), - ) - })?; - let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static( - "/remote.ETHBACKEND/NodeInfo", - ); + self.inner.ready().await.map_err(|e| { + tonic::Status::unknown(format!("Service was not ready: {}", e.into())) + })?; + let codec = tonic_prost::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static("/remote.ETHBACKEND/NodeInfo"); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("remote.ETHBACKEND", "NodeInfo")); @@ -1174,59 +1011,39 @@ pub mod ethbackend_client { &mut self, request: impl tonic::IntoRequest<()>, ) -> std::result::Result, tonic::Status> { - self.inner - .ready() - .await - .map_err(|e| { - tonic::Status::unknown( - format!("Service was not ready: {}", e.into()), - ) - })?; - let codec = tonic::codec::ProstCodec::default(); + self.inner.ready().await.map_err(|e| { + tonic::Status::unknown(format!("Service was not ready: {}", e.into())) + })?; + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static("/remote.ETHBACKEND/Peers"); let mut req = request.into_request(); - req.extensions_mut().insert(GrpcMethod::new("remote.ETHBACKEND", "Peers")); + req.extensions_mut() + .insert(GrpcMethod::new("remote.ETHBACKEND", "Peers")); self.inner.unary(req, path, codec).await } pub async fn add_peer( &mut self, request: impl tonic::IntoRequest, ) -> std::result::Result, tonic::Status> { - self.inner - .ready() - .await - .map_err(|e| { - tonic::Status::unknown( - format!("Service was not ready: {}", e.into()), - ) - })?; - let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static( - "/remote.ETHBACKEND/AddPeer", - ); + self.inner.ready().await.map_err(|e| { + tonic::Status::unknown(format!("Service was not ready: {}", e.into())) + })?; + let codec = tonic_prost::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static("/remote.ETHBACKEND/AddPeer"); let mut req = request.into_request(); - req.extensions_mut().insert(GrpcMethod::new("remote.ETHBACKEND", "AddPeer")); + req.extensions_mut() + .insert(GrpcMethod::new("remote.ETHBACKEND", "AddPeer")); self.inner.unary(req, path, codec).await } pub async fn remove_peer( &mut self, request: impl tonic::IntoRequest, - ) -> std::result::Result< - tonic::Response, - tonic::Status, - > { - self.inner - .ready() - .await - .map_err(|e| { - tonic::Status::unknown( - format!("Service was not ready: {}", e.into()), - ) - })?; - let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static( - "/remote.ETHBACKEND/RemovePeer", - ); + ) -> std::result::Result, tonic::Status> { + self.inner.ready().await.map_err(|e| { + tonic::Status::unknown(format!("Service was not ready: {}", e.into())) + })?; + let codec = tonic_prost::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static("/remote.ETHBACKEND/RemovePeer"); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("remote.ETHBACKEND", "RemovePeer")); @@ -1236,22 +1053,12 @@ pub mod ethbackend_client { pub async fn pending_block( &mut self, request: impl tonic::IntoRequest<()>, - ) -> std::result::Result< - tonic::Response, - tonic::Status, - > { - self.inner - .ready() - .await - .map_err(|e| { - tonic::Status::unknown( - format!("Service was not ready: {}", e.into()), - ) - })?; - let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static( - "/remote.ETHBACKEND/PendingBlock", - ); + ) -> std::result::Result, tonic::Status> { + self.inner.ready().await.map_err(|e| { + tonic::Status::unknown(format!("Service was not ready: {}", e.into())) + })?; + let codec = tonic_prost::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static("/remote.ETHBACKEND/PendingBlock"); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("remote.ETHBACKEND", "PendingBlock")); @@ -1260,22 +1067,12 @@ pub mod ethbackend_client { pub async fn bor_txn_lookup( &mut self, request: impl tonic::IntoRequest, - ) -> std::result::Result< - tonic::Response, - tonic::Status, - > { - self.inner - .ready() - .await - .map_err(|e| { - tonic::Status::unknown( - format!("Service was not ready: {}", e.into()), - ) - })?; - let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static( - "/remote.ETHBACKEND/BorTxnLookup", - ); + ) -> std::result::Result, tonic::Status> { + self.inner.ready().await.map_err(|e| { + tonic::Status::unknown(format!("Service was not ready: {}", e.into())) + })?; + let codec = tonic_prost::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static("/remote.ETHBACKEND/BorTxnLookup"); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("remote.ETHBACKEND", "BorTxnLookup")); @@ -1285,18 +1082,11 @@ pub mod ethbackend_client { &mut self, request: impl tonic::IntoRequest, ) -> std::result::Result, tonic::Status> { - self.inner - .ready() - .await - .map_err(|e| { - tonic::Status::unknown( - format!("Service was not ready: {}", e.into()), - ) - })?; - let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static( - "/remote.ETHBACKEND/BorEvents", - ); + self.inner.ready().await.map_err(|e| { + tonic::Status::unknown(format!("Service was not ready: {}", e.into())) + })?; + let codec = tonic_prost::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static("/remote.ETHBACKEND/BorEvents"); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("remote.ETHBACKEND", "BorEvents")); @@ -1305,22 +1095,12 @@ pub mod ethbackend_client { pub async fn aa_validation( &mut self, request: impl tonic::IntoRequest, - ) -> std::result::Result< - tonic::Response, - tonic::Status, - > { - self.inner - .ready() - .await - .map_err(|e| { - tonic::Status::unknown( - format!("Service was not ready: {}", e.into()), - ) - })?; - let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static( - "/remote.ETHBACKEND/AAValidation", - ); + ) -> std::result::Result, tonic::Status> { + self.inner.ready().await.map_err(|e| { + tonic::Status::unknown(format!("Service was not ready: {}", e.into())) + })?; + let codec = tonic_prost::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static("/remote.ETHBACKEND/AAValidation"); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("remote.ETHBACKEND", "AAValidation")); @@ -1329,22 +1109,13 @@ pub mod ethbackend_client { pub async fn block_for_tx_num( &mut self, request: impl tonic::IntoRequest, - ) -> std::result::Result< - tonic::Response, - tonic::Status, - > { - self.inner - .ready() - .await - .map_err(|e| { - tonic::Status::unknown( - format!("Service was not ready: {}", e.into()), - ) - })?; - let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static( - "/remote.ETHBACKEND/BlockForTxNum", - ); + ) -> std::result::Result, tonic::Status> + { + self.inner.ready().await.map_err(|e| { + tonic::Status::unknown(format!("Service was not ready: {}", e.into())) + })?; + let codec = tonic_prost::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static("/remote.ETHBACKEND/BlockForTxNum"); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("remote.ETHBACKEND", "BlockForTxNum")); @@ -1353,32 +1124,26 @@ pub mod ethbackend_client { pub async fn minimum_block_available( &mut self, request: impl tonic::IntoRequest<()>, - ) -> std::result::Result< - tonic::Response, - tonic::Status, - > { - self.inner - .ready() - .await - .map_err(|e| { - tonic::Status::unknown( - format!("Service was not ready: {}", e.into()), - ) - })?; - let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static( - "/remote.ETHBACKEND/MinimumBlockAvailable", - ); + ) -> std::result::Result, tonic::Status> + { + self.inner.ready().await.map_err(|e| { + tonic::Status::unknown(format!("Service was not ready: {}", e.into())) + })?; + let codec = tonic_prost::ProstCodec::default(); + let path = + http::uri::PathAndQuery::from_static("/remote.ETHBACKEND/MinimumBlockAvailable"); let mut req = request.into_request(); - req.extensions_mut() - .insert(GrpcMethod::new("remote.ETHBACKEND", "MinimumBlockAvailable")); + req.extensions_mut().insert(GrpcMethod::new( + "remote.ETHBACKEND", + "MinimumBlockAvailable", + )); self.inner.unary(req, path, codec).await } } } #[allow(dead_code)] #[allow(clippy::enum_variant_names)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct Cursor { #[prost(enumeration = "Op", tag = "1")] pub op: i32, @@ -1393,7 +1158,7 @@ pub struct Cursor { } #[allow(dead_code)] #[allow(clippy::enum_variant_names)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct Pair { #[prost(bytes = "vec", tag = "1")] pub k: ::prost::alloc::vec::Vec, @@ -1411,7 +1176,7 @@ pub struct Pair { } #[allow(dead_code)] #[allow(clippy::enum_variant_names)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct StorageChange { #[prost(message, optional, tag = "1")] pub location: ::core::option::Option, @@ -1480,7 +1245,7 @@ pub struct StateChange { } #[allow(dead_code)] #[allow(clippy::enum_variant_names)] -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct StateChangeRequest { #[prost(bool, tag = "1")] pub with_storage: bool, @@ -1489,11 +1254,11 @@ pub struct StateChangeRequest { } #[allow(dead_code)] #[allow(clippy::enum_variant_names)] -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct SnapshotsRequest {} #[allow(dead_code)] #[allow(clippy::enum_variant_names)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct SnapshotsReply { #[prost(string, repeated, tag = "1")] pub blocks_files: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, @@ -1502,7 +1267,7 @@ pub struct SnapshotsReply { } #[allow(dead_code)] #[allow(clippy::enum_variant_names)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct RangeReq { /// returned by .Tx() #[prost(uint64, tag = "1")] @@ -1516,12 +1281,12 @@ pub struct RangeReq { pub to_prefix: ::prost::alloc::vec::Vec, #[prost(bool, tag = "5")] pub order_ascend: bool, - /// <= 0 means no limit + /// \<= 0 means no limit #[prost(sint64, tag = "6")] pub limit: i64, /// pagination params /// - /// <= 0 means server will choose + /// \<= 0 means server will choose #[prost(int32, tag = "7")] pub page_size: i32, #[prost(string, tag = "8")] @@ -1530,7 +1295,7 @@ pub struct RangeReq { /// `kv.Sequence` method #[allow(dead_code)] #[allow(clippy::enum_variant_names)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct SequenceReq { /// returned by .Tx() #[prost(uint64, tag = "1")] @@ -1541,7 +1306,7 @@ pub struct SequenceReq { } #[allow(dead_code)] #[allow(clippy::enum_variant_names)] -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct SequenceReply { #[prost(uint64, tag = "1")] pub value: u64, @@ -1549,7 +1314,7 @@ pub struct SequenceReply { /// Temporal methods #[allow(dead_code)] #[allow(clippy::enum_variant_names)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct GetLatestReq { /// returned by .Tx() #[prost(uint64, tag = "1")] @@ -1569,7 +1334,7 @@ pub struct GetLatestReq { } #[allow(dead_code)] #[allow(clippy::enum_variant_names)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct GetLatestReply { #[prost(bytes = "vec", tag = "1")] pub v: ::prost::alloc::vec::Vec, @@ -1578,7 +1343,7 @@ pub struct GetLatestReply { } #[allow(dead_code)] #[allow(clippy::enum_variant_names)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct HistorySeekReq { /// returned by .Tx() #[prost(uint64, tag = "1")] @@ -1592,7 +1357,7 @@ pub struct HistorySeekReq { } #[allow(dead_code)] #[allow(clippy::enum_variant_names)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct HistorySeekReply { #[prost(bytes = "vec", tag = "1")] pub v: ::prost::alloc::vec::Vec, @@ -1601,7 +1366,7 @@ pub struct HistorySeekReply { } #[allow(dead_code)] #[allow(clippy::enum_variant_names)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct IndexRangeReq { /// returned by .Tx() #[prost(uint64, tag = "1")] @@ -1619,12 +1384,12 @@ pub struct IndexRangeReq { pub to_ts: i64, #[prost(bool, tag = "6")] pub order_ascend: bool, - /// <= 0 means no limit + /// \<= 0 means no limit #[prost(sint64, tag = "7")] pub limit: i64, /// pagination params /// - /// <= 0 means server will choose + /// \<= 0 means server will choose #[prost(int32, tag = "8")] pub page_size: i32, #[prost(string, tag = "9")] @@ -1632,7 +1397,7 @@ pub struct IndexRangeReq { } #[allow(dead_code)] #[allow(clippy::enum_variant_names)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct IndexRangeReply { /// TODO: it can be a bitmap #[prost(uint64, repeated, tag = "1")] @@ -1642,7 +1407,7 @@ pub struct IndexRangeReply { } #[allow(dead_code)] #[allow(clippy::enum_variant_names)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct HistoryRangeReq { /// returned by .Tx() #[prost(uint64, tag = "1")] @@ -1658,12 +1423,12 @@ pub struct HistoryRangeReq { pub to_ts: i64, #[prost(bool, tag = "6")] pub order_ascend: bool, - /// <= 0 means no limit + /// \<= 0 means no limit #[prost(sint64, tag = "7")] pub limit: i64, /// pagination params /// - /// <= 0 means server will choose + /// \<= 0 means server will choose #[prost(int32, tag = "8")] pub page_size: i32, #[prost(string, tag = "9")] @@ -1671,7 +1436,7 @@ pub struct HistoryRangeReq { } #[allow(dead_code)] #[allow(clippy::enum_variant_names)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct RangeAsOfReq { /// returned by .Tx() #[prost(uint64, tag = "1")] @@ -1692,12 +1457,12 @@ pub struct RangeAsOfReq { pub latest: bool, #[prost(bool, tag = "7")] pub order_ascend: bool, - /// <= 0 means no limit + /// \<= 0 means no limit #[prost(sint64, tag = "8")] pub limit: i64, /// pagination params /// - /// <= 0 means server will choose + /// \<= 0 means server will choose #[prost(int32, tag = "9")] pub page_size: i32, #[prost(string, tag = "10")] @@ -1705,20 +1470,20 @@ pub struct RangeAsOfReq { } #[allow(dead_code)] #[allow(clippy::enum_variant_names)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct Pairs { /// TODO: replace by lengtsh+arena? Anyway on server we need copy (serialization happening outside tx) #[prost(bytes = "vec", repeated, tag = "1")] pub keys: ::prost::alloc::vec::Vec<::prost::alloc::vec::Vec>, #[prost(bytes = "vec", repeated, tag = "2")] pub values: ::prost::alloc::vec::Vec<::prost::alloc::vec::Vec>, - /// uint32 estimateTotal = 3; // send once after stream creation + /// uint32 estimateTotal = 3; // send once after stream creation #[prost(string, tag = "3")] pub next_page_token: ::prost::alloc::string::String, } #[allow(dead_code)] #[allow(clippy::enum_variant_names)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct PairsPagination { #[prost(bytes = "vec", tag = "1")] pub next_key: ::prost::alloc::vec::Vec, @@ -1727,7 +1492,7 @@ pub struct PairsPagination { } #[allow(dead_code)] #[allow(clippy::enum_variant_names)] -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct IndexPagination { #[prost(sint64, tag = "1")] pub next_time_stamp: i64, @@ -1736,7 +1501,7 @@ pub struct IndexPagination { } #[allow(dead_code)] #[allow(clippy::enum_variant_names)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct HasPrefixReq { #[prost(uint64, tag = "1")] pub tx_id: u64, @@ -1747,7 +1512,7 @@ pub struct HasPrefixReq { } #[allow(dead_code)] #[allow(clippy::enum_variant_names)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct HasPrefixReply { #[prost(bytes = "vec", tag = "1")] pub first_key: ::prost::alloc::vec::Vec, @@ -1758,7 +1523,7 @@ pub struct HasPrefixReply { } #[allow(dead_code)] #[allow(clippy::enum_variant_names)] -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct HistoryStartFromReq { #[prost(uint32, tag = "1")] pub domain: u32, @@ -1768,14 +1533,14 @@ pub struct HistoryStartFromReq { } #[allow(dead_code)] #[allow(clippy::enum_variant_names)] -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct HistoryStartFromReply { #[prost(uint64, tag = "1")] pub start_from: u64, } #[allow(dead_code)] #[allow(clippy::enum_variant_names)] -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct CurrentDomainVersionReq { #[prost(uint64, tag = "1")] pub tx_id: u64, @@ -1784,7 +1549,7 @@ pub struct CurrentDomainVersionReq { } #[allow(dead_code)] #[allow(clippy::enum_variant_names)] -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct CurrentDomainVersionReply { #[prost(uint64, tag = "1")] pub major: u64, @@ -1793,14 +1558,14 @@ pub struct CurrentDomainVersionReply { } #[allow(dead_code)] #[allow(clippy::enum_variant_names)] -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct StepSizeReq { #[prost(uint64, tag = "1")] pub tx_id: u64, } #[allow(dead_code)] #[allow(clippy::enum_variant_names)] -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct StepSizeReply { #[prost(uint64, tag = "1")] pub step: u64, @@ -1958,10 +1723,10 @@ pub mod kv_client { dead_code, missing_docs, clippy::wildcard_imports, - clippy::let_unit_value, + clippy::let_unit_value )] - use tonic::codegen::*; use tonic::codegen::http::Uri; + use tonic::codegen::*; /// Provides methods to access key-value data #[derive(Debug, Clone)] pub struct KvClient { @@ -1993,10 +1758,7 @@ pub mod kv_client { let inner = tonic::client::Grpc::with_origin(inner, origin); Self { inner } } - pub fn with_interceptor( - inner: T, - interceptor: F, - ) -> KvClient> + pub fn with_interceptor(inner: T, interceptor: F) -> KvClient> where F: tonic::service::Interceptor, T::ResponseBody: Default, @@ -2006,9 +1768,8 @@ pub mod kv_client { >::ResponseBody, >, >, - , - >>::Error: Into + std::marker::Send + std::marker::Sync, + >>::Error: + Into + std::marker::Send + std::marker::Sync, { KvClient::new(InterceptedService::new(inner, interceptor)) } @@ -2047,22 +1808,16 @@ pub mod kv_client { pub async fn version( &mut self, request: impl tonic::IntoRequest<()>, - ) -> std::result::Result< - tonic::Response, - tonic::Status, - > { - self.inner - .ready() - .await - .map_err(|e| { - tonic::Status::unknown( - format!("Service was not ready: {}", e.into()), - ) - })?; - let codec = tonic::codec::ProstCodec::default(); + ) -> std::result::Result, tonic::Status> + { + self.inner.ready().await.map_err(|e| { + tonic::Status::unknown(format!("Service was not ready: {}", e.into())) + })?; + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static("/remote.KV/Version"); let mut req = request.into_request(); - req.extensions_mut().insert(GrpcMethod::new("remote.KV", "Version")); + req.extensions_mut() + .insert(GrpcMethod::new("remote.KV", "Version")); self.inner.unary(req, path, codec).await } /// Tx exposes read-only transactions for the key-value store @@ -2073,22 +1828,16 @@ pub mod kv_client { pub async fn tx( &mut self, request: impl tonic::IntoStreamingRequest, - ) -> std::result::Result< - tonic::Response>, - tonic::Status, - > { - self.inner - .ready() - .await - .map_err(|e| { - tonic::Status::unknown( - format!("Service was not ready: {}", e.into()), - ) - })?; - let codec = tonic::codec::ProstCodec::default(); + ) -> std::result::Result>, tonic::Status> + { + self.inner.ready().await.map_err(|e| { + tonic::Status::unknown(format!("Service was not ready: {}", e.into())) + })?; + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static("/remote.KV/Tx"); let mut req = request.into_streaming_request(); - req.extensions_mut().insert(GrpcMethod::new("remote.KV", "Tx")); + req.extensions_mut() + .insert(GrpcMethod::new("remote.KV", "Tx")); self.inner.streaming(req, path, codec).await } pub async fn state_changes( @@ -2098,18 +1847,14 @@ pub mod kv_client { tonic::Response>, tonic::Status, > { - self.inner - .ready() - .await - .map_err(|e| { - tonic::Status::unknown( - format!("Service was not ready: {}", e.into()), - ) - })?; - let codec = tonic::codec::ProstCodec::default(); + self.inner.ready().await.map_err(|e| { + tonic::Status::unknown(format!("Service was not ready: {}", e.into())) + })?; + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static("/remote.KV/StateChanges"); let mut req = request.into_request(); - req.extensions_mut().insert(GrpcMethod::new("remote.KV", "StateChanges")); + req.extensions_mut() + .insert(GrpcMethod::new("remote.KV", "StateChanges")); self.inner.server_streaming(req, path, codec).await } /// Snapshots returns list of current snapshot files. Then client can just open all of them. @@ -2117,58 +1862,46 @@ pub mod kv_client { &mut self, request: impl tonic::IntoRequest, ) -> std::result::Result, tonic::Status> { - self.inner - .ready() - .await - .map_err(|e| { - tonic::Status::unknown( - format!("Service was not ready: {}", e.into()), - ) - })?; - let codec = tonic::codec::ProstCodec::default(); + self.inner.ready().await.map_err(|e| { + tonic::Status::unknown(format!("Service was not ready: {}", e.into())) + })?; + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static("/remote.KV/Snapshots"); let mut req = request.into_request(); - req.extensions_mut().insert(GrpcMethod::new("remote.KV", "Snapshots")); + req.extensions_mut() + .insert(GrpcMethod::new("remote.KV", "Snapshots")); self.inner.unary(req, path, codec).await } - /// Range [from, to) - /// Range(from, nil) means [from, EndOfTable) - /// Range(nil, to) means [StartOfTable, to) - /// If orderAscend=false server expecting `from`<`to`. Example: Range("B", "A") + /// Range \[from, to) + /// Range(from, nil) means \[from, EndOfTable) + /// Range(nil, to) means \[StartOfTable, to) + /// If orderAscend=false server expecting `from`\<`to`. Example: Range("B", "A") pub async fn range( &mut self, request: impl tonic::IntoRequest, ) -> std::result::Result, tonic::Status> { - self.inner - .ready() - .await - .map_err(|e| { - tonic::Status::unknown( - format!("Service was not ready: {}", e.into()), - ) - })?; - let codec = tonic::codec::ProstCodec::default(); + self.inner.ready().await.map_err(|e| { + tonic::Status::unknown(format!("Service was not ready: {}", e.into())) + })?; + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static("/remote.KV/Range"); let mut req = request.into_request(); - req.extensions_mut().insert(GrpcMethod::new("remote.KV", "Range")); + req.extensions_mut() + .insert(GrpcMethod::new("remote.KV", "Range")); self.inner.unary(req, path, codec).await } pub async fn sequence( &mut self, request: impl tonic::IntoRequest, ) -> std::result::Result, tonic::Status> { - self.inner - .ready() - .await - .map_err(|e| { - tonic::Status::unknown( - format!("Service was not ready: {}", e.into()), - ) - })?; - let codec = tonic::codec::ProstCodec::default(); + self.inner.ready().await.map_err(|e| { + tonic::Status::unknown(format!("Service was not ready: {}", e.into())) + })?; + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static("/remote.KV/Sequence"); let mut req = request.into_request(); - req.extensions_mut().insert(GrpcMethod::new("remote.KV", "Sequence")); + req.extensions_mut() + .insert(GrpcMethod::new("remote.KV", "Sequence")); self.inner.unary(req, path, codec).await } /// Temporal methods @@ -2176,135 +1909,96 @@ pub mod kv_client { &mut self, request: impl tonic::IntoRequest, ) -> std::result::Result, tonic::Status> { - self.inner - .ready() - .await - .map_err(|e| { - tonic::Status::unknown( - format!("Service was not ready: {}", e.into()), - ) - })?; - let codec = tonic::codec::ProstCodec::default(); + self.inner.ready().await.map_err(|e| { + tonic::Status::unknown(format!("Service was not ready: {}", e.into())) + })?; + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static("/remote.KV/GetLatest"); let mut req = request.into_request(); - req.extensions_mut().insert(GrpcMethod::new("remote.KV", "GetLatest")); + req.extensions_mut() + .insert(GrpcMethod::new("remote.KV", "GetLatest")); self.inner.unary(req, path, codec).await } pub async fn history_seek( &mut self, request: impl tonic::IntoRequest, - ) -> std::result::Result< - tonic::Response, - tonic::Status, - > { - self.inner - .ready() - .await - .map_err(|e| { - tonic::Status::unknown( - format!("Service was not ready: {}", e.into()), - ) - })?; - let codec = tonic::codec::ProstCodec::default(); + ) -> std::result::Result, tonic::Status> { + self.inner.ready().await.map_err(|e| { + tonic::Status::unknown(format!("Service was not ready: {}", e.into())) + })?; + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static("/remote.KV/HistorySeek"); let mut req = request.into_request(); - req.extensions_mut().insert(GrpcMethod::new("remote.KV", "HistorySeek")); + req.extensions_mut() + .insert(GrpcMethod::new("remote.KV", "HistorySeek")); self.inner.unary(req, path, codec).await } pub async fn index_range( &mut self, request: impl tonic::IntoRequest, - ) -> std::result::Result< - tonic::Response, - tonic::Status, - > { - self.inner - .ready() - .await - .map_err(|e| { - tonic::Status::unknown( - format!("Service was not ready: {}", e.into()), - ) - })?; - let codec = tonic::codec::ProstCodec::default(); + ) -> std::result::Result, tonic::Status> { + self.inner.ready().await.map_err(|e| { + tonic::Status::unknown(format!("Service was not ready: {}", e.into())) + })?; + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static("/remote.KV/IndexRange"); let mut req = request.into_request(); - req.extensions_mut().insert(GrpcMethod::new("remote.KV", "IndexRange")); + req.extensions_mut() + .insert(GrpcMethod::new("remote.KV", "IndexRange")); self.inner.unary(req, path, codec).await } pub async fn history_range( &mut self, request: impl tonic::IntoRequest, ) -> std::result::Result, tonic::Status> { - self.inner - .ready() - .await - .map_err(|e| { - tonic::Status::unknown( - format!("Service was not ready: {}", e.into()), - ) - })?; - let codec = tonic::codec::ProstCodec::default(); + self.inner.ready().await.map_err(|e| { + tonic::Status::unknown(format!("Service was not ready: {}", e.into())) + })?; + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static("/remote.KV/HistoryRange"); let mut req = request.into_request(); - req.extensions_mut().insert(GrpcMethod::new("remote.KV", "HistoryRange")); + req.extensions_mut() + .insert(GrpcMethod::new("remote.KV", "HistoryRange")); self.inner.unary(req, path, codec).await } pub async fn range_as_of( &mut self, request: impl tonic::IntoRequest, ) -> std::result::Result, tonic::Status> { - self.inner - .ready() - .await - .map_err(|e| { - tonic::Status::unknown( - format!("Service was not ready: {}", e.into()), - ) - })?; - let codec = tonic::codec::ProstCodec::default(); + self.inner.ready().await.map_err(|e| { + tonic::Status::unknown(format!("Service was not ready: {}", e.into())) + })?; + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static("/remote.KV/RangeAsOf"); let mut req = request.into_request(); - req.extensions_mut().insert(GrpcMethod::new("remote.KV", "RangeAsOf")); + req.extensions_mut() + .insert(GrpcMethod::new("remote.KV", "RangeAsOf")); self.inner.unary(req, path, codec).await } pub async fn has_prefix( &mut self, request: impl tonic::IntoRequest, ) -> std::result::Result, tonic::Status> { - self.inner - .ready() - .await - .map_err(|e| { - tonic::Status::unknown( - format!("Service was not ready: {}", e.into()), - ) - })?; - let codec = tonic::codec::ProstCodec::default(); + self.inner.ready().await.map_err(|e| { + tonic::Status::unknown(format!("Service was not ready: {}", e.into())) + })?; + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static("/remote.KV/HasPrefix"); let mut req = request.into_request(); - req.extensions_mut().insert(GrpcMethod::new("remote.KV", "HasPrefix")); + req.extensions_mut() + .insert(GrpcMethod::new("remote.KV", "HasPrefix")); self.inner.unary(req, path, codec).await } pub async fn history_start_from( &mut self, request: impl tonic::IntoRequest, - ) -> std::result::Result< - tonic::Response, - tonic::Status, - > { - self.inner - .ready() - .await - .map_err(|e| { - tonic::Status::unknown( - format!("Service was not ready: {}", e.into()), - ) - })?; - let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static( - "/remote.KV/HistoryStartFrom", - ); + ) -> std::result::Result, tonic::Status> + { + self.inner.ready().await.map_err(|e| { + tonic::Status::unknown(format!("Service was not ready: {}", e.into())) + })?; + let codec = tonic_prost::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static("/remote.KV/HistoryStartFrom"); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("remote.KV", "HistoryStartFrom")); @@ -2313,22 +2007,13 @@ pub mod kv_client { pub async fn current_domain_version( &mut self, request: impl tonic::IntoRequest, - ) -> std::result::Result< - tonic::Response, - tonic::Status, - > { - self.inner - .ready() - .await - .map_err(|e| { - tonic::Status::unknown( - format!("Service was not ready: {}", e.into()), - ) - })?; - let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static( - "/remote.KV/CurrentDomainVersion", - ); + ) -> std::result::Result, tonic::Status> + { + self.inner.ready().await.map_err(|e| { + tonic::Status::unknown(format!("Service was not ready: {}", e.into())) + })?; + let codec = tonic_prost::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static("/remote.KV/CurrentDomainVersion"); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("remote.KV", "CurrentDomainVersion")); @@ -2338,18 +2023,14 @@ pub mod kv_client { &mut self, request: impl tonic::IntoRequest, ) -> std::result::Result, tonic::Status> { - self.inner - .ready() - .await - .map_err(|e| { - tonic::Status::unknown( - format!("Service was not ready: {}", e.into()), - ) - })?; - let codec = tonic::codec::ProstCodec::default(); + self.inner.ready().await.map_err(|e| { + tonic::Status::unknown(format!("Service was not ready: {}", e.into())) + })?; + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static("/remote.KV/StepSize"); let mut req = request.into_request(); - req.extensions_mut().insert(GrpcMethod::new("remote.KV", "StepSize")); + req.extensions_mut() + .insert(GrpcMethod::new("remote.KV", "StepSize")); self.inner.unary(req, path, codec).await } } diff --git a/crates/bridges/evm/erigon-bridge/src/generated/types.rs b/crates/bridges/evm/erigon-bridge/src/generated/types.rs index 51c2634..a77bbb3 100644 --- a/crates/bridges/evm/erigon-bridge/src/generated/types.rs +++ b/crates/bridges/evm/erigon-bridge/src/generated/types.rs @@ -1,7 +1,7 @@ // This file is @generated by prost-build. #[allow(dead_code)] #[allow(clippy::enum_variant_names)] -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct H128 { #[prost(uint64, tag = "1")] pub hi: u64, @@ -10,7 +10,7 @@ pub struct H128 { } #[allow(dead_code)] #[allow(clippy::enum_variant_names)] -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct H160 { #[prost(message, optional, tag = "1")] pub hi: ::core::option::Option, @@ -19,7 +19,7 @@ pub struct H160 { } #[allow(dead_code)] #[allow(clippy::enum_variant_names)] -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct H256 { #[prost(message, optional, tag = "1")] pub hi: ::core::option::Option, @@ -28,7 +28,7 @@ pub struct H256 { } #[allow(dead_code)] #[allow(clippy::enum_variant_names)] -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct H512 { #[prost(message, optional, tag = "1")] pub hi: ::core::option::Option, @@ -37,7 +37,7 @@ pub struct H512 { } #[allow(dead_code)] #[allow(clippy::enum_variant_names)] -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct H1024 { #[prost(message, optional, tag = "1")] pub hi: ::core::option::Option, @@ -46,7 +46,7 @@ pub struct H1024 { } #[allow(dead_code)] #[allow(clippy::enum_variant_names)] -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct H2048 { #[prost(message, optional, tag = "1")] pub hi: ::core::option::Option, @@ -56,7 +56,7 @@ pub struct H2048 { /// Reply message containing the current service version on the service side #[allow(dead_code)] #[allow(clippy::enum_variant_names)] -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct VersionReply { #[prost(uint32, tag = "1")] pub major: u32, @@ -65,7 +65,8 @@ pub struct VersionReply { #[prost(uint32, tag = "3")] pub patch: u32, } -/// ------------------------------------------------------------------------ +/// --- +/// /// Engine API types /// See #[allow(dead_code)] @@ -112,7 +113,7 @@ pub struct ExecutionPayload { } #[allow(dead_code)] #[allow(clippy::enum_variant_names)] -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct Withdrawal { #[prost(uint64, tag = "1")] pub index: u64, @@ -125,7 +126,7 @@ pub struct Withdrawal { } #[allow(dead_code)] #[allow(clippy::enum_variant_names)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct BlobsBundle { /// TODO(eip-4844): define a protobuf message for type KZGCommitment #[prost(bytes = "vec", repeated, tag = "1")] @@ -138,14 +139,14 @@ pub struct BlobsBundle { } #[allow(dead_code)] #[allow(clippy::enum_variant_names)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct RequestsBundle { #[prost(bytes = "vec", repeated, tag = "1")] pub requests: ::prost::alloc::vec::Vec<::prost::alloc::vec::Vec>, } #[allow(dead_code)] #[allow(clippy::enum_variant_names)] -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct NodeInfoPorts { #[prost(uint32, tag = "1")] pub discovery: u32, @@ -154,7 +155,7 @@ pub struct NodeInfoPorts { } #[allow(dead_code)] #[allow(clippy::enum_variant_names)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct NodeInfoReply { #[prost(string, tag = "1")] pub id: ::prost::alloc::string::String, @@ -173,7 +174,7 @@ pub struct NodeInfoReply { } #[allow(dead_code)] #[allow(clippy::enum_variant_names)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct PeerInfo { #[prost(string, tag = "1")] pub id: ::prost::alloc::string::String, @@ -248,7 +249,7 @@ pub struct AccountAbstractionTransaction { } #[allow(dead_code)] #[allow(clippy::enum_variant_names)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct Authorization { #[prost(uint64, tag = "1")] pub chain_id: u64, diff --git a/crates/bridges/evm/erigon-bridge/src/main.rs b/crates/bridges/evm/erigon-bridge/src/main.rs index 688f77c..f18cc67 100644 --- a/crates/bridges/evm/erigon-bridge/src/main.rs +++ b/crates/bridges/evm/erigon-bridge/src/main.rs @@ -23,7 +23,7 @@ use tracing_subscriber::EnvFilter; use validators_evm::ExecutorType; use bridge::ErigonFlightBridge; -use phaser_bridge::FlightBridgeServer; +use phaser_server::FlightBridgeServer; #[derive(Parser, Debug)] #[command(name = "erigon-bridge")] @@ -104,7 +104,7 @@ async fn main() -> Result<()> { EnvFilter::try_from_default_env() .unwrap_or_else(|_| EnvFilter::new("erigon_bridge=info")), ) - .with(tracing_subscriber::fmt::layer()) + .with(tracing_subscriber::fmt::layer().with_ansi(false)) .with(metrics::MetricsLayer::new("erigon-bridge")) .init(); @@ -226,9 +226,9 @@ async fn main() -> Result<()> { // Create the Flight server let flight_server = FlightBridgeServer::new(bridge); - // Configure global maximum message size (256MB) + // Configure global maximum message size (512MB) // Per-stream limits are negotiated via StreamPreferences - const MAX_MESSAGE_SIZE: usize = 256 * 1024 * 1024; + const MAX_MESSAGE_SIZE: usize = 512 * 1024 * 1024; // Configure compression based on CLI flag let mut flight_service = flight_server diff --git a/crates/bridges/evm/erigon-bridge/src/segment_worker.rs b/crates/bridges/evm/erigon-bridge/src/segment_worker.rs index 5adb7b2..11e106f 100644 --- a/crates/bridges/evm/erigon-bridge/src/segment_worker.rs +++ b/crates/bridges/evm/erigon-bridge/src/segment_worker.rs @@ -14,11 +14,12 @@ use alloy_primitives::Bytes; use alloy_rlp::Decodable; use arrow_array::RecordBatch; use futures::stream::StreamExt; -use phaser_bridge::{BatchWithRange, StreamError}; +use phaser_server::{BatchWithRange, StreamError}; +use prost::Message as ProstMessage; use std::collections::HashMap; use std::sync::Arc; use std::time::Instant; -use tracing::{debug, error, info, warn}; +use tracing::{debug, error, warn}; use validators_evm::ValidationExecutor; /// Configuration for segment-based processing and validation @@ -187,7 +188,7 @@ impl SegmentWorker { let blocks_phase_start = Instant::now(); let mut yielded_batches = 0u64; - info!( + debug!( "Worker {} segment {} processing transactions for blocks {} to {} ({} blocks)", worker_id, segment_id, @@ -297,8 +298,11 @@ impl SegmentWorker { match batch_result { Ok(tx_batch) => { batch_count += 1; - debug!("Worker {} segment {}: Received transaction batch {} with {} transactions", - worker_id, segment_id, batch_count, tx_batch.transactions.len()); + // Track gRPC message size for capacity planning + let msg_size = tx_batch.encoded_len(); + self.metrics.grpc_message_size("transactions", msg_size); + debug!("Worker {} segment {}: Received transaction batch {} with {} transactions ({} bytes)", + worker_id, segment_id, batch_count, tx_batch.transactions.len(), msg_size); let tx_count = tx_batch.transactions.len() as u64; all_transactions.extend(tx_batch.transactions); self.metrics @@ -316,7 +320,7 @@ impl SegmentWorker { } // Stream completed successfully - info!("Worker {} segment {}: Transaction stream completed for blocks {}-{}, received {} batches with {} total transactions", + debug!("Worker {} segment {}: Transaction stream completed for blocks {}-{}, received {} batches with {} total transactions", worker_id, segment_id, start_block, end_block, batch_count, all_transactions.len()); self.metrics.grpc_stream_dec("transactions"); @@ -395,7 +399,7 @@ impl SegmentWorker { // Yield this batch immediately - no buffering! // Wrap with responsibility range metadata yielded_batches += 1; - info!( + debug!( "BRIDGE YIELD: Worker {} segment {}: Yielding transaction batch {}, blocks {}-{}", worker_id, segment_id, yielded_batches, start_block, end_block ); @@ -431,7 +435,7 @@ impl SegmentWorker { return; } - info!( + debug!( "Worker {} segment {}: Completed transaction processing in {:.2}s, yielded {} RecordBatches", worker_id, segment_id, @@ -460,7 +464,7 @@ impl SegmentWorker { let mut yielded_batches = 0u64; let mut total_receipts_processed = 0u64; - info!( + debug!( "Worker {} segment {} processing logs for blocks {} to {} ({} blocks)", worker_id, segment_id, @@ -485,7 +489,7 @@ impl SegmentWorker { current_block = chunk_end + 1; } - info!( + debug!( "Worker {} segment {}: Created {} work units for parallel execution", worker_id, segment_id, work_units.len() ); @@ -644,7 +648,7 @@ impl SegmentWorker { // Record phase duration and metrics let duration = phase_start.elapsed().as_secs_f64(); - info!( + debug!( "Worker {} segment {}: Completed log processing, yielded {} RecordBatches, processed {} receipts in {:.2}s", worker_id, segment_id, @@ -669,7 +673,7 @@ impl SegmentWorker { batch_size: u32, metrics: &BridgeMetrics, ) -> Result, ErigonBridgeError> { - info!( + debug!( "ERIGON REQUEST: Requesting blocks {}-{} from Erigon (expecting {} blocks)", segment_start, segment_end, @@ -701,13 +705,17 @@ impl SegmentWorker { match batch_result { Ok(block_batch) => { batch_count += 1; + // Track gRPC message size for capacity planning + let msg_size = block_batch.encoded_len(); + metrics.grpc_message_size("blocks", msg_size); debug!( - "Blocks {}-{}: Received header batch {} with {} blocks", + "Blocks {}-{}: Received header batch {} with {} blocks ({} bytes)", segment_start, segment_end, batch_count, - block_batch.blocks.len() + block_batch.blocks.len(), + msg_size ); for block in block_batch.blocks { @@ -758,7 +766,7 @@ impl SegmentWorker { segment_start, segment_end, headers.len(), expected_count, min_received, max_received, missing_count ); } else { - info!( + debug!( "ERIGON RESPONSE: Blocks {}-{}: Received {} headers (complete), range {}-{}", segment_start, segment_end, @@ -924,6 +932,9 @@ impl SegmentWorker { match batch_result { Ok(receipt_batch) => { batch_count += 1; + // Track gRPC message size for capacity planning (logs come from receipts) + let msg_size = receipt_batch.encoded_len(); + metrics.grpc_message_size("logs", msg_size); for receipt in receipt_batch.receipts { let block_num = receipt.block_number; receipts_by_block diff --git a/crates/bridges/evm/jsonrpc-bridge/Cargo.toml b/crates/bridges/evm/jsonrpc-bridge/Cargo.toml index 8bdc455..df2f5ac 100644 --- a/crates/bridges/evm/jsonrpc-bridge/Cargo.toml +++ b/crates/bridges/evm/jsonrpc-bridge/Cargo.toml @@ -13,7 +13,8 @@ path = "src/main.rs" [dependencies] # Bridge framework -phaser-bridge = { workspace = true } +phaser-server = { workspace = true } +phaser-metrics = { workspace = true } evm-common = { workspace = true } validators-evm = { workspace = true } @@ -29,8 +30,8 @@ alloy-pubsub = { workspace = true } # Arrow and async arrow = { workspace = true } arrow-flight = { workspace = true } -arrow-array = "56.1" -arrow-schema = "56.1" +arrow-array = { workspace = true } +arrow-schema = { workspace = true } typed-arrow = { workspace = true } tokio = { workspace = true } async-trait = { workspace = true } @@ -51,4 +52,6 @@ hex = "0.4" tokio-stream = "0.1" tower = "0.5" hyper-util = "0.1" -num_cpus = { workspace = true } \ No newline at end of file +num_cpus = { workspace = true } +axum = { workspace = true } +prometheus = "0.13" \ No newline at end of file diff --git a/crates/bridges/evm/jsonrpc-bridge/src/bridge.rs b/crates/bridges/evm/jsonrpc-bridge/src/bridge.rs index 84fb0d6..ab32b43 100644 --- a/crates/bridges/evm/jsonrpc-bridge/src/bridge.rs +++ b/crates/bridges/evm/jsonrpc-bridge/src/bridge.rs @@ -1,5 +1,6 @@ use crate::client::JsonRpcClient; use crate::converter::JsonRpcConverter; +use crate::metrics::{BridgeMetrics, SegmentMetrics}; use crate::streaming::StreamingService; use arrow_flight::{ encode::FlightDataEncoderBuilder, flight_service_server::FlightService, Action, ActionType, @@ -8,24 +9,126 @@ use arrow_flight::{ }; use async_trait::async_trait; use futures::{stream, Stream, StreamExt}; -use phaser_bridge::{ - bridge::{BridgeCapabilities, FlightBridge}, - descriptors::{BridgeInfo, StreamType}, - subscription::QueryMode, +use phaser_server::{ + BridgeCapabilities, BridgeInfo, DiscoveryCapabilities, FlightBridge, GenericQuery, + GenericQueryMode, StreamType, TableDescriptor, }; use std::pin::Pin; use std::sync::Arc; +use std::time::Instant; use tonic::{Request, Response, Status, Streaming}; -use tracing::{error, info}; +use tracing::{debug, error, info}; use validators_evm::ValidationExecutor; +/// Configuration for segment-based parallel fetching +/// +/// ## Architecture: Segments and Batches +/// +/// Data fetching is organized into two levels: +/// - **Segments**: Large chunks of blocks (e.g., 10K-500K) processed as units +/// - **Batches**: Smaller groups within a segment (e.g., 50-100 blocks) fetched together +/// +/// ```text +/// Total Range: blocks 0 to 100,000 +/// ├── Segment 0: blocks 0-9,999 (processed in parallel) +/// │ ├── Batch 0: blocks 0-49 (fetched concurrently) +/// │ ├── Batch 1: blocks 50-99 +/// │ └── ... (200 batches per segment) +/// ├── Segment 1: blocks 10,000-19,999 (processed in parallel) +/// └── ... (10 segments total) +/// ``` +/// +/// ## Node-Specific Tuning +/// +/// - **Generic JSON-RPC**: Default settings work well (10K segments, 50 block batches) +/// - **Erigon via JSON-RPC**: Consider `segment_size: 500_000` to align with Erigon snapshots +/// - **Rate-limited nodes**: Reduce `max_concurrent_requests` to avoid 429 errors +/// +/// ## Logs vs Blocks/Transactions +/// +/// Logs use range-based `eth_getLogs` calls (one call per batch range), while +/// blocks/transactions still fetch per-block (with concurrent requests within batch). +#[derive(Clone, Debug)] +pub struct SegmentConfig { + /// Size of each segment in blocks + /// + /// Segments are the unit of parallel processing and progress tracking. + /// Default: 10,000 blocks per segment. + /// + /// Tuning: + /// - Larger segments = fewer segment boundaries, less overhead + /// - Smaller segments = more granular progress updates, better error isolation + /// - For Erigon, 500,000 matches the snapshot segment size + pub segment_size: u64, + + /// Maximum segments to process in parallel + /// + /// Controls top-level concurrency. Each segment streams independently. + /// Default: 4 concurrent segments. + /// + /// Tuning: + /// - More segments = higher throughput but more memory/connections + /// - For memory-constrained environments, reduce to 1-2 + pub max_concurrent_segments: usize, + + /// Blocks per batch within a segment + /// + /// Controls granularity of fetching within a segment. + /// Default: 50 blocks per batch. + /// + /// For blocks/transactions: This many blocks are fetched concurrently. + /// For logs: A single `eth_getLogs` call covers this many blocks. + /// + /// Tuning: + /// - Larger batches = fewer RPC round-trips + /// - Smaller batches = better progress granularity + pub blocks_per_batch: usize, + + /// Maximum concurrent RPC requests within a batch + /// + /// For blocks/transactions, this limits how many `eth_getBlockByNumber` + /// calls run simultaneously. For logs, this is ignored (single call per batch). + /// Default: 50 (same as blocks_per_batch for maximum parallelism). + /// + /// Tuning: + /// - Match to blocks_per_batch for full parallelism + /// - Reduce for rate-limited or overloaded nodes + pub max_concurrent_requests: usize, +} + +impl Default for SegmentConfig { + fn default() -> Self { + Self { + segment_size: 10_000, + max_concurrent_segments: 4, + blocks_per_batch: 50, + max_concurrent_requests: 50, + } + } +} + +/// Split a block range into segments +pub fn split_into_segments(start: u64, end: u64, segment_size: u64) -> Vec<(u64, u64)> { + let mut segments = Vec::new(); + let mut current = start; + + while current <= end { + let segment_end = (current + segment_size - 1).min(end); + segments.push((current, segment_end)); + current = segment_end + 1; + } + + segments +} + /// A bridge that connects to any JSON-RPC compatible node pub struct JsonRpcFlightBridge { client: Arc, chain_id: u64, streaming_service: Arc, - max_batch_size: usize, + segment_config: SegmentConfig, validator: Option>, + metrics: Option, } impl JsonRpcFlightBridge { @@ -69,15 +172,31 @@ impl JsonRpcFlightBridge { } }); + // Initialize metrics + let metrics = BridgeMetrics::new("jsonrpc_bridge", chain_id, "jsonrpc"); + Ok(Self { client, chain_id, streaming_service, - max_batch_size: 1000, // Default batch size, matches BridgeCapabilities + segment_config: SegmentConfig::default(), validator, + metrics: Some(metrics), }) } + /// Create a new JSON-RPC bridge with custom segment configuration + pub async fn with_segment_config( + node_url: String, + chain_id: Option, + validator_config: Option, + segment_config: SegmentConfig, + ) -> std::result::Result { + let mut bridge = Self::new(node_url, chain_id, validator_config).await?; + bridge.segment_config = segment_config; + Ok(bridge) + } + /// Get bridge information pub fn bridge_info(&self) -> BridgeInfo { let mut capabilities = vec![ @@ -104,15 +223,27 @@ impl JsonRpcFlightBridge { } } - /// Parse a FlightDescriptor to extract the BlockchainDescriptor - fn parse_descriptor( - descriptor: &FlightDescriptor, - ) -> std::result::Result> { - if let Some(first) = descriptor.path.first() { - serde_json::from_str::(first) - .map_err(|e| Box::new(Status::invalid_argument(format!("Invalid descriptor: {e}")))) - } else { - Err(Box::new(Status::invalid_argument("Empty descriptor path"))) + /// Parse a GenericQuery from a Flight Ticket + fn parse_ticket(ticket: &Ticket) -> Result { + GenericQuery::from_ticket(ticket) + .map_err(|e| Status::invalid_argument(format!("Invalid query in ticket: {e}"))) + } + + /// Parse a GenericQuery from a FlightDescriptor + fn parse_descriptor(descriptor: &FlightDescriptor) -> Result { + GenericQuery::from_flight_descriptor(descriptor) + .map_err(|e| Status::invalid_argument(format!("Invalid query in descriptor: {e}"))) + } + + /// Map table name to StreamType + fn table_to_stream_type(table: &str) -> Result { + match table { + "blocks" => Ok(StreamType::Blocks), + "transactions" => Ok(StreamType::Transactions), + "logs" => Ok(StreamType::Logs), + other => Err(Status::invalid_argument(format!( + "Unknown table: {other}. Available: blocks, transactions, logs" + ))), } } @@ -131,167 +262,690 @@ impl JsonRpcFlightBridge { } /// Create a stream for historical data (specific block range) + /// Returns BatchWithRange for phaser-query progress tracking + /// + /// Uses segment-level parallelism: multiple segments are processed concurrently, + /// with parallel block fetching within each segment. fn create_historical_stream( &self, stream_type: StreamType, start_block: u64, end_block: u64, validate: bool, - ) -> impl Stream> + Send { + ) -> impl Stream> + Send { + // Split range into segments + let segments = + split_into_segments(start_block, end_block, self.segment_config.segment_size); + let max_concurrent_segments = self.segment_config.max_concurrent_segments; + + info!( + "Creating historical {:?} stream for blocks {} to {} ({} segments, {} concurrent)", + stream_type, + start_block, + end_block, + segments.len(), + max_concurrent_segments + ); + + // Clone values needed for the async closure let client = self.client.clone(); - let batch_size = self.max_batch_size as u64; + let config = self.segment_config.clone(); let validator = self.validator.clone(); + let metrics = self.metrics.clone(); + + // Process segments in parallel, yielding results in order + futures::stream::iter(segments) + .map(move |(seg_start, seg_end)| { + let client = client.clone(); + let config = config.clone(); + let validator = validator.clone(); + let metrics = metrics.clone(); + + async move { + // Process this segment and return a stream of batches + Self::process_segment( + client, + seg_start, + seg_end, + stream_type, + validate, + validator, + metrics, + config, + ) + } + }) + .buffered(max_concurrent_segments) + .flatten() + } + + /// Process a single segment: fetch blocks in parallel batches and yield results + /// + /// For logs, uses optimized range-based fetching (single eth_getLogs call per batch) + /// instead of per-block fetching, significantly reducing RPC round-trips. + #[allow(clippy::too_many_arguments)] + fn process_segment( + client: Arc, + seg_start: u64, + seg_end: u64, + stream_type: StreamType, + validate: bool, + validator: Option>, + metrics: Option, + config: SegmentConfig, + ) -> Pin> + Send>> { + // For logs, use optimized range-based fetching + if stream_type == StreamType::Logs { + return Box::pin(Self::process_segment_logs( + client, seg_start, seg_end, metrics, config, + )); + } + + // For blocks/transactions, use parallel per-block fetching + Box::pin(Self::process_segment_blocks_txs( + client, + seg_start, + seg_end, + stream_type, + validate, + validator, + metrics, + config, + )) + } + + /// Process segment for logs using range-based eth_getLogs calls + /// + /// This is much more efficient than per-block fetching because: + /// 1. Single RPC call per batch instead of one per block + /// 2. Logs are returned already sorted by block + /// 3. No need to fetch block headers for context (log includes block_num, block_hash) + fn process_segment_logs( + client: Arc, + seg_start: u64, + seg_end: u64, + metrics: Option, + config: SegmentConfig, + ) -> impl Stream> + Send { + let batch_size = config.blocks_per_batch as u64; + let segment_num = seg_start / config.segment_size; async_stream::stream! { - use alloy::eips::BlockNumberOrTag; use alloy_rpc_types_eth::Filter; - use arrow::compute::concat_batches; - use tracing::debug; - info!("Fetching historical {:?} from block {} to {} (batch size: {})", - stream_type, start_block, end_block, batch_size); + let total_blocks = seg_end - seg_start + 1; + info!( + "Segment {}: Processing logs for blocks {} to {} ({} blocks, batch size: {})", + segment_num, seg_start, seg_end, total_blocks, batch_size + ); - let mut current_block = start_block; + // Track active workers (mirrors erigon-bridge pattern) + if let Some(ref m) = metrics { + m.active_workers_inc("logs"); + } - while current_block <= end_block { - let batch_end = std::cmp::min(current_block + batch_size - 1, end_block); - let batch_count = (batch_end - current_block + 1) as usize; + let segment_start_time = Instant::now(); + let mut current_block = seg_start; + let mut total_logs_in_segment: u64 = 0; + let mut batches_fetched: u64 = 0; - debug!("Fetching batch: blocks {} to {} ({} blocks)", current_block, batch_end, batch_count); + while current_block <= seg_end { + let batch_end = std::cmp::min(current_block + batch_size - 1, seg_end); + let block_range_size = batch_end - current_block + 1; - // Collect RecordBatches for this batch - let mut record_batches = Vec::new(); + debug!( + "Segment {}: Fetching logs for block range {} to {} ({} blocks)", + segment_num, current_block, batch_end, block_range_size + ); - for block_num in current_block..=batch_end { - // Fetch block with transactions - let block = match client.get_block_with_txs(BlockNumberOrTag::Number(block_num)).await { - Ok(Some(block)) => block, - Ok(None) => { - error!("Block #{} not found", block_num); - continue; - } - Err(e) => { - error!("Failed to fetch block #{}: {}", block_num, e); - yield Err(Status::internal(format!("Failed to fetch block {block_num}: {e}"))); - continue; + // Track active RPC request (similar to grpc_stream_inc in erigon-bridge) + if let Some(ref m) = metrics { + m.grpc_stream_inc("logs"); + } + + // Single eth_getLogs call for the entire batch range + let filter = Filter::new() + .from_block(current_block) + .to_block(batch_end); + + let log_start = Instant::now(); + match client.get_logs(filter).await { + Ok(logs) => { + let fetch_duration_ms = log_start.elapsed().as_millis() as f64; + let log_count = logs.len(); + batches_fetched += 1; + + if let Some(ref m) = metrics { + m.grpc_stream_dec("logs"); + + // Record fetch duration with method indicating range-based fetching + m.grpc_request_duration_logs( + segment_num, + "eth_getLogs_range", + fetch_duration_ms, + ); + + // Track response size (estimate: ~200 bytes per log on average) + // This mirrors grpc_message_size in erigon-bridge + let estimated_size = log_count * 200; + m.grpc_message_size("logs", estimated_size); } - }; - match stream_type { - StreamType::Blocks => { - // Convert block header to RecordBatch - match evm_common::rpc_conversions::convert_any_header(&block.header) { + if !logs.is_empty() { + total_logs_in_segment += log_count as u64; + + // Use multi-block conversion - extracts block context from each log + match JsonRpcConverter::convert_logs_multi_block(&logs) { Ok(batch) => { - record_batches.push(batch); - }, - Err(e) => { - error!("Failed to convert block header #{}: {}", block_num, e); - yield Err(Status::internal(format!("Conversion error: {e}"))); - } - } - } - StreamType::Transactions => { - // Convert transactions (if any) - if !block.transactions.is_empty() { - // Validate transactions if requested and validator is available - if validate { - if let Some(ref val) = validator { - // Extract block record and transaction records for validation - // This validates our conversion: TransactionRecord → TxEnvelope → RLP → merkle root - let block_record = evm_common::rpc_conversions::convert_any_header_to_record(&block.header); - - match evm_common::rpc_conversions::extract_transaction_records(&block) { - Ok(tx_records) => { - // Validate the transactions against the block's transactions_root - // Using spawn_validate_records() for post-conversion validation - match val.spawn_validate_records(block_record, tx_records).await { - Ok(()) => { - debug!("Validated {} transactions for block #{}", block.transactions.len(), block_num); - }, - Err(e) => { - error!("Transaction validation failed for block #{}: {}", block_num, e); - yield Err(Status::internal(format!("Validation error for block {block_num}: {e}"))); - continue; - } - } - }, - Err(e) => { - error!("Failed to extract transaction records for validation: {}", e); - yield Err(Status::internal(format!("Failed to extract records for validation: {e}"))); - continue; - } - } - } else { - // Validation requested but no validator available - error!("Validation requested but validator not configured"); - yield Err(Status::failed_precondition("Validation requested but validator not configured")); - return; + if let Some(ref m) = metrics { + // Track logs processed (worker_id=0 for single-threaded) + m.items_processed_inc(0, segment_num, "logs", log_count as u64); + + // Update progress + let blocks_processed = batch_end - seg_start + 1; + let progress_pct = (blocks_processed as f64 / total_blocks as f64) * 100.0; + m.set_worker_progress(0, segment_num, "logs", progress_pct); } - } - // Convert to RecordBatch after validation - match JsonRpcConverter::convert_transactions(&block) { - Ok(batch) => { - record_batches.push(batch); - }, - Err(e) => { - error!("Failed to convert transactions for block #{}: {}", block_num, e); - yield Err(Status::internal(format!("Conversion error: {e}"))); - } - } - } - } - StreamType::Logs => { - // Fetch and convert logs - let filter = Filter::new().from_block(block_num).to_block(block_num); - - match client.get_logs(filter).await { - Ok(logs) if !logs.is_empty() => { - let block_hash = block.header.hash; - match JsonRpcConverter::convert_logs(&logs, block_num, block_hash, block.header.timestamp) { - Ok(batch) => record_batches.push(batch), - Err(e) => { - error!("Failed to convert logs for block #{}: {}", block_num, e); - yield Err(Status::internal(format!("Conversion error: {e}"))); - } - } - } - Ok(_) => { - // No logs in this block, skip + info!( + "Segment {}: Yielding {} logs for blocks {} to {} ({:.0}ms, {:.1} logs/block)", + segment_num, batch.num_rows(), current_block, batch_end, + fetch_duration_ms, + log_count as f64 / block_range_size as f64 + ); + yield Ok(phaser_server::BatchWithRange::new(batch, current_block, batch_end)); } Err(e) => { - error!("Failed to fetch logs for block #{}: {}", block_num, e); - yield Err(Status::internal(format!("Failed to fetch logs: {e}"))); + error!("Segment {}: Failed to convert logs: {}", segment_num, e); + if let Some(ref m) = metrics { + m.error("conversion_error", "logs"); + m.active_workers_dec("logs"); + m.segment_attempt(false); + } + yield Err(Status::internal(format!("Failed to convert logs: {e}"))); + return; } } + } else { + // No logs in this range - still track it for progress + debug!( + "Segment {}: No logs in block range {} to {} ({:.0}ms)", + segment_num, current_block, batch_end, fetch_duration_ms + ); + // Don't yield anything for empty ranges - the range will still + // be considered processed based on the next batch's start } - StreamType::Trie => { - yield Err(Status::unimplemented("Trie streaming not supported via JSON-RPC")); - return; + } + Err(e) => { + if let Some(ref m) = metrics { + m.grpc_stream_dec("logs"); + } + + // Categorize error for better monitoring + let error_type = Self::categorize_rpc_error(&e); + error!( + "Segment {}: Failed to fetch logs for range {} to {}: {} (type: {})", + segment_num, current_block, batch_end, e, error_type + ); + if let Some(ref m) = metrics { + m.error(&error_type, "logs"); + m.active_workers_dec("logs"); + m.segment_attempt(false); + } + yield Err(Status::internal(format!( + "Failed to fetch logs for range {current_block}-{batch_end}: {e}" + ))); + return; + } + } + + current_block = batch_end + 1; + } + + // Record segment-level metrics (success path only - errors return early) + let duration = segment_start_time.elapsed(); + if let Some(ref m) = metrics { + m.segment_duration("logs", duration.as_secs_f64()); + m.active_workers_dec("logs"); + m.segment_attempt(true); + } + + let logs_per_second = if duration.as_secs_f64() > 0.0 { + total_logs_in_segment as f64 / duration.as_secs_f64() + } else { + 0.0 + }; + + info!( + "Segment {}: Completed logs for blocks {} to {} in {:.2}s ({} logs in {} batches, {:.0} logs/sec)", + segment_num, seg_start, seg_end, duration.as_secs_f64(), + total_logs_in_segment, batches_fetched, logs_per_second + ); + } + } + + /// Categorize RPC error for metrics (mirrors erigon-bridge's categorize_error pattern) + fn categorize_rpc_error(error: &anyhow::Error) -> String { + let err_str = error.to_string(); + let err_lower = err_str.to_lowercase(); + + if err_lower.contains("timeout") || err_lower.contains("timed out") { + "timeout".to_string() + } else if err_lower.contains("connection") || err_lower.contains("connect") { + "connection".to_string() + } else if err_lower.contains("rate limit") || err_lower.contains("429") { + "rate_limit".to_string() + } else if err_lower.contains("not found") || err_lower.contains("404") { + "not_found".to_string() + } else if err_lower.contains("server error") || err_lower.contains("500") { + "server_error".to_string() + } else if err_lower.contains("invalid") || err_lower.contains("parse") { + "invalid_response".to_string() + } else { + // For unknown errors, include truncated message + let pattern = err_str + .split(':') + .next() + .unwrap_or(&err_str) + .trim() + .chars() + .take(50) + .collect::(); + format!("unknown:{pattern}") + } + } + + /// Process segment for blocks/transactions using parallel per-block fetching + /// + /// Fetches blocks concurrently using `max_concurrent_requests` to limit parallelism. + /// Each block requires a separate `eth_getBlockByNumber` RPC call. + #[allow(clippy::too_many_arguments)] + fn process_segment_blocks_txs( + client: Arc, + seg_start: u64, + seg_end: u64, + stream_type: StreamType, + validate: bool, + validator: Option>, + metrics: Option, + config: SegmentConfig, + ) -> impl Stream> + Send { + let batch_size = config.blocks_per_batch as u64; + let max_concurrent = config.max_concurrent_requests; + let segment_num = seg_start / config.segment_size; + + let data_type = match stream_type { + StreamType::Blocks => "blocks", + StreamType::Transactions => "transactions", + StreamType::Logs => "logs", + StreamType::Trie => "trie", + }; + + async_stream::stream! { + use arrow::compute::concat_batches; + use futures::stream::{self as fstream, StreamExt as FuturesStreamExt}; + + let total_blocks = seg_end - seg_start + 1; + info!( + "Segment {}: Processing {} for blocks {} to {} ({} blocks, batch size: {}, concurrency: {})", + segment_num, data_type, seg_start, seg_end, total_blocks, batch_size, max_concurrent + ); + + // Track active workers (mirrors erigon-bridge pattern) + if let Some(ref m) = metrics { + m.active_workers_inc(data_type); + } + + let segment_start_time = Instant::now(); + let mut current_block = seg_start; + let mut total_items: u64 = 0; + let mut batches_yielded: u64 = 0; + + while current_block <= seg_end { + let batch_end = std::cmp::min(current_block + batch_size - 1, seg_end); + let batch_count = (batch_end - current_block + 1) as usize; + + debug!( + "Segment {}: Fetching {} blocks {} to {} ({} blocks)", + segment_num, data_type, current_block, batch_end, batch_count + ); + + // Create block range iterator + let block_range: Vec = (current_block..=batch_end).collect(); + + // Track active RPC requests + if let Some(ref m) = metrics { + m.grpc_stream_inc(data_type); + } + + let batch_start = Instant::now(); + + // Fetch blocks in parallel using buffered stream + let client_clone = client.clone(); + let validator_clone = validator.clone(); + let stream_type_clone = stream_type; + let metrics_clone = metrics.clone(); + + let results: Vec, Status>> = fstream::iter(block_range) + .map(move |block_num| { + let client = client_clone.clone(); + let validator = validator_clone.clone(); + let metrics = metrics_clone.clone(); + async move { + Self::fetch_and_convert_block( + &client, + block_num, + stream_type_clone, + validate, + validator.as_ref(), + metrics.as_ref(), + segment_num, + ).await + } + }) + .buffered(max_concurrent) + .collect() + .await; + + let batch_duration_ms = batch_start.elapsed().as_millis() as f64; + + if let Some(ref m) = metrics { + m.grpc_stream_dec(data_type); + } + + // Process results - collect successful batches and report errors + let mut record_batches = Vec::with_capacity(batch_count); + let mut error_count = 0; + + for result in results { + match result { + Ok(Some(batch)) => { + total_items += batch.num_rows() as u64; + record_batches.push(batch); + } + Ok(None) => { + // Block had no data (e.g., empty transactions) - skip + } + Err(e) => { + // Log but continue - we want to process as many blocks as possible + let error_type = Self::categorize_rpc_error_from_status(&e); + error!("Segment {}: Error fetching block: {} (type: {})", segment_num, e, error_type); + if let Some(ref m) = metrics { + m.error(&error_type, data_type); + } + error_count += 1; } } } - // If we collected any batches, concatenate and yield them + // If we collected any batches, concatenate and yield them wrapped with range if !record_batches.is_empty() { let schema = record_batches[0].schema(); match concat_batches(&schema, &record_batches) { Ok(combined_batch) => { - debug!("Yielding combined batch with {} rows for blocks {} to {}", - combined_batch.num_rows(), current_block, batch_end); - yield Ok(combined_batch); + if let Some(ref m) = metrics { + // Track items processed + m.items_processed_inc(0, segment_num, data_type, combined_batch.num_rows() as u64); + + // Update progress + let blocks_processed = batch_end - seg_start + 1; + let progress_pct = (blocks_processed as f64 / total_blocks as f64) * 100.0; + m.set_worker_progress(0, segment_num, data_type, progress_pct); + } + + batches_yielded += 1; + info!( + "Segment {}: Yielding {} rows for blocks {} to {} ({:.0}ms, {} errors)", + segment_num, combined_batch.num_rows(), current_block, batch_end, + batch_duration_ms, error_count + ); + yield Ok(phaser_server::BatchWithRange::new(combined_batch, current_block, batch_end)); } Err(e) => { - error!("Failed to concatenate batches: {}", e); + error!("Segment {}: Failed to concatenate batches: {}", segment_num, e); + if let Some(ref m) = metrics { + m.error("concatenate_error", data_type); + m.active_workers_dec(data_type); + m.segment_attempt(false); + } yield Err(Status::internal(format!("Failed to concatenate batches: {e}"))); + return; } } + } else if error_count > 0 { + // All blocks had errors - report the batch as failed + error!( + "Segment {}: All {} blocks failed in range {}-{}", + segment_num, batch_count, current_block, batch_end + ); + if let Some(ref m) = metrics { + m.error("all_blocks_failed", data_type); + m.active_workers_dec(data_type); + m.segment_attempt(false); + } + yield Err(Status::internal(format!( + "Segment {segment_num}: Failed to fetch any blocks in range {current_block}-{batch_end}" + ))); + return; } current_block = batch_end + 1; } - info!("Completed historical {:?} query for blocks {} to {}", stream_type, start_block, end_block); + // Record segment-level metrics (success path only - errors return early) + let duration = segment_start_time.elapsed(); + if let Some(ref m) = metrics { + m.segment_duration(data_type, duration.as_secs_f64()); + m.active_workers_dec(data_type); + m.segment_attempt(true); + } + + let items_per_second = if duration.as_secs_f64() > 0.0 { + total_items as f64 / duration.as_secs_f64() + } else { + 0.0 + }; + + info!( + "Segment {}: Completed {} for blocks {} to {} in {:.2}s ({} items in {} batches, {:.0} items/sec)", + segment_num, data_type, seg_start, seg_end, duration.as_secs_f64(), + total_items, batches_yielded, items_per_second + ); + } + } + + /// Categorize error from Status for metrics + fn categorize_rpc_error_from_status(status: &Status) -> String { + let msg = status.message().to_lowercase(); + if msg.contains("timeout") || msg.contains("timed out") { + "timeout".to_string() + } else if msg.contains("connection") || msg.contains("connect") { + "connection".to_string() + } else if msg.contains("rate limit") || msg.contains("429") { + "rate_limit".to_string() + } else if msg.contains("not found") || msg.contains("404") { + "not_found".to_string() + } else if msg.contains("server error") || msg.contains("500") { + "server_error".to_string() + } else { + "rpc_error".to_string() + } + } + + /// Fetch a single block and convert it to RecordBatch based on stream type + async fn fetch_and_convert_block( + client: &JsonRpcClient, + block_num: u64, + stream_type: StreamType, + validate: bool, + validator: Option<&Arc>, + metrics: Option<&BridgeMetrics>, + segment_num: u64, + ) -> Result, Status> { + use alloy::eips::BlockNumberOrTag; + use alloy_rpc_types_eth::Filter; + + // Fetch block with transactions + let start = Instant::now(); + let block = match client + .get_block_with_txs(BlockNumberOrTag::Number(block_num)) + .await + { + Ok(Some(block)) => { + if let Some(m) = metrics { + m.grpc_request_duration_blocks( + segment_num, + "eth_getBlockByNumber", + start.elapsed().as_millis() as f64, + ); + } + block + } + Ok(None) => { + error!("Block #{} not found", block_num); + if let Some(m) = metrics { + m.error("not_found", "blocks"); + } + return Err(Status::not_found(format!("Block {block_num} not found"))); + } + Err(e) => { + error!("Failed to fetch block #{}: {}", block_num, e); + if let Some(m) = metrics { + m.error("fetch_error", "blocks"); + } + return Err(Status::internal(format!( + "Failed to fetch block {block_num}: {e}" + ))); + } + }; + + match stream_type { + StreamType::Blocks => { + // Convert block header to RecordBatch + match evm_common::rpc_conversions::convert_any_header(&block.header) { + Ok(batch) => Ok(Some(batch)), + Err(e) => { + error!("Failed to convert block header #{}: {}", block_num, e); + Err(Status::internal(format!("Conversion error: {e}"))) + } + } + } + StreamType::Transactions => { + // Convert transactions (if any) + if block.transactions.is_empty() { + return Ok(None); + } + + // Validate transactions if requested and validator is available + if validate { + if let Some(val) = validator { + let block_record = + evm_common::rpc_conversions::convert_any_header_to_record( + &block.header, + ); + + match evm_common::rpc_conversions::extract_transaction_records(&block) { + Ok(tx_records) => { + match val.spawn_validate_records(block_record, tx_records).await { + Ok(()) => { + debug!( + "Validated {} transactions for block #{}", + block.transactions.len(), + block_num + ); + } + Err(e) => { + error!( + "Transaction validation failed for block #{}: {}", + block_num, e + ); + return Err(Status::internal(format!( + "Validation error for block {block_num}: {e}" + ))); + } + } + } + Err(e) => { + error!( + "Failed to extract transaction records for validation: {}", + e + ); + return Err(Status::internal(format!( + "Failed to extract records for validation: {e}" + ))); + } + } + } else { + error!("Validation requested but validator not configured"); + return Err(Status::failed_precondition( + "Validation requested but validator not configured", + )); + } + } + + // Convert to RecordBatch + match JsonRpcConverter::convert_transactions(&block) { + Ok(batch) => Ok(Some(batch)), + Err(e) => { + error!( + "Failed to convert transactions for block #{}: {}", + block_num, e + ); + Err(Status::internal(format!("Conversion error: {e}"))) + } + } + } + StreamType::Logs => { + // Fetch and convert logs + let filter = Filter::new().from_block(block_num).to_block(block_num); + + let log_start = Instant::now(); + match client.get_logs(filter).await { + Ok(logs) if !logs.is_empty() => { + if let Some(m) = metrics { + m.grpc_request_duration_logs( + segment_num, + "eth_getLogs", + log_start.elapsed().as_millis() as f64, + ); + } + let block_hash = block.header.hash; + match JsonRpcConverter::convert_logs( + &logs, + block_num, + block_hash, + block.header.timestamp, + ) { + Ok(batch) => Ok(Some(batch)), + Err(e) => { + error!("Failed to convert logs for block #{}: {}", block_num, e); + Err(Status::internal(format!("Conversion error: {e}"))) + } + } + } + Ok(_) => { + // No logs in this block + if let Some(m) = metrics { + m.grpc_request_duration_logs( + segment_num, + "eth_getLogs", + log_start.elapsed().as_millis() as f64, + ); + } + Ok(None) + } + Err(e) => { + error!("Failed to fetch logs for block #{}: {}", block_num, e); + if let Some(m) = metrics { + m.error("fetch_error", "logs"); + } + Err(Status::internal(format!("Failed to fetch logs: {e}"))) + } + } + } + StreamType::Trie => Err(Status::unimplemented( + "Trie streaming not supported via JSON-RPC", + )), } } } @@ -309,7 +963,107 @@ impl FlightBridge for JsonRpcFlightBridge { supports_reorg_notifications: false, supports_filters: true, supports_validation: self.validator.is_some(), - max_batch_size: self.max_batch_size, + max_batch_size: self.segment_config.blocks_per_batch, + }) + } + + async fn get_discovery_capabilities(&self) -> Result { + // Query current block from node + let current_block = self.client.get_block_number().await.unwrap_or(0); + + // Get node capabilities (probed at connection time) + let caps = self.client.capabilities(); + + // Define available tables (JSON-RPC doesn't support trie) + // If eth_getBlockReceipts is supported, we could add a "receipts" table + let mut tables = vec![ + TableDescriptor::new("blocks", "_block_num") + .with_modes(vec!["historical", "live"]) + .with_sorted_by(vec!["_block_num"]), + TableDescriptor::new("transactions", "_block_num") + .with_modes(vec!["historical", "live"]) + .with_sorted_by(vec!["_block_num", "_tx_idx"]), + TableDescriptor::new("logs", "_block_num") + .with_modes(vec!["historical", "live"]) + .with_sorted_by(vec!["_block_num", "_tx_idx", "_log_idx"]), + ]; + + // Add receipts table if eth_getBlockReceipts is supported + if caps.supports_block_receipts { + tables.push( + TableDescriptor::new("receipts", "_block_num") + .with_modes(vec!["historical"]) + .with_sorted_by(vec!["_block_num", "_tx_idx"]), + ); + } + + // Build metadata from node capabilities + let mut metadata = std::collections::HashMap::new(); + + // Chain info + metadata.insert( + "chain_id".to_string(), + serde_json::Value::Number(self.chain_id.into()), + ); + + // Transport capabilities + metadata.insert( + "supports_subscriptions".to_string(), + serde_json::Value::Bool(self.client.supports_subscriptions()), + ); + + // Node capabilities (from probing at connection time) + metadata.insert( + "client_version".to_string(), + serde_json::Value::String(caps.client_version.clone()), + ); + metadata.insert( + "supports_block_receipts".to_string(), + serde_json::Value::Bool(caps.supports_block_receipts), + ); + metadata.insert( + "supports_debug_namespace".to_string(), + serde_json::Value::Bool(caps.supports_debug_namespace), + ); + + // Query hints for clients + if let Some(max_range) = caps.max_logs_block_range { + metadata.insert( + "max_logs_block_range".to_string(), + serde_json::Value::Number(max_range.into()), + ); + } + metadata.insert( + "recommended_logs_batch_size".to_string(), + serde_json::Value::Number(self.client.recommended_logs_batch_size().into()), + ); + + // Node type hints + if caps.is_erigon { + metadata.insert( + "node_type".to_string(), + serde_json::Value::String("erigon".to_string()), + ); + } else if caps.is_geth { + metadata.insert( + "node_type".to_string(), + serde_json::Value::String("geth".to_string()), + ); + } + metadata.insert( + "is_managed_provider".to_string(), + serde_json::Value::Bool(caps.is_managed_provider), + ); + + Ok(DiscoveryCapabilities { + name: "jsonrpc-bridge".to_string(), + version: env!("CARGO_PKG_VERSION").to_string(), + protocol: "evm".to_string(), + position_label: "block_number".to_string(), + current_position: current_block, + oldest_position: 0, + tables, + metadata, }) } @@ -363,9 +1117,10 @@ impl FlightBridge for JsonRpcFlightBridge { request: Request, ) -> std::result::Result, Status> { let descriptor = request.into_inner(); - let blockchain_desc = Self::parse_descriptor(&descriptor).map_err(|e| *e)?; + let query = Self::parse_descriptor(&descriptor)?; + let stream_type = Self::table_to_stream_type(&query.table)?; - let info = create_flight_info(blockchain_desc.stream_type).map_err(|e| *e)?; + let info = create_flight_info(stream_type).map_err(|e| *e)?; Ok(Response::new(info)) } @@ -375,8 +1130,9 @@ impl FlightBridge for JsonRpcFlightBridge { request: Request, ) -> std::result::Result, Status> { let descriptor = request.into_inner(); - let blockchain_desc = Self::parse_descriptor(&descriptor).map_err(|e| *e)?; - let schema = Self::get_schema_for_type(blockchain_desc.stream_type).map_err(|e| *e)?; + let query = Self::parse_descriptor(&descriptor)?; + let stream_type = Self::table_to_stream_type(&query.table)?; + let schema = Self::get_schema_for_type(stream_type).map_err(|e| *e)?; // Convert Arrow schema to IPC format for Flight let ipc_message = { @@ -412,52 +1168,40 @@ impl FlightBridge for JsonRpcFlightBridge { > { let ticket = request.into_inner(); - // Parse ticket to determine what data to stream - let blockchain_desc = String::from_utf8(ticket.ticket.to_vec()) - .map_err(|_| Status::invalid_argument("Ticket is not valid UTF-8")) - .and_then(|s| { - serde_json::from_str::(&s) - .map_err(|e| { - Status::invalid_argument(format!("Invalid descriptor in ticket: {e}")) - }) - })?; + // Parse the GenericQuery from the ticket + let query = Self::parse_ticket(&ticket)?; + let stream_type = Self::table_to_stream_type(&query.table)?; - let stream_type = blockchain_desc.stream_type; - let query_mode = blockchain_desc.query_mode.clone(); info!( - "Processing do_get for {:?} in {:?} mode", - stream_type, query_mode + "Processing do_get for table '{}' ({:?}) with mode {:?}", + query.table, stream_type, query.mode ); // Get schema for the stream type let schema = Self::get_schema_for_type(stream_type).map_err(|e| *e)?; - // Determine if we should do conversion validation - // JSON-RPC doesn't give us raw RLP, so we can only do conversion validation - use phaser_bridge::ValidationStage; - let should_validate_conversion = matches!( - blockchain_desc.validation, - ValidationStage::Conversion | ValidationStage::Both - ); - - // Branch based on query mode + // Build the batch stream based on query mode + // Historical streams return BatchWithRange, live streams return plain RecordBatch let batch_stream: Pin< - Box> + Send>, - > = match query_mode { - QueryMode::Historical { start, end } => { - // Historical query - fetch specific block range + Box> + Send>, + > = match query.mode { + GenericQueryMode::Range { start, end } => { info!( - "Creating historical stream for {:?} blocks {}-{} (validation: {:?})", - stream_type, start, end, blockchain_desc.validation + "Creating historical stream for {:?} positions {}-{}", + stream_type, start, end ); Box::pin(self.create_historical_stream( stream_type, start, end, - should_validate_conversion, + false, // No validation via generic query )) } - QueryMode::Live => { + GenericQueryMode::Snapshot { at } => { + info!("Creating snapshot at position {}", at); + Box::pin(self.create_historical_stream(stream_type, at, at, false)) + } + GenericQueryMode::Live => { // Live streaming - subscribe to broadcast channels let receiver = match stream_type { StreamType::Blocks => self.streaming_service.subscribe_blocks(), @@ -470,31 +1214,72 @@ impl FlightBridge for JsonRpcFlightBridge { } }; + // Wrap live batches with placeholder range (0,0) since they're real-time Box::pin(async_stream::stream! { let mut rx = receiver; while let Ok(batch) = rx.recv().await { - yield Ok(batch); + yield Ok(phaser_server::BatchWithRange::new(batch, 0, 0)); } }) } }; - // Convert Status errors to FlightError for the encoder - let flight_batch_stream = batch_stream.map(|result| { - result.map_err(|status| arrow_flight::error::FlightError::Tonic(Box::new(status))) - }); - - // Encode as Flight data - let encoder = FlightDataEncoderBuilder::new() - .with_schema(schema) - .build(flight_batch_stream); - - let flight_stream = encoder.map(|result| { - result.map_err(|e| { - error!("Error encoding flight data: {}", e); - Status::internal(format!("Encoding error: {e}")) - }) - }); + // Manually construct FlightData to include app_metadata with responsibility ranges + // This mirrors the erigon-bridge approach for phaser-query compatibility + use arrow::ipc::writer::IpcWriteOptions; + use arrow_flight::utils::batches_to_flight_data; + + let flight_stream = async_stream::stream! { + // First, send the schema + let schema_flight_data: FlightData = arrow_flight::SchemaAsIpc::new(&schema, &IpcWriteOptions::default()) + .into(); + yield Ok(schema_flight_data); + + // Then stream batches with app_metadata containing responsibility ranges + let mut batch_stream = batch_stream; + while let Some(batch_result) = batch_stream.next().await { + match batch_result { + Ok(batch_with_range) => { + // Encode the batch metadata (responsibility range) + let metadata = match batch_with_range.encode_metadata() { + Ok(m) => m, + Err(e) => { + error!("Failed to encode batch metadata: {}", e); + yield Err(Status::internal(format!("Metadata encoding error: {e}"))); + continue; + } + }; + + // Convert RecordBatch to FlightData using arrow-flight utilities + let batches = vec![batch_with_range.batch]; + match batches_to_flight_data(&schema, batches) { + Ok(flight_data_vec) => { + // batches_to_flight_data includes a schema message as the first element + // Skip it since we already sent the schema + let data_messages: Vec<_> = flight_data_vec + .into_iter() + .skip(1) // Skip the schema message + .collect(); + + // Attach app_metadata to each FlightData + for mut flight_data in data_messages { + flight_data.app_metadata = metadata.clone().into(); + yield Ok(flight_data); + } + } + Err(e) => { + error!("Error encoding batch to flight data: {}", e); + yield Err(Status::internal(format!("Batch encoding error: {e}"))); + } + } + } + Err(e) => { + error!("Error in batch stream: {}", e); + yield Err(Status::internal(format!("Stream error: {e}"))); + } + } + } + }; Ok(Response::new(Box::pin(flight_stream))) } @@ -518,7 +1303,7 @@ impl FlightBridge for JsonRpcFlightBridge { let stream_type = if let Some(desc) = first.flight_descriptor { Self::parse_descriptor(&desc) - .map(|bd| bd.stream_type) + .and_then(|q| Self::table_to_stream_type(&q.table)) .unwrap_or(StreamType::Blocks) // Safe fallback for failed descriptor parsing } else { StreamType::Blocks @@ -635,16 +1420,18 @@ impl FlightService for JsonRpcFlightBridge { async fn do_action( &self, - _request: Request, + request: Request, ) -> std::result::Result, Status> { - Err(Status::unimplemented("do_action not supported")) + ::do_action(self, request).await } async fn list_actions( &self, _request: Request, ) -> std::result::Result, Status> { - Err(Status::unimplemented("list_actions not supported")) + let actions = ::list_actions(self).await?; + let stream = stream::iter(actions.into_iter().map(Ok)); + Ok(Response::new(Box::pin(stream))) } async fn get_schema( @@ -652,8 +1439,9 @@ impl FlightService for JsonRpcFlightBridge { request: Request, ) -> std::result::Result, Status> { let descriptor = request.into_inner(); - let stream_type = Self::parse_descriptor(&descriptor).map_err(|e| *e)?; - let schema = Self::get_schema_for_type(stream_type.stream_type).map_err(|e| *e)?; + let query = Self::parse_descriptor(&descriptor)?; + let stream_type = Self::table_to_stream_type(&query.table)?; + let schema = Self::get_schema_for_type(stream_type).map_err(|e| *e)?; // Convert Schema to IPC format let options = arrow::ipc::writer::IpcWriteOptions::default(); @@ -694,3 +1482,53 @@ fn create_flight_info(stream_type: StreamType) -> Result .with_total_records(0) .with_total_bytes(0)) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_split_into_segments() { + // Test full segments + let segments = split_into_segments(0, 29_999, 10_000); + assert_eq!(segments.len(), 3); + assert_eq!(segments[0], (0, 9_999)); + assert_eq!(segments[1], (10_000, 19_999)); + assert_eq!(segments[2], (20_000, 29_999)); + } + + #[test] + fn test_split_partial_segment() { + // Test partial last segment + let segments = split_into_segments(0, 15_000, 10_000); + assert_eq!(segments.len(), 2); + assert_eq!(segments[0], (0, 9_999)); + assert_eq!(segments[1], (10_000, 15_000)); + } + + #[test] + fn test_split_single_segment() { + // Test range smaller than segment size + let segments = split_into_segments(0, 5_000, 10_000); + assert_eq!(segments.len(), 1); + assert_eq!(segments[0], (0, 5_000)); + } + + #[test] + fn test_split_unaligned_start() { + // Test starting from an unaligned block + let segments = split_into_segments(5_000, 25_000, 10_000); + assert_eq!(segments.len(), 3); + assert_eq!(segments[0], (5_000, 14_999)); + assert_eq!(segments[1], (15_000, 24_999)); + assert_eq!(segments[2], (25_000, 25_000)); + } + + #[test] + fn test_split_single_block() { + // Test single block range + let segments = split_into_segments(100, 100, 10_000); + assert_eq!(segments.len(), 1); + assert_eq!(segments[0], (100, 100)); + } +} diff --git a/crates/bridges/evm/jsonrpc-bridge/src/client.rs b/crates/bridges/evm/jsonrpc-bridge/src/client.rs index c53aabb..9a2b1f5 100644 --- a/crates/bridges/evm/jsonrpc-bridge/src/client.rs +++ b/crates/bridges/evm/jsonrpc-bridge/src/client.rs @@ -1,10 +1,62 @@ -use alloy::network::{AnyHeader, AnyNetwork, AnyRpcBlock}; +use alloy::network::{AnyHeader, AnyNetwork, AnyRpcBlock, ReceiptResponse}; use alloy::providers::{Provider, ProviderBuilder}; use alloy_pubsub::Subscription; use alloy_rpc_types_eth::{BlockNumberOrTag, Filter, Header, Log}; use anyhow::{anyhow, Result}; use std::sync::Arc; -use tracing::{debug, info}; +use tracing::{debug, info, warn}; + +/// Detected capabilities of the connected node +/// +/// Probed at connection time to enable optimal fetching strategies: +/// - `eth_getBlockReceipts` is much faster than individual receipt fetches +/// - `debug_*` methods enable trace data collection +/// - Log range limits affect batch sizing +#[derive(Clone, Debug, Default)] +pub struct NodeCapabilities { + /// Node supports `eth_getBlockReceipts` (EIP-1474) + /// Returns all receipts for a block in one call + pub supports_block_receipts: bool, + + /// Node supports `debug_*` namespace methods + pub supports_debug_namespace: bool, + + /// Maximum block range for eth_getLogs (None = unlimited) + /// Some providers limit: Infura=10000, Alchemy=2000, QuickNode=10000 + pub max_logs_block_range: Option, + + /// Node client version string (from web3_clientVersion) + pub client_version: String, + + /// Is this an Erigon node? (enables Erigon-specific optimizations) + pub is_erigon: bool, + + /// Is this a Geth node? + pub is_geth: bool, + + /// Is this a managed provider (Infura, Alchemy, etc.)? + pub is_managed_provider: bool, +} + +impl NodeCapabilities { + /// Log a summary of detected capabilities + pub fn log_summary(&self) { + info!("Node capabilities detected:"); + info!(" Client: {}", self.client_version); + info!(" eth_getBlockReceipts: {}", self.supports_block_receipts); + info!(" debug namespace: {}", self.supports_debug_namespace); + if let Some(limit) = self.max_logs_block_range { + info!(" eth_getLogs max range: {} blocks", limit); + } else { + info!(" eth_getLogs max range: unlimited"); + } + if self.is_erigon { + info!(" Node type: Erigon"); + } else if self.is_geth { + info!(" Node type: Geth"); + } + } +} /// Client for connecting to JSON-RPC nodes #[derive(Clone)] @@ -12,6 +64,7 @@ pub struct JsonRpcClient { provider: Arc>, chain_id: u64, supports_subscriptions: bool, + capabilities: NodeCapabilities, } impl JsonRpcClient { @@ -60,13 +113,103 @@ impl JsonRpcClient { let block_num = provider.get_block_number().await?; info!("Current block number: {}", block_num); + // Detect node capabilities + let capabilities = Self::detect_capabilities(&provider).await; + capabilities.log_summary(); + Ok(Self { provider, chain_id, supports_subscriptions, + capabilities, }) } + /// Detect node capabilities by probing supported methods + async fn detect_capabilities(provider: &Arc>) -> NodeCapabilities { + // Get client version first + let client_version = provider + .get_client_version() + .await + .unwrap_or_else(|_| "unknown".to_string()); + + // Detect node type from version string + let version_lower = client_version.to_lowercase(); + let is_erigon = version_lower.contains("erigon"); + let is_geth = version_lower.contains("geth"); + let is_managed_provider = version_lower.contains("infura") + || version_lower.contains("alchemy") + || version_lower.contains("quicknode") + || version_lower.contains("ankr"); + + // Probe supported methods + let supports_block_receipts = Self::probe_block_receipts_support(provider).await; + let supports_debug_namespace = Self::probe_debug_support(provider).await; + + // Set known limits for managed providers + let max_logs_block_range = if is_managed_provider { + Some(2000) // Conservative default - most providers limit to 2000-10000 + } else { + None + }; + + NodeCapabilities { + supports_block_receipts, + supports_debug_namespace, + max_logs_block_range, + client_version, + is_erigon, + is_geth, + is_managed_provider, + } + } + + /// Probe if eth_getBlockReceipts is supported + async fn probe_block_receipts_support(provider: &Arc>) -> bool { + // Try to get receipts for block 1 (should exist on any chain) + match provider + .get_block_receipts(BlockNumberOrTag::Number(1).into()) + .await + { + Ok(Some(_)) => { + debug!("eth_getBlockReceipts is supported"); + true + } + Ok(None) => { + // Block exists but no receipts - method still works + debug!("eth_getBlockReceipts is supported (no receipts in block 1)"); + true + } + Err(e) => { + let err_str = e.to_string().to_lowercase(); + // Method not supported vs other errors + if err_str.contains("method not found") + || err_str.contains("not supported") + || err_str.contains("unknown method") + { + debug!("eth_getBlockReceipts not supported: {}", e); + false + } else { + // Other error (rate limit, etc) - assume supported + warn!( + "eth_getBlockReceipts probe error (assuming supported): {}", + e + ); + true + } + } + } + } + + /// Probe if debug namespace is supported + async fn probe_debug_support(_provider: &Arc>) -> bool { + // We'd need raw RPC calls here since Provider doesn't expose debug_* + // For now, infer from node type + // Erigon and Geth both support debug namespace by default + // Managed providers typically don't expose it + false // TODO: Implement raw RPC probe + } + /// Get the chain ID pub fn chain_id(&self) -> u64 { self.chain_id @@ -140,4 +283,41 @@ impl JsonRpcClient { let version = self.provider.get_client_version().await?; Ok(version) } + + /// Get detected node capabilities + pub fn capabilities(&self) -> &NodeCapabilities { + &self.capabilities + } + + /// Get all receipts for a block (requires eth_getBlockReceipts support) + /// + /// This is much more efficient than fetching individual receipts. + /// Check `capabilities().supports_block_receipts` before calling. + /// + /// Returns receipts implementing `ReceiptResponse` trait for accessing + /// common fields like `status()`, `gas_used()`, `logs()`, etc. + pub async fn get_block_receipts( + &self, + block: BlockNumberOrTag, + ) -> Result>> { + if !self.capabilities.supports_block_receipts { + return Err(anyhow!("eth_getBlockReceipts not supported by this node")); + } + + debug!("Fetching all receipts for block {:?}", block); + let receipts = self.provider.get_block_receipts(block.into()).await?; + + if let Some(ref r) = receipts { + debug!("Got {} receipts", r.len()); + } + + Ok(receipts) + } + + /// Get recommended batch size for eth_getLogs based on node capabilities + /// + /// Returns the maximum block range that should be used per eth_getLogs call. + pub fn recommended_logs_batch_size(&self) -> u64 { + self.capabilities.max_logs_block_range.unwrap_or(10_000) // Default for self-hosted nodes + } } diff --git a/crates/bridges/evm/jsonrpc-bridge/src/converter.rs b/crates/bridges/evm/jsonrpc-bridge/src/converter.rs index a89e408..a298dd8 100644 --- a/crates/bridges/evm/jsonrpc-bridge/src/converter.rs +++ b/crates/bridges/evm/jsonrpc-bridge/src/converter.rs @@ -35,7 +35,9 @@ impl JsonRpcConverter { Ok(rpc_conversions::convert_rpc_transactions(block)?) } - /// Convert logs to RecordBatch + /// Convert logs to RecordBatch (single block version) + /// + /// For logs from a single block where you already have the block context. pub fn convert_logs( logs: &[Log], block_num: u64, @@ -46,4 +48,13 @@ impl JsonRpcConverter { logs, block_num, block_hash, timestamp, )?) } + + /// Convert logs to RecordBatch (multi-block version) + /// + /// For logs fetched via eth_getLogs with a block range. Extracts block_num, + /// block_hash, and timestamp from each log. More efficient than per-block + /// fetching for historical sync. + pub fn convert_logs_multi_block(logs: &[Log]) -> Result { + Ok(rpc_conversions::convert_rpc_logs_multi_block(logs)?) + } } diff --git a/crates/bridges/evm/jsonrpc-bridge/src/lib.rs b/crates/bridges/evm/jsonrpc-bridge/src/lib.rs index d33fd11..b1a4107 100644 --- a/crates/bridges/evm/jsonrpc-bridge/src/lib.rs +++ b/crates/bridges/evm/jsonrpc-bridge/src/lib.rs @@ -2,9 +2,11 @@ pub mod bridge; pub mod client; pub mod converter; pub mod error; +pub mod metrics; pub mod streaming; -pub use bridge::JsonRpcFlightBridge; -pub use client::JsonRpcClient; +pub use bridge::{split_into_segments, JsonRpcFlightBridge, SegmentConfig}; +pub use client::{JsonRpcClient, NodeCapabilities}; pub use converter::JsonRpcConverter; +pub use metrics::{gather_metrics, BridgeMetrics, MetricsLayer, SegmentMetrics, WorkerStage}; pub use streaming::StreamingService; diff --git a/crates/bridges/evm/jsonrpc-bridge/src/main.rs b/crates/bridges/evm/jsonrpc-bridge/src/main.rs index b0598a5..04e9283 100644 --- a/crates/bridges/evm/jsonrpc-bridge/src/main.rs +++ b/crates/bridges/evm/jsonrpc-bridge/src/main.rs @@ -1,10 +1,12 @@ use anyhow::Result; use arrow_flight::flight_service_server::FlightServiceServer; use clap::Parser; -use jsonrpc_bridge::JsonRpcFlightBridge; +use jsonrpc_bridge::{gather_metrics, JsonRpcFlightBridge, MetricsLayer, SegmentConfig}; use std::net::SocketAddr; use tonic::transport::Server; use tracing::{error, info}; +use tracing_subscriber::layer::SubscriberExt; +use tracing_subscriber::util::SubscriberInitExt; use validators_evm::ExecutorType; #[derive(Parser, Debug)] @@ -40,22 +42,54 @@ struct Args { #[arg(long, env = "EXECUTOR_THREADS")] threads: Option, + /// Segment size in blocks for parallel fetching (default: 10000) + #[arg(long, env = "SEGMENT_SIZE", default_value_t = 10_000)] + segment_size: u64, + + /// Maximum number of segments to process in parallel (default: 4) + #[arg(long, env = "MAX_CONCURRENT_SEGMENTS", default_value_t = 4)] + max_concurrent_segments: usize, + + /// Number of blocks to fetch in parallel within a segment (default: 50) + /// + /// For logs: One eth_getLogs call covers this many blocks. + /// For blocks/txs: This many blocks are fetched concurrently. + #[arg(long, env = "BLOCKS_PER_BATCH", default_value_t = 50)] + blocks_per_batch: usize, + + /// Maximum concurrent RPC requests (default: same as blocks_per_batch) + /// + /// Limits how many eth_getBlockByNumber calls run simultaneously. + /// Reduce for rate-limited nodes (e.g., 10-20 for Infura/Alchemy). + #[arg(long, env = "MAX_CONCURRENT_REQUESTS")] + max_concurrent_requests: Option, + /// Enable debug logging #[arg(long, short, env = "DEBUG")] debug: bool, + + /// Prometheus metrics port (default: 9091) + #[arg(long, env = "METRICS_PORT", default_value_t = 9091)] + metrics_port: u16, } #[tokio::main] async fn main() -> Result<()> { let args = Args::parse(); - // Initialize logging - let log_level = if args.debug { "debug" } else { "info" }; - tracing_subscriber::fmt() - .with_env_filter( + // Initialize tracing with metrics layer + let log_level = if args.debug { + "jsonrpc_bridge=debug" + } else { + "jsonrpc_bridge=info" + }; + tracing_subscriber::registry() + .with( tracing_subscriber::EnvFilter::try_from_default_env() .unwrap_or_else(|_| tracing_subscriber::EnvFilter::new(log_level)), ) + .with(tracing_subscriber::fmt::layer().with_ansi(false)) + .with(MetricsLayer::new("jsonrpc-bridge")) .init(); info!("Starting JSON-RPC Flight Bridge"); @@ -70,14 +104,40 @@ async fn main() -> Result<()> { info!("Validation enabled with executor: {:?}", config); } - // Create the bridge - let bridge = - JsonRpcFlightBridge::new(args.jsonrpc_url.clone(), args.chain_id, validator_config) - .await - .map_err(|e| { - error!("Failed to create JSON-RPC bridge: {}", e); - e - })?; + // Build segment config from CLI args + let segment_config = SegmentConfig { + segment_size: args.segment_size, + max_concurrent_segments: args.max_concurrent_segments, + blocks_per_batch: args.blocks_per_batch, + max_concurrent_requests: args + .max_concurrent_requests + .unwrap_or(args.blocks_per_batch), + }; + + info!("Segment configuration:"); + info!(" Segment size: {} blocks", segment_config.segment_size); + info!( + " Max concurrent segments: {}", + segment_config.max_concurrent_segments + ); + info!(" Blocks per batch: {}", segment_config.blocks_per_batch); + info!( + " Max concurrent RPC requests: {}", + segment_config.max_concurrent_requests + ); + + // Create the bridge with segment config + let bridge = JsonRpcFlightBridge::with_segment_config( + args.jsonrpc_url.clone(), + args.chain_id, + validator_config, + segment_config, + ) + .await + .map_err(|e| { + error!("Failed to create JSON-RPC bridge: {}", e); + e + })?; let bridge_info = bridge.bridge_info(); info!( @@ -85,6 +145,41 @@ async fn main() -> Result<()> { serde_json::to_string_pretty(&bridge_info)? ); + // Start Prometheus metrics server + let metrics_port = args.metrics_port; + tokio::spawn(async move { + use axum::{response::IntoResponse, routing::get, Router}; + + async fn metrics_handler() -> impl IntoResponse { + match gather_metrics() { + Ok(metrics) => ( + axum::http::StatusCode::OK, + [("content-type", "text/plain; version=0.0.4")], + metrics, + ), + Err(e) => ( + axum::http::StatusCode::INTERNAL_SERVER_ERROR, + [("content-type", "text/plain")], + format!("Error gathering metrics: {e}"), + ), + } + } + + let app = Router::new().route("/metrics", get(metrics_handler)); + + let listener = tokio::net::TcpListener::bind(format!("0.0.0.0:{metrics_port}")) + .await + .unwrap(); + info!( + "Prometheus metrics server listening on {}", + listener.local_addr().unwrap() + ); + + if let Err(e) = axum::serve(listener, app).await { + error!("Metrics server error: {}", e); + } + }); + // Create the Flight service let flight_service = FlightServiceServer::new(bridge); diff --git a/crates/bridges/evm/jsonrpc-bridge/src/metrics.rs b/crates/bridges/evm/jsonrpc-bridge/src/metrics.rs new file mode 100644 index 0000000..35cd2e5 --- /dev/null +++ b/crates/bridges/evm/jsonrpc-bridge/src/metrics.rs @@ -0,0 +1,7 @@ +//! Prometheus metrics for jsonrpc-bridge +//! +//! Re-exports BridgeMetrics and MetricsLayer from phaser-metrics crate + +pub use phaser_metrics::{ + gather_metrics, BridgeMetrics, MetricsLayer, SegmentMetrics, WorkerStage, +}; diff --git a/crates/parquet-index/src/reader.rs b/crates/parquet-index/src/reader.rs index 8d77b22..11aa2cf 100644 --- a/crates/parquet-index/src/reader.rs +++ b/crates/parquet-index/src/reader.rs @@ -4,6 +4,7 @@ use crate::{FileRegistry, ReaderError}; use arrow::record_batch::RecordBatch; use fusio::disk::LocalFs; +use fusio::executor::NoopExecutor; use fusio::path::Path; use fusio::DynFs; use fusio_parquet::reader::AsyncReader; @@ -56,8 +57,8 @@ impl PageReader { // Get file size for AsyncReader let content_length = file.size().await?; - // Create async parquet reader - let async_reader = AsyncReader::new(Box::new(file), content_length).await?; + // Create async parquet reader (NoopExecutor for tokio runtime) + let async_reader = AsyncReader::new(Box::new(file), content_length, NoopExecutor).await?; // Build the stream reader for the specific row group let builder = ParquetRecordBatchStreamBuilder::new(async_reader).await?; diff --git a/crates/phaser-bridge/src/bridge.rs b/crates/phaser-bridge/src/bridge.rs deleted file mode 100644 index acab113..0000000 --- a/crates/phaser-bridge/src/bridge.rs +++ /dev/null @@ -1,73 +0,0 @@ -use arrow_flight::{ - Criteria, FlightData, FlightDescriptor, FlightInfo, HandshakeRequest, HandshakeResponse, - SchemaResult, Ticket, -}; -use async_trait::async_trait; -use futures::Stream; -use std::pin::Pin; -use tonic::{Request, Response, Status, Streaming}; - -use crate::descriptors::BridgeInfo; - -/// Capabilities that a bridge can advertise -#[derive(Debug, Clone)] -pub struct BridgeCapabilities { - pub supports_historical: bool, - pub supports_streaming: bool, - pub supports_reorg_notifications: bool, - pub supports_filters: bool, - pub supports_validation: bool, - pub max_batch_size: usize, -} - -/// Trait for blockchain data bridges to implement -#[async_trait] -pub trait FlightBridge: Send + Sync + 'static { - /// Get information about this bridge - async fn get_info(&self) -> Result; - - /// Get capabilities of this bridge - async fn get_capabilities(&self) -> Result; - - /// Handshake for authentication/authorization - async fn handshake( - &self, - request: Request>, - ) -> Result< - Response> + Send>>>, - Status, - >; - - /// Get flight information for a descriptor - async fn get_flight_info( - &self, - request: Request, - ) -> Result, Status>; - - /// Get schema for a stream type - async fn get_schema( - &self, - request: Request, - ) -> Result, Status>; - - /// List available flights - async fn list_flights( - &self, - request: Request, - ) -> Result> + Send>>>, Status>; - - /// Stream data for a ticket - async fn do_get( - &self, - request: Request, - ) -> Result> + Send>>>, Status>; - - /// Bidirectional streaming for real-time subscriptions - async fn do_exchange( - &self, - request: Request>, - ) -> Result> + Send>>>, Status>; - - /// Check health/readiness of the bridge - async fn health_check(&self) -> Result; -} diff --git a/crates/phaser-client/Cargo.toml b/crates/phaser-client/Cargo.toml new file mode 100644 index 0000000..d5eb413 --- /dev/null +++ b/crates/phaser-client/Cargo.toml @@ -0,0 +1,38 @@ +[package] +name = "phaser-client" +version = "0.1.0" +edition = "2021" +description = "Flight client for connecting to phaser bridges" + +[dependencies] +# Core types +phaser-types = { path = "../phaser-types" } + +# Arrow and Flight +arrow = { workspace = true } +arrow-flight = { workspace = true } +arrow-array = { workspace = true } +arrow-schema = { workspace = true } +arrow-ipc = { workspace = true } + +# Async and gRPC +tokio = { workspace = true } +tonic = { workspace = true } +futures = "0.3" +tower = "0.5" +hyper-util = "0.1" +async-stream = "0.3" + +# Serialization +serde = { workspace = true } +serde_json = { workspace = true } +prost = { workspace = true } + +# Error handling +thiserror = { workspace = true } + +# Logging +tracing = { workspace = true } + +# HTTP types (for InvalidUri) +http = "1.0" diff --git a/crates/phaser-bridge/src/client.rs b/crates/phaser-client/src/client.rs similarity index 57% rename from crates/phaser-bridge/src/client.rs rename to crates/phaser-client/src/client.rs index 73aef48..2b950ee 100644 --- a/crates/phaser-bridge/src/client.rs +++ b/crates/phaser-client/src/client.rs @@ -1,21 +1,31 @@ +//! Flight client for connecting to blockchain data bridges + use arrow::datatypes::Schema; use arrow_array::RecordBatch; -use arrow_flight::{flight_service_client::FlightServiceClient, FlightClient, FlightInfo}; +use arrow_flight::{flight_service_client::FlightServiceClient, Action, FlightClient, FlightInfo}; use futures::stream::StreamExt; use tonic::transport::Channel; use tracing::{debug, error, info}; -use crate::descriptors::{BlockchainDescriptor, BridgeInfo}; +use phaser_types::{ + BatchMetadata, BlockchainDescriptor, BridgeInfo, DiscoveryCapabilities, GenericQuery, + ACTION_DESCRIBE, +}; + +use crate::error::BridgeError; + +/// Result type for bridge client operations +pub type Result = std::result::Result; /// Client for connecting to blockchain data bridges -pub struct FlightBridgeClient { +pub struct PhaserClient { client: FlightClient, info: Option, } -impl FlightBridgeClient { +impl PhaserClient { /// Connect to a bridge endpoint (TCP or Unix domain socket) - pub async fn connect(endpoint: String) -> Result { + pub async fn connect(endpoint: String) -> Result { info!("Connecting to bridge at {}", endpoint); let channel = if endpoint.starts_with("unix:") @@ -62,8 +72,8 @@ impl FlightBridgeClient { #[cfg(not(unix))] { - return Err(anyhow::anyhow!( - "Unix domain sockets are not supported on this platform" + return Err(BridgeError::platform_not_supported( + "Unix domain sockets are not supported on this platform", )); } } else { @@ -76,17 +86,16 @@ impl FlightBridgeClient { Channel::from_shared(uri)?.connect().await? }; - // Configure message size limits (256MB global max for large batches) + // Configure message size limits (512MB global max for large batches) // This allows the client to receive large messages from the bridge - const MAX_MESSAGE_SIZE: usize = 256 * 1024 * 1024; + const MAX_MESSAGE_SIZE: usize = 512 * 1024 * 1024; // Client accepts compression if server sends it // Compression is controlled by the bridge's --compression flag let flight_service_client = FlightServiceClient::new(channel) .max_decoding_message_size(MAX_MESSAGE_SIZE) .max_encoding_message_size(MAX_MESSAGE_SIZE) - .accept_compressed(tonic::codec::CompressionEncoding::Gzip) - .accept_compressed(tonic::codec::CompressionEncoding::Zstd); + .accept_compressed(tonic::codec::CompressionEncoding::Gzip); let client = FlightClient::new_from_inner(flight_service_client); @@ -94,7 +103,7 @@ impl FlightBridgeClient { } /// Get bridge information - pub async fn get_info(&mut self) -> Result { + pub async fn get_info(&mut self) -> Result { // This would typically be implemented as a custom action // For now, return cached info or a placeholder if let Some(ref info) = self.info { @@ -116,7 +125,7 @@ impl FlightBridgeClient { pub async fn get_flight_info( &mut self, descriptor: &BlockchainDescriptor, - ) -> Result { + ) -> Result { let flight_desc = descriptor.to_flight_descriptor(); let response = self.client.get_flight_info(flight_desc).await?; Ok(response) @@ -126,7 +135,7 @@ impl FlightBridgeClient { pub async fn stream_data( &mut self, descriptor: &BlockchainDescriptor, - ) -> Result, anyhow::Error> { + ) -> Result> { let ticket = descriptor.to_ticket(); info!("Requesting data stream from bridge"); @@ -165,9 +174,11 @@ impl FlightBridgeClient { descriptor: &BlockchainDescriptor, ) -> Result< impl futures::Stream< - Item = Result<(RecordBatch, crate::BatchMetadata), arrow_flight::error::FlightError>, + Item = std::result::Result< + (RecordBatch, BatchMetadata), + arrow_flight::error::FlightError, + >, >, - anyhow::Error, > { let ticket = descriptor.to_ticket(); @@ -216,7 +227,7 @@ impl FlightBridgeClient { } // Extract and decode app_metadata (required) - let metadata = crate::BatchMetadata::decode(&flight_data.app_metadata) + let metadata = BatchMetadata::decode(&flight_data.app_metadata) .map_err(|e| arrow_flight::error::FlightError::DecodeError( format!("Failed to decode batch metadata: {e}") ))?; @@ -240,17 +251,14 @@ impl FlightBridgeClient { } /// Get schema for a stream type - pub async fn get_schema( - &mut self, - descriptor: &BlockchainDescriptor, - ) -> Result { + pub async fn get_schema(&mut self, descriptor: &BlockchainDescriptor) -> Result { let flight_desc = descriptor.to_flight_descriptor(); let schema = self.client.get_schema(flight_desc).await?; Ok(schema) } /// List available data streams - pub async fn list_available_streams(&mut self) -> Result, anyhow::Error> { + pub async fn list_available_streams(&mut self) -> Result> { use prost::bytes::Bytes; let mut stream = self.client.list_flights(Bytes::new()).await?; @@ -269,12 +277,12 @@ impl FlightBridgeClient { } /// Check if the bridge is healthy - pub async fn health_check(&mut self) -> Result { - // Implement a simple health check by trying to get flight info - let descriptor = - BlockchainDescriptor::historical(crate::descriptors::StreamType::Blocks, 0, 0); + pub async fn health_check(&mut self) -> Result { + // Implement a simple health check by trying to get flight info for blocks table + let query = GenericQuery::historical("blocks", 0, 0); + let flight_desc = query.to_flight_descriptor(); - match self.get_flight_info(&descriptor).await { + match self.client.get_flight_info(flight_desc).await { Ok(_) => Ok(true), Err(e) => { error!("Health check failed: {}", e); @@ -282,4 +290,153 @@ impl FlightBridgeClient { } } } + + // ==================== Protocol-Agnostic Discovery API ==================== + + /// Discover bridge capabilities using the "describe" action + /// + /// Returns protocol-agnostic information about available tables, + /// position semantics, and supported filters. + pub async fn discover(&mut self) -> Result { + info!("Discovering bridge capabilities"); + + let action = Action { + r#type: ACTION_DESCRIBE.to_string(), + body: Default::default(), + }; + + let mut inner_client = self.client.inner().clone(); + let mut stream = inner_client.do_action(action).await?.into_inner(); + + // The describe action returns a single result with JSON body + let result = stream + .next() + .await + .ok_or_else(|| BridgeError::discovery("No response from describe action"))??; + + let capabilities: DiscoveryCapabilities = serde_json::from_slice(&result.body)?; + + info!( + "Discovered bridge: {} v{} (protocol: {}, {} tables)", + capabilities.name, + capabilities.version, + capabilities.protocol, + capabilities.tables.len() + ); + + Ok(capabilities) + } + + /// Get schema for a table using generic query format + /// + /// Uses Flight's native GetSchema with the generic query descriptor. + pub async fn get_table_schema(&mut self, table: &str) -> Result { + let query = GenericQuery::historical(table, 0, 0); + let flight_desc = query.to_flight_descriptor(); + let schema = self.client.get_schema(flight_desc).await?; + Ok(schema) + } + + /// Stream data using generic query format + /// + /// Protocol-agnostic streaming that works with any bridge. + /// Returns raw RecordBatches - clients can import schema crates + /// for typed deserialization if needed. + pub async fn query( + &mut self, + query: GenericQuery, + ) -> Result< + impl futures::Stream>, + > { + let ticket = query.to_ticket(); + + info!( + "Querying table '{}' with mode {:?}", + query.table, query.mode + ); + let decoder = self.client.do_get(ticket).await?; + + Ok(decoder) + } + + /// Stream data with metadata using generic query format + /// + /// Like `query`, but also returns batch metadata (responsibility ranges). + pub async fn query_with_metadata( + &mut self, + query: GenericQuery, + ) -> Result< + impl futures::Stream< + Item = std::result::Result< + (RecordBatch, BatchMetadata), + arrow_flight::error::FlightError, + >, + >, + > { + let ticket = query.to_ticket(); + + info!( + "Querying table '{}' with metadata, mode {:?}", + query.table, query.mode + ); + + // Access the inner FlightServiceClient to get raw FlightData stream + let mut inner_client = self.client.inner().clone(); + let response = inner_client.do_get(ticket).await?; + let flight_data_stream = response.into_inner(); + + // Process FlightData to preserve app_metadata + let stream = async_stream::try_stream! { + use arrow_ipc::{root_as_message, convert::fb_to_schema}; + use std::sync::Arc; + + let mut flight_data_stream = flight_data_stream; + + let mut schema: Option = None; + let dictionaries_by_id = std::collections::HashMap::new(); + + while let Some(flight_data_result) = flight_data_stream.next().await { + let flight_data = flight_data_result + .map_err(|e| arrow_flight::error::FlightError::Tonic(Box::new(e)))?; + + // First message should be the schema + if schema.is_none() { + let message = root_as_message(&flight_data.data_header[..]) + .map_err(|err| arrow_flight::error::FlightError::DecodeError( + format!("Cannot get root as message: {err:?}") + ))?; + + let ipc_schema = message + .header_as_schema() + .ok_or_else(|| arrow_flight::error::FlightError::DecodeError( + "First message should be schema".to_string() + ))?; + + schema = Some(Arc::new(fb_to_schema(ipc_schema))); + continue; + } + + // Extract and decode app_metadata (required) + let metadata = BatchMetadata::decode(&flight_data.app_metadata) + .map_err(|e| arrow_flight::error::FlightError::DecodeError( + format!("Failed to decode batch metadata: {e}") + ))?; + + // Decode the RecordBatch from FlightData using the schema + if !flight_data.data_body.is_empty() { + if let Some(ref schema_ref) = schema { + let batch = arrow_flight::utils::flight_data_to_arrow_batch( + &flight_data, + schema_ref.clone(), + &dictionaries_by_id + )?; + + yield (batch, metadata); + } + } + } + }; + + Ok(stream) + } } diff --git a/crates/phaser-client/src/error.rs b/crates/phaser-client/src/error.rs new file mode 100644 index 0000000..0701ceb --- /dev/null +++ b/crates/phaser-client/src/error.rs @@ -0,0 +1,81 @@ +//! Error types for the bridge client + +use thiserror::Error; + +/// Errors that can occur when using the bridge client +#[derive(Error, Debug)] +pub enum BridgeError { + /// Failed to establish connection to the bridge + #[error("Connection failed: {0}")] + Connection(#[from] tonic::transport::Error), + + /// gRPC-level error from the bridge + #[error("gRPC error: {0}")] + Grpc(#[source] tonic::Status), + + /// Discovery action failed + #[error("Discovery failed: {message}")] + Discovery { message: String }, + + /// Schema-related error + #[error("Schema error: {message}")] + Schema { message: String }, + + /// Error during Flight stream processing + #[error("Stream error: {0}")] + Stream(#[from] arrow_flight::error::FlightError), + + /// Protocol-level error (malformed messages, missing metadata, etc.) + #[error("Protocol error: {message}")] + Protocol { message: String }, + + /// JSON serialization/deserialization error + #[error("Serialization error: {0}")] + Serialization(#[from] serde_json::Error), + + /// Platform not supported (e.g., Unix sockets on Windows) + #[error("Platform not supported: {message}")] + PlatformNotSupported { message: String }, + + /// Invalid URI + #[error("Invalid URI: {0}")] + InvalidUri(#[from] http::uri::InvalidUri), +} + +impl BridgeError { + /// Create a discovery error + pub fn discovery(message: impl Into) -> Self { + Self::Discovery { + message: message.into(), + } + } + + /// Create a schema error + pub fn schema(message: impl Into) -> Self { + Self::Schema { + message: message.into(), + } + } + + /// Create a protocol error + pub fn protocol(message: impl Into) -> Self { + Self::Protocol { + message: message.into(), + } + } + + /// Create a platform not supported error + pub fn platform_not_supported(message: impl Into) -> Self { + Self::PlatformNotSupported { + message: message.into(), + } + } +} + +// Manual impl to avoid conflict with #[from] for tonic::Status +// (tonic::Status is not actually used with #[from] to give us more control) +impl From for BridgeError { + fn from(status: tonic::Status) -> Self { + Self::Grpc(status) + } +} diff --git a/crates/phaser-client/src/lib.rs b/crates/phaser-client/src/lib.rs new file mode 100644 index 0000000..0ba393a --- /dev/null +++ b/crates/phaser-client/src/lib.rs @@ -0,0 +1,62 @@ +//! Flight client for connecting to phaser bridges +//! +//! This crate provides a minimal client for connecting to bridges without +//! pulling in server dependencies. Use this in consumers like flight-streamer. +//! +//! # Example +//! +//! ```ignore +//! use phaser_client::{PhaserClient, BridgeError, GenericQuery}; +//! +//! async fn example() -> Result<(), BridgeError> { +//! let mut client = PhaserClient::connect("http://localhost:50051".into()).await?; +//! +//! // Discover available tables +//! let capabilities = client.discover().await?; +//! println!("Bridge: {} with {} tables", capabilities.name, capabilities.tables.len()); +//! +//! // Query data +//! let query = GenericQuery::historical("blocks", 0, 100); +//! let stream = client.query(query).await?; +//! +//! Ok(()) +//! } +//! ``` + +mod client; +mod error; + +pub use client::PhaserClient; +pub use error::BridgeError; + +// Re-export core types for convenience +pub use phaser_types::{ + // Subscription + BackpressureStrategy, + // Batch metadata + BatchMetadata, + BatchWithRange, + BlockRange, + // Descriptors + BlockchainDescriptor, + BridgeInfo, + Compression, + ControlAction, + DataAvailability, + DataSource, + // Discovery + DiscoveryCapabilities, + FilterDescriptor, + FilterSpec, + GenericQuery, + GenericQueryMode, + QueryMode, + ResponsibilityRange, + StreamPreferences, + StreamType, + SubscriptionInfo, + SubscriptionOptions, + TableDescriptor, + ValidationStage, + ACTION_DESCRIBE, +}; diff --git a/crates/phaser-integration-test/Cargo.toml b/crates/phaser-integration-test/Cargo.toml index 7751cc7..bf540a9 100644 --- a/crates/phaser-integration-test/Cargo.toml +++ b/crates/phaser-integration-test/Cargo.toml @@ -15,7 +15,7 @@ alloy-rlp = { workspace = true } alloy-trie = { workspace = true } typed-arrow = { workspace = true, features = ["views"] } arrow-array = { workspace = true } -parquet = "56.1" +parquet = { workspace = true } tokio = { workspace = true } anyhow = { workspace = true } clap = { version = "4.5", features = ["derive"] } diff --git a/crates/phaser-metrics/Cargo.toml b/crates/phaser-metrics/Cargo.toml index cb8ab2f..d6d4cf1 100644 --- a/crates/phaser-metrics/Cargo.toml +++ b/crates/phaser-metrics/Cargo.toml @@ -4,7 +4,6 @@ version = "0.1.0" edition = "2021" [dependencies] -phaser-bridge = { path = "../phaser-bridge" } prometheus = "0.13" tracing = "0.1" tracing-subscriber = "0.3" diff --git a/crates/phaser-metrics/src/segment_metrics.rs b/crates/phaser-metrics/src/segment_metrics.rs index 9396b4c..8d93c29 100644 --- a/crates/phaser-metrics/src/segment_metrics.rs +++ b/crates/phaser-metrics/src/segment_metrics.rs @@ -105,6 +105,7 @@ pub struct BridgeMetrics { grpc_request_duration_blocks: HistogramVec, grpc_request_duration_transactions: HistogramVec, grpc_request_duration_logs: HistogramVec, + grpc_message_size_bytes: HistogramVec, } impl SegmentMetrics for BridgeMetrics { @@ -397,6 +398,29 @@ impl BridgeMetrics { ] ) .unwrap(), + grpc_message_size_bytes: register_histogram_vec!( + format!("{}_grpc_message_size_bytes", service_name_str), + "Size of gRPC messages received from Erigon in bytes", + &["chain_id", "bridge_name", "data_type"], + // 1KB, 10KB, 100KB, 1MB, 5MB, 10MB, 25MB, 50MB, 100MB, 150MB, 200MB, 256MB, 384MB, 512MB + vec![ + 1024.0, + 10240.0, + 102400.0, + 1048576.0, + 5242880.0, + 10485760.0, + 26214400.0, + 52428800.0, + 104857600.0, + 157286400.0, + 209715200.0, + 268435456.0, + 402653184.0, + 536870912.0 + ] + ) + .unwrap(), } } @@ -459,6 +483,13 @@ impl BridgeMetrics { ]) .observe(duration_millis); } + + /// Record gRPC message size in bytes + pub fn grpc_message_size(&self, data_type: &str, size_bytes: usize) { + self.grpc_message_size_bytes + .with_label_values(&[&self.base.chain_id, &self.base.bridge_name, data_type]) + .observe(size_bytes as f64); + } } /// Worker processing stages diff --git a/crates/phaser-query/Cargo.toml b/crates/phaser-query/Cargo.toml index 461fb2e..0b9aa82 100644 --- a/crates/phaser-query/Cargo.toml +++ b/crates/phaser-query/Cargo.toml @@ -13,7 +13,7 @@ path = "src/bin/phaser-cli.rs" [dependencies] # Local dependencies -phaser-bridge = { workspace = true } +phaser-client = { workspace = true } phaser-metrics = { path = "../phaser-metrics" } erigon-bridge = { workspace = true } evm-common = { workspace = true } @@ -38,6 +38,7 @@ anyhow.workspace = true axum.workspace = true jsonrpsee.workspace = true tonic.workspace = true +tonic-prost = "0.14" prost.workspace = true prost-types.workspace = true num_cpus.workspace = true @@ -49,7 +50,7 @@ futures = "0.3" url = "2.5" bincode = "1.3" clap = { version = "4.5", features = ["derive", "env"] } -parquet = "56.1" +parquet = { workspace = true } serde_yaml = "0.9" uuid = { version = "1.18", features = ["v4"] } tokio-stream = "0.1" @@ -59,7 +60,7 @@ chrono = "0.4.42" scopeguard = "1.2" [build-dependencies] -tonic-build = "0.13" +tonic-prost-build = "0.14" [dev-dependencies] tempfile = "3.10" diff --git a/crates/phaser-query/build.rs b/crates/phaser-query/build.rs index 2c58380..ce030e9 100644 --- a/crates/phaser-query/build.rs +++ b/crates/phaser-query/build.rs @@ -1,6 +1,6 @@ fn main() -> Result<(), Box> { // Build clients for remote services - tonic_build::configure() + tonic_prost_build::configure() .build_server(false) .build_client(true) .out_dir("src/generated") @@ -14,7 +14,7 @@ fn main() -> Result<(), Box> { )?; // Build server and client for admin services - tonic_build::configure() + tonic_prost_build::configure() .build_server(true) .build_client(true) .out_dir("src/generated") diff --git a/crates/phaser-query/src/bin/phaser-query.rs b/crates/phaser-query/src/bin/phaser-query.rs index d9339a5..109494d 100644 --- a/crates/phaser-query/src/bin/phaser-query.rs +++ b/crates/phaser-query/src/bin/phaser-query.rs @@ -58,10 +58,10 @@ async fn main() -> Result<()> { .with( tracing_subscriber::EnvFilter::from_default_env() .add_directive("phaser_query=info".parse()?) - .add_directive("phaser_bridge=info".parse()?) + .add_directive("phaser_client=info".parse()?) .add_directive("erigon_bridge=info".parse()?), ) - .with(tracing_subscriber::fmt::layer()) + .with(tracing_subscriber::fmt::layer().with_ansi(false)) .with(phaser_query::sync::metrics::MetricsLayer::new( "phaser-query", )) diff --git a/crates/phaser-query/src/generated/phaser.admin.rs b/crates/phaser-query/src/generated/phaser.admin.rs index 99ad302..c455c68 100644 --- a/crates/phaser-query/src/generated/phaser.admin.rs +++ b/crates/phaser-query/src/generated/phaser.admin.rs @@ -1,5 +1,5 @@ // This file is @generated by prost-build. -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct SyncRequest { /// Chain ID to sync (must match a configured bridge) #[prost(uint64, tag = "1")] @@ -53,7 +53,7 @@ pub struct GapAnalysis { #[prost(message, repeated, tag = "7")] pub incomplete_details: ::prost::alloc::vec::Vec, } -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct BlockRange { /// Start block (inclusive) #[prost(uint64, tag = "1")] @@ -83,7 +83,7 @@ pub struct IncompleteSegment { #[prost(message, repeated, tag = "7")] pub missing_logs_ranges: ::prost::alloc::vec::Vec, } -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct SyncStatusRequest { /// Job ID to query #[prost(string, tag = "1")] @@ -130,7 +130,7 @@ pub struct SyncStatusResponse { #[prost(message, optional, tag = "13")] pub data_progress: ::core::option::Option, } -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct ListSyncJobsRequest { /// Optional: filter by status #[prost(enumeration = "SyncStatus", optional, tag = "1")] @@ -142,13 +142,13 @@ pub struct ListSyncJobsResponse { #[prost(message, repeated, tag = "1")] pub jobs: ::prost::alloc::vec::Vec, } -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct CancelSyncRequest { /// Job ID to cancel #[prost(string, tag = "1")] pub job_id: ::prost::alloc::string::String, } -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct CancelSyncResponse { /// Whether cancellation was successful #[prost(bool, tag = "1")] @@ -157,7 +157,7 @@ pub struct CancelSyncResponse { #[prost(string, tag = "2")] pub message: ::prost::alloc::string::String, } -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct SyncProgressRequest { /// Job ID to stream progress for #[prost(string, tag = "1")] @@ -191,7 +191,7 @@ pub struct SyncProgressUpdate { #[prost(message, optional, tag = "9")] pub data_progress: ::core::option::Option, } -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct WorkerProgress { /// Worker ID #[prost(uint32, tag = "1")] @@ -208,7 +208,7 @@ pub struct WorkerProgress { #[prost(string, tag = "5")] pub status: ::prost::alloc::string::String, } -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct AnalyzeGapsRequest { /// Chain ID to analyze #[prost(uint64, tag = "1")] @@ -265,7 +265,7 @@ pub struct DataTypeProgress { pub highest_continuous: u64, } /// File and disk statistics -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct FileStatistics { /// Total number of parquet files #[prost(uint32, tag = "1")] @@ -435,7 +435,7 @@ pub mod sync_service_client { format!("Service was not ready: {}", e.into()), ) })?; - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/phaser.admin.SyncService/StartSync", ); @@ -460,7 +460,7 @@ pub mod sync_service_client { format!("Service was not ready: {}", e.into()), ) })?; - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/phaser.admin.SyncService/GetSyncStatus", ); @@ -485,7 +485,7 @@ pub mod sync_service_client { format!("Service was not ready: {}", e.into()), ) })?; - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/phaser.admin.SyncService/ListSyncJobs", ); @@ -510,7 +510,7 @@ pub mod sync_service_client { format!("Service was not ready: {}", e.into()), ) })?; - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/phaser.admin.SyncService/CancelSync", ); @@ -535,7 +535,7 @@ pub mod sync_service_client { format!("Service was not ready: {}", e.into()), ) })?; - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/phaser.admin.SyncService/StreamSyncProgress", ); @@ -562,7 +562,7 @@ pub mod sync_service_client { format!("Service was not ready: {}", e.into()), ) })?; - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/phaser.admin.SyncService/AnalyzeGaps", ); @@ -743,7 +743,7 @@ pub mod sync_service_server { let inner = self.inner.clone(); let fut = async move { let method = StartSyncSvc(inner); - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) .apply_compression_config( accept_compression_encodings, @@ -788,7 +788,7 @@ pub mod sync_service_server { let inner = self.inner.clone(); let fut = async move { let method = GetSyncStatusSvc(inner); - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) .apply_compression_config( accept_compression_encodings, @@ -833,7 +833,7 @@ pub mod sync_service_server { let inner = self.inner.clone(); let fut = async move { let method = ListSyncJobsSvc(inner); - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) .apply_compression_config( accept_compression_encodings, @@ -878,7 +878,7 @@ pub mod sync_service_server { let inner = self.inner.clone(); let fut = async move { let method = CancelSyncSvc(inner); - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) .apply_compression_config( accept_compression_encodings, @@ -925,7 +925,7 @@ pub mod sync_service_server { let inner = self.inner.clone(); let fut = async move { let method = StreamSyncProgressSvc(inner); - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) .apply_compression_config( accept_compression_encodings, @@ -970,7 +970,7 @@ pub mod sync_service_server { let inner = self.inner.clone(); let fut = async move { let method = AnalyzeGapsSvc(inner); - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) .apply_compression_config( accept_compression_encodings, diff --git a/crates/phaser-query/src/generated/remote.rs b/crates/phaser-query/src/generated/remote.rs index c00c25c..0f93b85 100644 --- a/crates/phaser-query/src/generated/remote.rs +++ b/crates/phaser-query/src/generated/remote.rs @@ -1,31 +1,31 @@ // This file is @generated by prost-build. -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct BorTxnLookupRequest { #[prost(message, optional, tag = "1")] pub bor_tx_hash: ::core::option::Option, } -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct BorTxnLookupReply { #[prost(bool, tag = "1")] pub present: bool, #[prost(uint64, tag = "2")] pub block_number: u64, } -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct BorEventsRequest { #[prost(uint64, tag = "1")] pub block_num: u64, #[prost(message, optional, tag = "2")] pub block_hash: ::core::option::Option, } -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct BorEventsReply { #[prost(string, tag = "1")] pub state_receiver_contract_address: ::prost::alloc::string::String, #[prost(bytes = "vec", repeated, tag = "2")] pub event_rlps: ::prost::alloc::vec::Vec<::prost::alloc::vec::Vec>, } -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct BorProducersRequest { #[prost(uint64, tag = "1")] pub block_num: u64, @@ -37,7 +37,7 @@ pub struct BorProducersResponse { #[prost(message, repeated, tag = "2")] pub validators: ::prost::alloc::vec::Vec, } -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct Validator { #[prost(uint64, tag = "1")] pub id: u64, @@ -155,7 +155,7 @@ pub mod bridge_backend_client { format!("Service was not ready: {}", e.into()), ) })?; - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/remote.BridgeBackend/Version", ); @@ -179,7 +179,7 @@ pub mod bridge_backend_client { format!("Service was not ready: {}", e.into()), ) })?; - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/remote.BridgeBackend/BorTxnLookup", ); @@ -200,7 +200,7 @@ pub mod bridge_backend_client { format!("Service was not ready: {}", e.into()), ) })?; - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/remote.BridgeBackend/BorEvents", ); @@ -318,7 +318,7 @@ pub mod heimdall_backend_client { format!("Service was not ready: {}", e.into()), ) })?; - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/remote.HeimdallBackend/Version", ); @@ -342,7 +342,7 @@ pub mod heimdall_backend_client { format!("Service was not ready: {}", e.into()), ) })?; - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/remote.HeimdallBackend/Producers", ); @@ -353,16 +353,16 @@ pub mod heimdall_backend_client { } } } -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct EtherbaseRequest {} -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct EtherbaseReply { #[prost(message, optional, tag = "1")] pub address: ::core::option::Option, } -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct NetVersionRequest {} -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct NetVersionReply { #[prost(uint64, tag = "1")] pub id: u64, @@ -382,7 +382,7 @@ pub struct SyncingReply { } /// Nested message and enum types in `SyncingReply`. pub mod syncing_reply { - #[derive(Clone, PartialEq, ::prost::Message)] + #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct StageProgress { #[prost(string, tag = "1")] pub stage_name: ::prost::alloc::string::String, @@ -390,67 +390,67 @@ pub mod syncing_reply { pub block_number: u64, } } -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct NetPeerCountRequest {} -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct NetPeerCountReply { #[prost(uint64, tag = "1")] pub count: u64, } -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct ProtocolVersionRequest {} -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct ProtocolVersionReply { #[prost(uint64, tag = "1")] pub id: u64, } -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct ClientVersionRequest {} -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct ClientVersionReply { #[prost(string, tag = "1")] pub node_name: ::prost::alloc::string::String, } -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct CanonicalHashRequest { #[prost(uint64, tag = "1")] pub block_number: u64, } -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct CanonicalHashReply { #[prost(message, optional, tag = "1")] pub hash: ::core::option::Option, } -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct HeaderNumberRequest { #[prost(message, optional, tag = "1")] pub hash: ::core::option::Option, } -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct HeaderNumberReply { #[prost(uint64, optional, tag = "1")] pub number: ::core::option::Option, } -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct CanonicalBodyForStorageRequest { #[prost(uint64, tag = "1")] pub block_number: u64, } -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct CanonicalBodyForStorageReply { #[prost(bytes = "vec", tag = "1")] pub body: ::prost::alloc::vec::Vec, } -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct SubscribeRequest { #[prost(enumeration = "Event", tag = "1")] pub r#type: i32, } -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct SubscribeReply { #[prost(enumeration = "Event", tag = "1")] pub r#type: i32, - /// serialized data + /// serialized data #[prost(bytes = "vec", tag = "2")] pub data: ::prost::alloc::vec::Vec, } @@ -486,43 +486,43 @@ pub struct SubscribeLogsReply { #[prost(bool, tag = "9")] pub removed: bool, } -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct BlockRequest { #[prost(uint64, tag = "2")] pub block_height: u64, #[prost(message, optional, tag = "3")] pub block_hash: ::core::option::Option, } -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct BlockReply { #[prost(bytes = "vec", tag = "1")] pub block_rlp: ::prost::alloc::vec::Vec, #[prost(bytes = "vec", tag = "2")] pub senders: ::prost::alloc::vec::Vec, } -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct TxnLookupRequest { #[prost(message, optional, tag = "1")] pub txn_hash: ::core::option::Option, } -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct TxnLookupReply { #[prost(uint64, tag = "1")] pub block_number: u64, #[prost(uint64, tag = "2")] pub tx_number: u64, } -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct NodesInfoRequest { #[prost(uint32, tag = "1")] pub limit: u32, } -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct AddPeerRequest { #[prost(string, tag = "1")] pub url: ::prost::alloc::string::String, } -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct RemovePeerRequest { #[prost(string, tag = "1")] pub url: ::prost::alloc::string::String, @@ -537,17 +537,17 @@ pub struct PeersReply { #[prost(message, repeated, tag = "1")] pub peers: ::prost::alloc::vec::Vec, } -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct AddPeerReply { #[prost(bool, tag = "1")] pub success: bool, } -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct RemovePeerReply { #[prost(bool, tag = "1")] pub success: bool, } -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct PendingBlockReply { #[prost(bytes = "vec", tag = "1")] pub block_rlp: ::prost::alloc::vec::Vec, @@ -557,7 +557,7 @@ pub struct EngineGetPayloadBodiesByHashV1Request { #[prost(message, repeated, tag = "1")] pub hashes: ::prost::alloc::vec::Vec, } -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct EngineGetPayloadBodiesByRangeV1Request { #[prost(uint64, tag = "1")] pub start: u64, @@ -569,24 +569,24 @@ pub struct AaValidationRequest { #[prost(message, optional, tag = "1")] pub tx: ::core::option::Option, } -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct AaValidationReply { #[prost(bool, tag = "1")] pub valid: bool, } -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct BlockForTxNumRequest { #[prost(uint64, tag = "1")] pub txnum: u64, } -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct BlockForTxNumResponse { #[prost(uint64, tag = "1")] pub block_number: u64, #[prost(bool, tag = "2")] pub present: bool, } -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct MinimumBlockAvailableReply { #[prost(uint64, tag = "1")] pub block_num: u64, @@ -729,7 +729,7 @@ pub mod ethbackend_client { format!("Service was not ready: {}", e.into()), ) })?; - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/remote.ETHBACKEND/Etherbase", ); @@ -753,7 +753,7 @@ pub mod ethbackend_client { format!("Service was not ready: {}", e.into()), ) })?; - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/remote.ETHBACKEND/NetVersion", ); @@ -777,7 +777,7 @@ pub mod ethbackend_client { format!("Service was not ready: {}", e.into()), ) })?; - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/remote.ETHBACKEND/NetPeerCount", ); @@ -802,7 +802,7 @@ pub mod ethbackend_client { format!("Service was not ready: {}", e.into()), ) })?; - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/remote.ETHBACKEND/Version", ); @@ -823,7 +823,7 @@ pub mod ethbackend_client { format!("Service was not ready: {}", e.into()), ) })?; - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/remote.ETHBACKEND/Syncing", ); @@ -847,7 +847,7 @@ pub mod ethbackend_client { format!("Service was not ready: {}", e.into()), ) })?; - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/remote.ETHBACKEND/ProtocolVersion", ); @@ -872,7 +872,7 @@ pub mod ethbackend_client { format!("Service was not ready: {}", e.into()), ) })?; - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/remote.ETHBACKEND/ClientVersion", ); @@ -896,7 +896,7 @@ pub mod ethbackend_client { format!("Service was not ready: {}", e.into()), ) })?; - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/remote.ETHBACKEND/Subscribe", ); @@ -921,7 +921,7 @@ pub mod ethbackend_client { format!("Service was not ready: {}", e.into()), ) })?; - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/remote.ETHBACKEND/SubscribeLogs", ); @@ -945,7 +945,7 @@ pub mod ethbackend_client { format!("Service was not ready: {}", e.into()), ) })?; - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static("/remote.ETHBACKEND/Block"); let mut req = request.into_request(); req.extensions_mut().insert(GrpcMethod::new("remote.ETHBACKEND", "Block")); @@ -967,7 +967,7 @@ pub mod ethbackend_client { format!("Service was not ready: {}", e.into()), ) })?; - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/remote.ETHBACKEND/CanonicalBodyForStorage", ); @@ -992,7 +992,7 @@ pub mod ethbackend_client { format!("Service was not ready: {}", e.into()), ) })?; - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/remote.ETHBACKEND/CanonicalHash", ); @@ -1017,7 +1017,7 @@ pub mod ethbackend_client { format!("Service was not ready: {}", e.into()), ) })?; - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/remote.ETHBACKEND/HeaderNumber", ); @@ -1040,7 +1040,7 @@ pub mod ethbackend_client { format!("Service was not ready: {}", e.into()), ) })?; - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/remote.ETHBACKEND/TxnLookup", ); @@ -1062,7 +1062,7 @@ pub mod ethbackend_client { format!("Service was not ready: {}", e.into()), ) })?; - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/remote.ETHBACKEND/NodeInfo", ); @@ -1084,7 +1084,7 @@ pub mod ethbackend_client { format!("Service was not ready: {}", e.into()), ) })?; - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static("/remote.ETHBACKEND/Peers"); let mut req = request.into_request(); req.extensions_mut().insert(GrpcMethod::new("remote.ETHBACKEND", "Peers")); @@ -1102,7 +1102,7 @@ pub mod ethbackend_client { format!("Service was not ready: {}", e.into()), ) })?; - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/remote.ETHBACKEND/AddPeer", ); @@ -1125,7 +1125,7 @@ pub mod ethbackend_client { format!("Service was not ready: {}", e.into()), ) })?; - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/remote.ETHBACKEND/RemovePeer", ); @@ -1150,7 +1150,7 @@ pub mod ethbackend_client { format!("Service was not ready: {}", e.into()), ) })?; - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/remote.ETHBACKEND/PendingBlock", ); @@ -1174,7 +1174,7 @@ pub mod ethbackend_client { format!("Service was not ready: {}", e.into()), ) })?; - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/remote.ETHBACKEND/BorTxnLookup", ); @@ -1195,7 +1195,7 @@ pub mod ethbackend_client { format!("Service was not ready: {}", e.into()), ) })?; - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/remote.ETHBACKEND/BorEvents", ); @@ -1219,7 +1219,7 @@ pub mod ethbackend_client { format!("Service was not ready: {}", e.into()), ) })?; - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/remote.ETHBACKEND/AAValidation", ); @@ -1243,7 +1243,7 @@ pub mod ethbackend_client { format!("Service was not ready: {}", e.into()), ) })?; - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/remote.ETHBACKEND/BlockForTxNum", ); @@ -1267,7 +1267,7 @@ pub mod ethbackend_client { format!("Service was not ready: {}", e.into()), ) })?; - let codec = tonic::codec::ProstCodec::default(); + let codec = tonic_prost::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( "/remote.ETHBACKEND/MinimumBlockAvailable", ); diff --git a/crates/phaser-query/src/generated/types.rs b/crates/phaser-query/src/generated/types.rs index 7ba4e98..f7eb908 100644 --- a/crates/phaser-query/src/generated/types.rs +++ b/crates/phaser-query/src/generated/types.rs @@ -1,40 +1,40 @@ // This file is @generated by prost-build. -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct H128 { #[prost(uint64, tag = "1")] pub hi: u64, #[prost(uint64, tag = "2")] pub lo: u64, } -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct H160 { #[prost(message, optional, tag = "1")] pub hi: ::core::option::Option, #[prost(uint32, tag = "2")] pub lo: u32, } -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct H256 { #[prost(message, optional, tag = "1")] pub hi: ::core::option::Option, #[prost(message, optional, tag = "2")] pub lo: ::core::option::Option, } -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct H512 { #[prost(message, optional, tag = "1")] pub hi: ::core::option::Option, #[prost(message, optional, tag = "2")] pub lo: ::core::option::Option, } -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct H1024 { #[prost(message, optional, tag = "1")] pub hi: ::core::option::Option, #[prost(message, optional, tag = "2")] pub lo: ::core::option::Option, } -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct H2048 { #[prost(message, optional, tag = "1")] pub hi: ::core::option::Option, @@ -42,7 +42,7 @@ pub struct H2048 { pub lo: ::core::option::Option, } /// Reply message containing the current service version on the service side -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct VersionReply { #[prost(uint32, tag = "1")] pub major: u32, @@ -51,7 +51,8 @@ pub struct VersionReply { #[prost(uint32, tag = "3")] pub patch: u32, } -/// ------------------------------------------------------------------------ +/// --- +/// /// Engine API types /// See #[derive(Clone, PartialEq, ::prost::Message)] @@ -94,7 +95,7 @@ pub struct ExecutionPayload { #[prost(uint64, optional, tag = "18")] pub excess_blob_gas: ::core::option::Option, } -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct Withdrawal { #[prost(uint64, tag = "1")] pub index: u64, @@ -105,7 +106,7 @@ pub struct Withdrawal { #[prost(uint64, tag = "4")] pub amount: u64, } -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct BlobsBundle { /// TODO(eip-4844): define a protobuf message for type KZGCommitment #[prost(bytes = "vec", repeated, tag = "1")] @@ -116,19 +117,19 @@ pub struct BlobsBundle { #[prost(bytes = "vec", repeated, tag = "3")] pub proofs: ::prost::alloc::vec::Vec<::prost::alloc::vec::Vec>, } -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct RequestsBundle { #[prost(bytes = "vec", repeated, tag = "1")] pub requests: ::prost::alloc::vec::Vec<::prost::alloc::vec::Vec>, } -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct NodeInfoPorts { #[prost(uint32, tag = "1")] pub discovery: u32, #[prost(uint32, tag = "2")] pub listener: u32, } -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct NodeInfoReply { #[prost(string, tag = "1")] pub id: ::prost::alloc::string::String, @@ -145,7 +146,7 @@ pub struct NodeInfoReply { #[prost(bytes = "vec", tag = "7")] pub protocols: ::prost::alloc::vec::Vec, } -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct PeerInfo { #[prost(string, tag = "1")] pub id: ::prost::alloc::string::String, @@ -214,7 +215,7 @@ pub struct AccountAbstractionTransaction { #[prost(message, repeated, tag = "18")] pub authorizations: ::prost::alloc::vec::Vec, } -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct Authorization { #[prost(uint64, tag = "1")] pub chain_id: u64, diff --git a/crates/phaser-query/src/lib.rs b/crates/phaser-query/src/lib.rs index a3b91a5..785294c 100644 --- a/crates/phaser-query/src/lib.rs +++ b/crates/phaser-query/src/lib.rs @@ -215,7 +215,7 @@ pub struct PhaserConfig { #[serde(default)] pub parquet: Option, #[serde(default)] - pub validation_stage: phaser_bridge::ValidationStage, // Validation stage for sync (none, ingestion, conversion, both) + pub validation_stage: phaser_client::ValidationStage, // Validation stage for sync (none, ingestion, conversion, both) } fn default_segment_size() -> u64 { diff --git a/crates/phaser-query/src/streaming_with_writer.rs b/crates/phaser-query/src/streaming_with_writer.rs index de1be7c..11f140b 100644 --- a/crates/phaser-query/src/streaming_with_writer.rs +++ b/crates/phaser-query/src/streaming_with_writer.rs @@ -4,10 +4,7 @@ use anyhow::Result; use arrow::array::UInt64Array; use arrow::record_batch::RecordBatch; use futures::StreamExt; -use phaser_bridge::{ - descriptors::{BlockchainDescriptor, StreamType}, - FlightBridgeClient, -}; +use phaser_client::{GenericQuery, PhaserClient, StreamType}; use rocksdb::DB; use std::path::PathBuf; use std::sync::Arc; @@ -16,7 +13,7 @@ use tracing::{error, info}; /// Enhanced streaming service with parquet writing capabilities pub struct StreamingServiceWithWriter { - bridges: Vec, + bridges: Vec, data_dir: PathBuf, max_file_size_mb: u64, segment_size: u64, @@ -40,7 +37,7 @@ impl StreamingServiceWithWriter { for endpoint in bridge_endpoints { info!("Connecting to bridge at {}", endpoint); - let client = FlightBridgeClient::connect(endpoint).await?; + let client = PhaserClient::connect(endpoint).await?; bridges.push(client); } @@ -101,7 +98,7 @@ impl StreamingServiceWithWriter { stream_type: StreamType, mut stream: impl StreamExt< Item = Result< - (RecordBatch, phaser_bridge::BatchMetadata), + (RecordBatch, phaser_client::BatchMetadata), arrow_flight::error::FlightError, >, > + Send @@ -231,9 +228,9 @@ impl StreamingServiceWithWriter { } // Subscribe to blocks with metadata - let blocks_descriptor = BlockchainDescriptor::live(StreamType::Blocks); + let blocks_query = GenericQuery::live("blocks"); info!("Subscribing to blocks from bridge"); - let blocks_stream = bridge.subscribe_with_metadata(&blocks_descriptor).await?; + let blocks_stream = bridge.query_with_metadata(blocks_query).await?; Self::spawn_stream_processor( StreamType::Blocks, Box::pin(blocks_stream), @@ -244,9 +241,9 @@ impl StreamingServiceWithWriter { ); // Subscribe to transactions with metadata - let txs_descriptor = BlockchainDescriptor::live(StreamType::Transactions); + let txs_query = GenericQuery::live("transactions"); info!("Subscribing to transactions from bridge"); - let txs_stream = bridge.subscribe_with_metadata(&txs_descriptor).await?; + let txs_stream = bridge.query_with_metadata(txs_query).await?; Self::spawn_stream_processor( StreamType::Transactions, Box::pin(txs_stream), @@ -257,9 +254,10 @@ impl StreamingServiceWithWriter { ); // Subscribe to logs with metadata - let logs_descriptor = BlockchainDescriptor::live(StreamType::Logs).with_traces(true); + let logs_query = + GenericQuery::live("logs").with_filter("enable_traces", serde_json::json!(true)); info!("Subscribing to logs from bridge"); - let logs_stream = bridge.subscribe_with_metadata(&logs_descriptor).await?; + let logs_stream = bridge.query_with_metadata(logs_query).await?; Self::spawn_stream_processor( StreamType::Logs, Box::pin(logs_stream), @@ -283,16 +281,16 @@ impl StreamingServiceWithWriter { )?; for bridge in &mut self.bridges { - let descriptor = - BlockchainDescriptor::historical(StreamType::Blocks, start_block, end_block); + let query = GenericQuery::historical("blocks", start_block, end_block); info!( "Fetching historical blocks {} to {}", start_block, end_block ); - let batches = bridge.stream_data(&descriptor).await?; - for batch in batches { + let mut stream = bridge.query(query).await?; + while let Some(batch_result) = stream.next().await { + let batch = batch_result?; info!("Processing historical batch with {} rows", batch.num_rows()); writer.write_batch(batch).await?; } @@ -356,10 +354,10 @@ impl StreamingServiceWithWriter { } // Subscribe to trie stream with metadata - let trie_descriptor = BlockchainDescriptor::live(StreamType::Trie); + let trie_query = GenericQuery::live("trie"); info!("Subscribing to trie data from bridge"); - match bridge.subscribe_with_metadata(&trie_descriptor).await { + match bridge.query_with_metadata(trie_query).await { Ok(trie_stream) => { info!("Successfully subscribed to trie stream"); Self::spawn_stream_processor( diff --git a/crates/phaser-query/src/sync/data_scanner.rs b/crates/phaser-query/src/sync/data_scanner.rs index dd47cd1..d31db09 100644 --- a/crates/phaser-query/src/sync/data_scanner.rs +++ b/crates/phaser-query/src/sync/data_scanner.rs @@ -928,7 +928,7 @@ impl DataScanner { missing_parts.push(format!("logs ({} ranges)", missing_logs.len())); } - info!( + debug!( "Segment {} (blocks {}-{}) incomplete - missing: {}", segment_num, segment_start, diff --git a/crates/phaser-query/src/sync/error.rs b/crates/phaser-query/src/sync/error.rs index ad3dc82..7170f31 100644 --- a/crates/phaser-query/src/sync/error.rs +++ b/crates/phaser-query/src/sync/error.rs @@ -1,5 +1,65 @@ use std::fmt; +/// Categorize an error message into an ErrorCategory +/// This is the single source of truth for error categorization based on message content +fn categorize_error_message(err_lower: &str) -> ErrorCategory { + // Connection errors + if err_lower.contains("connection") || err_lower.contains("connect") { + return ErrorCategory::Connection; + } + + // Timeout errors + if err_lower.contains("timeout") || err_lower.contains("timed out") { + return ErrorCategory::Timeout; + } + + // Cancelled errors + if err_lower.contains("cancelled") || err_lower.contains("canceled") { + return ErrorCategory::Cancelled; + } + + // Block/data not found - non-transient for historical sync + if err_lower.contains("header not found") || err_lower.contains("block not found") { + return ErrorCategory::NoData; + } + + // No data / empty responses + if err_lower.contains("no data") || err_lower.contains("empty") { + return ErrorCategory::NoData; + } + + // Stuck worker errors + if err_lower.contains("failed to make progress") { + return ErrorCategory::StuckWorker; + } + + // Protocol errors (bridge returned zero batches, etc.) + if err_lower.contains("zero batches") || err_lower.contains("protocol error") { + return ErrorCategory::ProtocolError; + } + + // Disk I/O errors - check BEFORE validation since some validation errors might mention "file" + // Include parquet/arrow write errors which are typically disk or serialization issues + if err_lower.contains("io error") + || err_lower.contains("disk") + || err_lower.contains("parquet") + || err_lower.contains("arrow") + || err_lower.contains("write error") + || err_lower.contains("failed to write") + || (err_lower.contains("file") && !err_lower.contains("validation")) + { + return ErrorCategory::DiskIo; + } + + // Validation errors + if err_lower.contains("validation") || err_lower.contains("invalid") { + return ErrorCategory::Validation; + } + + // Unknown - catch-all for unrecognized errors + ErrorCategory::Unknown +} + /// Type of data being synced when error occurred #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum DataType { @@ -224,22 +284,7 @@ impl SyncError { let err_lower = message.to_lowercase(); // Categorize based on error message - let category = if err_lower.contains("connection") || err_lower.contains("connect") { - ErrorCategory::Connection - } else if err_lower.contains("timeout") || err_lower.contains("timed out") { - ErrorCategory::Timeout - } else if err_lower.contains("header not found") || err_lower.contains("block not found") { - // Block doesn't exist - likely beyond chain tip, not a transient error for historical sync - ErrorCategory::NoData - } else if err_lower.contains("validation") || err_lower.contains("invalid") { - ErrorCategory::Validation - } else if err_lower.contains("io error") || err_lower.contains("file") { - ErrorCategory::DiskIo - } else if err_lower.contains("cancelled") { - ErrorCategory::Cancelled - } else { - ErrorCategory::Unknown - }; + let category = categorize_error_message(&err_lower); Self { data_type, @@ -266,22 +311,7 @@ impl SyncError { let err_lower = err_str.to_lowercase(); // Categorize based on error message - let category = if err_lower.contains("connection") || err_lower.contains("connect") { - ErrorCategory::Connection - } else if err_lower.contains("timeout") || err_lower.contains("timed out") { - ErrorCategory::Timeout - } else if err_lower.contains("header not found") || err_lower.contains("block not found") { - // Block doesn't exist - likely beyond chain tip, not a transient error for historical sync - ErrorCategory::NoData - } else if err_lower.contains("validation") || err_lower.contains("invalid") { - ErrorCategory::Validation - } else if err_lower.contains("io error") || err_lower.contains("file") { - ErrorCategory::DiskIo - } else if err_lower.contains("cancelled") { - ErrorCategory::Cancelled - } else { - ErrorCategory::Unknown - }; + let category = categorize_error_message(&err_lower); let context_str = context.into(); let message = format!("{context_str}: {err_str}"); @@ -308,22 +338,7 @@ impl SyncError { let err_lower = err_str.to_lowercase(); // Categorize based on error message - let category = if err_lower.contains("connection") || err_lower.contains("connect") { - ErrorCategory::Connection - } else if err_lower.contains("timeout") || err_lower.contains("timed out") { - ErrorCategory::Timeout - } else if err_lower.contains("header not found") || err_lower.contains("block not found") { - // Block doesn't exist - likely beyond chain tip, not a transient error for historical sync - ErrorCategory::NoData - } else if err_lower.contains("validation") || err_lower.contains("invalid") { - ErrorCategory::Validation - } else if err_lower.contains("io error") || err_lower.contains("file") { - ErrorCategory::DiskIo - } else if err_lower.contains("cancelled") { - ErrorCategory::Cancelled - } else { - ErrorCategory::Unknown - }; + let category = categorize_error_message(&err_lower); let context_str = context.into(); let message = format!("{context_str}: {err_str}"); @@ -356,26 +371,8 @@ impl From for SyncError { DataType::Unknown }; - // Categorize error - let category = if err_lower.contains("connection") || err_lower.contains("connect") { - ErrorCategory::Connection - } else if err_lower.contains("timeout") || err_lower.contains("timed out") { - ErrorCategory::Timeout - } else if err_lower.contains("no data") || err_lower.contains("empty") { - ErrorCategory::NoData - } else if err_lower.contains("failed to make progress") { - ErrorCategory::StuckWorker - } else if err_lower.contains("validation") || err_lower.contains("invalid") { - ErrorCategory::Validation - } else if err_lower.contains("io error") || err_lower.contains("file") { - ErrorCategory::DiskIo - } else if err_lower.contains("cancelled") { - ErrorCategory::Cancelled - } else if err_lower.contains("zero batches") || err_lower.contains("protocol error") { - ErrorCategory::ProtocolError - } else { - ErrorCategory::Unknown - }; + // Categorize error using common function + let category = categorize_error_message(&err_lower); Self { data_type, diff --git a/crates/phaser-query/src/sync/service.rs b/crates/phaser-query/src/sync/service.rs index 9157c30..44d1d9b 100644 --- a/crates/phaser-query/src/sync/service.rs +++ b/crates/phaser-query/src/sync/service.rs @@ -6,7 +6,7 @@ use crate::sync::worker::{ProgressTracker, SyncWorker, SyncWorkerConfig}; use crate::PhaserConfig; use anyhow::Result; use core_executor::ThreadPoolExecutor; -use phaser_bridge::FlightBridgeClient; +use phaser_client::PhaserClient; use phaser_metrics::SegmentMetrics; use std::collections::{HashMap, VecDeque}; use std::sync::atomic::{AtomicBool, Ordering}; @@ -428,12 +428,13 @@ impl SyncServer { metrics.sync_errors(error_category, data_type); // Check if error is retryable + // Most errors should be retried - only validation and stuck workers are truly non-retryable use crate::sync::error::ErrorCategory; - let is_retryable = matches!( + let is_retryable = !matches!( sync_err.category, - ErrorCategory::Connection - | ErrorCategory::Timeout - | ErrorCategory::Cancelled + ErrorCategory::Validation + | ErrorCategory::StuckWorker + | ErrorCategory::NoData ); if !is_retryable { @@ -623,7 +624,7 @@ impl SyncService for SyncServer { // Validate that to_block doesn't exceed chain tip (only for historical syncs without live streaming) if historical_boundary.is_none() { // Connect to bridge to check chain tip - let mut client = FlightBridgeClient::connect(bridge.endpoint.clone()) + let mut client = PhaserClient::connect(bridge.endpoint.clone()) .await .map_err(|e| Status::unavailable(format!("Failed to connect to bridge: {e}")))?; diff --git a/crates/phaser-query/src/sync/worker.rs b/crates/phaser-query/src/sync/worker.rs index c1890b4..a909df2 100644 --- a/crates/phaser-query/src/sync/worker.rs +++ b/crates/phaser-query/src/sync/worker.rs @@ -6,8 +6,7 @@ use arrow::array as arrow_array; use evm_common::proof::{generate_transaction_proof, MerkleProofRecord}; use evm_common::transaction::TransactionRecord; use futures::StreamExt; -use phaser_bridge::client::FlightBridgeClient; -use phaser_bridge::descriptors::{BlockchainDescriptor, StreamType, ValidationStage}; +use phaser_client::{GenericQuery, PhaserClient, ValidationStage}; use phaser_metrics::SegmentMetrics; use std::collections::HashMap; use std::path::PathBuf; @@ -88,7 +87,7 @@ pub struct SyncWorker { _batch_size: u32, parquet_config: Option, progress_tracker: Option, - validation_stage: ValidationStage, + _validation_stage: phaser_client::ValidationStage, segment_work: crate::sync::data_scanner::SegmentWork, // Pre-computed missing ranges current_progress: Arc>, // Real-time progress state logs_semaphore: Arc, @@ -147,7 +146,7 @@ impl SyncWorker { _batch_size: config.batch_size, parquet_config: config.parquet_config, progress_tracker: None, - validation_stage: config.validation_stage, + _validation_stage: config.validation_stage, segment_work, current_progress, logs_semaphore: config.logs_semaphore, @@ -226,7 +225,7 @@ impl SyncWorker { } pub async fn run(&mut self) -> Result<(), SyncError> { - info!( + debug!( "Worker {} starting sync of blocks {}-{} from {}", self.worker_id, self.from_block, self.to_block, self.bridge_endpoint ); @@ -236,7 +235,7 @@ impl SyncWorker { let missing_txs = self.segment_work.missing_transactions.clone(); let missing_logs = self.segment_work.missing_logs.clone(); - info!( + debug!( "Worker {} segment work: blocks={} ranges, txs={} ranges, logs={} ranges", self.worker_id, missing_blocks.len(), @@ -282,7 +281,7 @@ impl SyncWorker { return Err(multi_err.into()); } - info!("Worker {} completed sync successfully", self.worker_id); + debug!("Worker {} completed sync successfully", self.worker_id); Ok(()) } @@ -296,7 +295,7 @@ impl SyncWorker { } // Connect to bridge - let mut client = FlightBridgeClient::connect(self.bridge_endpoint.clone()) + let mut client = PhaserClient::connect(self.bridge_endpoint.clone()) .await .map_err(|e| { SyncError::from_anyhow_with_context( @@ -304,12 +303,12 @@ impl SyncWorker { self.from_block, self.to_block, "Failed to connect to bridge", - e, + e.into(), ) })?; for range in &missing_blocks { - info!( + debug!( "Worker {} syncing blocks {}-{}", self.worker_id, range.start, range.end ); @@ -330,7 +329,7 @@ impl SyncWorker { } // Connect to bridge - let mut client = FlightBridgeClient::connect(self.bridge_endpoint.clone()) + let mut client = PhaserClient::connect(self.bridge_endpoint.clone()) .await .map_err(|e| { SyncError::from_anyhow_with_context( @@ -338,12 +337,12 @@ impl SyncWorker { self.from_block, self.to_block, "Failed to connect to bridge", - e, + e.into(), ) })?; for range in &missing_txs { - info!( + debug!( "Worker {} syncing transactions {}-{}", self.worker_id, range.start, range.end ); @@ -365,7 +364,7 @@ impl SyncWorker { // Acquire 1 permit for this entire segment // This limits how many segments can be processing logs concurrently - info!( + debug!( "Worker {} acquiring log semaphore permit for segment (blocks {}-{})", self.worker_id, self.from_block, self.to_block ); @@ -385,14 +384,14 @@ impl SyncWorker { ) })?; - info!( + debug!( "Worker {} acquired permit, starting log sync for {} ranges", self.worker_id, missing_logs.len() ); // Connect to bridge - let mut client = FlightBridgeClient::connect(self.bridge_endpoint.clone()) + let mut client = PhaserClient::connect(self.bridge_endpoint.clone()) .await .map_err(|e| { SyncError::from_anyhow_with_context( @@ -400,13 +399,13 @@ impl SyncWorker { self.from_block, self.to_block, "Failed to connect to bridge", - e, + e.into(), ) })?; // Process all ranges sequentially while holding all permits for range in &missing_logs { - info!( + debug!( "Worker {} syncing logs {}-{}", self.worker_id, range.start, range.end ); @@ -419,7 +418,7 @@ impl SyncWorker { async fn sync_blocks_range( &self, - client: &mut FlightBridgeClient, + client: &mut PhaserClient, from_block: u64, to_block: u64, ) -> Result<(), SyncError> { @@ -500,7 +499,7 @@ impl SyncWorker { if resume_from > to_block { // We actually completed, the error was after all data - info!( + debug!( "Worker {} completed blocks {}-{} before stream error", self.worker_id, from_block, to_block ); @@ -535,35 +534,25 @@ impl SyncWorker { async fn try_sync_blocks_stream( &self, - client: &mut FlightBridgeClient, + client: &mut PhaserClient, from_block: u64, to_block: u64, writer: &mut ParquetWriter, original_from: u64, ) -> Result<(), SyncError> { - // Create historical query descriptor with large message preferences - use phaser_bridge::descriptors::StreamPreferences; - let preferences = StreamPreferences { - max_message_bytes: 32 * 1024 * 1024, // 32MB for historical sync - compression: phaser_bridge::descriptors::Compression::None, - batch_size_hint: 100, - }; - let descriptor = BlockchainDescriptor::historical(StreamType::Blocks, from_block, to_block) - .with_preferences(preferences); + // Create historical query using GenericQuery + let query = GenericQuery::historical("blocks", from_block, to_block); // Subscribe to the block stream with metadata (returns RecordBatch + responsibility range) - let stream = client - .subscribe_with_metadata(&descriptor) - .await - .map_err(|e| { - SyncError::from_anyhow_with_context( - DataType::Blocks, - from_block, - to_block, - "Failed to subscribe to block stream", - e, - ) - })?; + let stream = client.query_with_metadata(query).await.map_err(|e| { + SyncError::from_anyhow_with_context( + DataType::Blocks, + from_block, + to_block, + "Failed to subscribe to block stream", + e.into(), + ) + })?; let mut stream = Box::pin(stream); let mut batches_processed = 0u64; @@ -675,7 +664,7 @@ impl SyncWorker { // This is valid when a block range legitimately has no blocks (shouldn't happen but handle gracefully). // Arrow Flight doesn't transmit 0-row RecordBatches, so receiving 0 batches // means the range was processed successfully but contains no data. - info!( + debug!( "Worker {} received ZERO batches for blocks {}-{}. \ Range processed successfully with no blocks.", self.worker_id, from_block, to_block @@ -688,7 +677,7 @@ impl SyncWorker { async fn sync_transactions_range( &self, - client: &mut FlightBridgeClient, + client: &mut PhaserClient, from_block: u64, to_block: u64, ) -> Result<(), SyncError> { @@ -723,7 +712,7 @@ impl SyncWorker { .unwrap_or(false); let mut proof_writer = if generate_proofs { - info!( + debug!( "Worker {} will generate merkle proofs for transactions", self.worker_id ); @@ -808,7 +797,7 @@ impl SyncWorker { if resume_from > to_block { // We actually completed, the error was after all data - info!( + debug!( "Worker {} completed transactions {}-{} before stream error", self.worker_id, from_block, to_block ); @@ -843,37 +832,27 @@ impl SyncWorker { async fn try_sync_transactions_stream( &self, - client: &mut FlightBridgeClient, + client: &mut PhaserClient, from_block: u64, to_block: u64, writer: &mut ParquetWriter, proof_writer: &mut Option, _original_from: u64, ) -> Result<(), SyncError> { - // Create historical query descriptor for transactions with configured validation stage - use phaser_bridge::descriptors::StreamPreferences; - let preferences = StreamPreferences { - max_message_bytes: 32 * 1024 * 1024, // 32MB for historical sync - compression: phaser_bridge::descriptors::Compression::None, - batch_size_hint: 100, - }; - let descriptor = - BlockchainDescriptor::historical(StreamType::Transactions, from_block, to_block) - .with_validation(self.validation_stage) - .with_preferences(preferences); + // Create historical query using GenericQuery + let query = GenericQuery::historical("transactions", from_block, to_block); - info!( - "Worker {} requesting transactions for blocks {}-{} (segment {}, validation: {:?})", + debug!( + "Worker {} requesting transactions for blocks {}-{} (segment {})", self.worker_id, from_block, to_block, from_block / 500000, - self.validation_stage ); // Subscribe to the transaction stream with metadata (returns RecordBatch + responsibility range) let stream = client - .subscribe_with_metadata(&descriptor) + .query_with_metadata(query) .await .context("Failed to subscribe to transaction stream")?; let mut stream = Box::pin(stream); @@ -937,7 +916,7 @@ impl SyncWorker { } // Log what we received from bridge - info!( + debug!( "PHASER RECEIVED: Worker {} transactions batch {}, blocks {}-{} ({} rows)", self.worker_id, batches_processed + 1, @@ -954,10 +933,11 @@ impl SyncWorker { if let Some(ref mut proof_w) = proof_writer { if let Ok(proof_batch) = self.generate_proofs_for_batch(&batch) { proof_w.write_batch(proof_batch).await.map_err(|e| { - anyhow::anyhow!( - "Failed to write proof batch. Write error: {:?}. Error chain: {}", - e, - e.to_string() + SyncError::disk_io_error( + DataType::Transactions, + from_block, + to_block, + format!("Failed to write proof batch: {e}"), ) })?; } else { @@ -970,10 +950,11 @@ impl SyncWorker { // Write Arrow RecordBatch directly to parquet and get actual bytes written let batch_bytes = writer.write_batch(batch).await.map_err(|e| { - anyhow::anyhow!( - "Failed to write transaction batch. Write error: {:?}. Error chain: {}", - e, - e.to_string() + SyncError::disk_io_error( + DataType::Transactions, + from_block, + to_block, + format!("Failed to write transaction batch: {e}"), ) })?; @@ -983,7 +964,7 @@ impl SyncWorker { batches_processed += 1; } - info!( + debug!( "Worker {} received {} batches for transactions {}-{} (first_block: {:?}, last_block: {:?})", self.worker_id, batches_processed, @@ -1044,7 +1025,7 @@ impl SyncWorker { // This is valid when a block range legitimately has no transactions. // Arrow Flight doesn't transmit 0-row RecordBatches, so receiving 0 batches // means the range was processed successfully but contains no data. - info!( + debug!( "Worker {} received ZERO batches for transactions {}-{}. \ Range processed successfully with no transactions.", self.worker_id, from_block, to_block @@ -1114,7 +1095,7 @@ impl SyncWorker { async fn sync_logs_range( &self, - client: &mut FlightBridgeClient, + client: &mut PhaserClient, from_block: u64, to_block: u64, ) -> Result<(), SyncError> { @@ -1195,7 +1176,7 @@ impl SyncWorker { if resume_from > to_block { // We actually completed, the error was after all data - info!( + debug!( "Worker {} completed logs {}-{} before stream error", self.worker_id, from_block, to_block ); @@ -1230,37 +1211,27 @@ impl SyncWorker { async fn try_sync_logs_stream( &self, - client: &mut FlightBridgeClient, + client: &mut PhaserClient, from_block: u64, to_block: u64, writer: &mut ParquetWriter, _original_from: u64, ) -> Result<(), SyncError> { - // Create historical query descriptor for logs with configured validation stage - use phaser_bridge::descriptors::StreamPreferences; - let preferences = StreamPreferences { - max_message_bytes: 32 * 1024 * 1024, // 32MB for historical sync - compression: phaser_bridge::descriptors::Compression::None, - batch_size_hint: 100, - }; - let descriptor = BlockchainDescriptor::historical(StreamType::Logs, from_block, to_block) - .with_validation(self.validation_stage) - .with_preferences(preferences) - .with_traces(true); + // Create historical query using GenericQuery + // Note: enable_traces can be passed as a filter if needed + let query = GenericQuery::historical("logs", from_block, to_block) + .with_filter("enable_traces", serde_json::json!(true)); // Subscribe to the log stream with metadata (returns RecordBatch + responsibility range) - let stream = client - .subscribe_with_metadata(&descriptor) - .await - .map_err(|e| { - SyncError::from_anyhow_with_context( - DataType::Logs, - from_block, - to_block, - "Failed to subscribe to log stream", - e, - ) - })?; + let stream = client.query_with_metadata(query).await.map_err(|e| { + SyncError::from_anyhow_with_context( + DataType::Logs, + from_block, + to_block, + "Failed to subscribe to log stream", + e.into(), + ) + })?; let mut stream = Box::pin(stream); let mut batches_processed = 0u64; @@ -1318,10 +1289,11 @@ impl SyncWorker { // Write Arrow RecordBatch directly to parquet and get actual bytes written let batch_bytes = writer.write_batch(batch).await.map_err(|e| { - anyhow::anyhow!( - "Failed to write log batch. Write error: {:?}. Error chain: {}", - e, - e.to_string() + SyncError::disk_io_error( + DataType::Logs, + from_block, + to_block, + format!("Failed to write log batch: {e}"), ) })?; @@ -1371,7 +1343,7 @@ impl SyncWorker { // This is valid when a block range legitimately has no logs. // Arrow Flight doesn't transmit 0-row RecordBatches, so receiving 0 batches // means the range was processed successfully but contains no data. - info!( + debug!( "Worker {} received ZERO batches for logs {}-{}. \ Range processed successfully with no logs.", self.worker_id, from_block, to_block diff --git a/crates/phaser-bridge/Cargo.toml b/crates/phaser-server/Cargo.toml similarity index 57% rename from crates/phaser-bridge/Cargo.toml rename to crates/phaser-server/Cargo.toml index d80ece5..f7a322b 100644 --- a/crates/phaser-bridge/Cargo.toml +++ b/crates/phaser-server/Cargo.toml @@ -1,34 +1,31 @@ [package] -name = "phaser-bridge" +name = "phaser-server" version = "0.1.0" edition = "2021" +description = "Flight server implementation for phaser bridges" [dependencies] -# Arrow and Flight dependencies +# Core types +phaser-types = { path = "../phaser-types" } + +# Arrow and Flight (includes server features) arrow = { workspace = true } arrow-flight = { workspace = true } -arrow-array = "56.1" -arrow-schema = "56.1" -arrow-ipc = "56.1" +arrow-array = { workspace = true } +arrow-schema = { workspace = true } # Async and gRPC tokio = { workspace = true } tonic = { workspace = true } async-trait = { workspace = true } futures = "0.3" -tower = "0.5" -hyper-util = "0.1" -async-stream = "0.3" # Serialization serde = { workspace = true } serde_json = { workspace = true } -prost = { workspace = true } -bincode = "1.3" # Error handling thiserror = { workspace = true } -anyhow = { workspace = true } # Logging -tracing = { workspace = true } \ No newline at end of file +tracing = { workspace = true } diff --git a/crates/phaser-server/src/bridge.rs b/crates/phaser-server/src/bridge.rs new file mode 100644 index 0000000..ba1afa0 --- /dev/null +++ b/crates/phaser-server/src/bridge.rs @@ -0,0 +1,131 @@ +//! FlightBridge trait for implementing blockchain data bridges + +use arrow_flight::{ + Action, Criteria, FlightData, FlightDescriptor, FlightInfo, HandshakeRequest, + HandshakeResponse, SchemaResult, Ticket, +}; +use async_trait::async_trait; +use futures::Stream; +use std::pin::Pin; +use tonic::{Request, Response, Status, Streaming}; + +use phaser_types::{BridgeInfo, DiscoveryCapabilities, ACTION_DESCRIBE}; + +/// Capabilities that a bridge can advertise +#[derive(Debug, Clone)] +pub struct BridgeCapabilities { + pub supports_historical: bool, + pub supports_streaming: bool, + pub supports_reorg_notifications: bool, + pub supports_filters: bool, + pub supports_validation: bool, + pub max_batch_size: usize, +} + +/// Trait for blockchain data bridges to implement +#[async_trait] +pub trait FlightBridge: Send + Sync + 'static { + /// Get information about this bridge + async fn get_info(&self) -> Result; + + /// Get capabilities of this bridge + async fn get_capabilities(&self) -> Result; + + /// Handshake for authentication/authorization + async fn handshake( + &self, + request: Request>, + ) -> Result< + Response> + Send>>>, + Status, + >; + + /// Get flight information for a descriptor + async fn get_flight_info( + &self, + request: Request, + ) -> Result, Status>; + + /// Get schema for a stream type + async fn get_schema( + &self, + request: Request, + ) -> Result, Status>; + + /// List available flights + async fn list_flights( + &self, + request: Request, + ) -> Result> + Send>>>, Status>; + + /// Stream data for a ticket + async fn do_get( + &self, + request: Request, + ) -> Result> + Send>>>, Status>; + + /// Bidirectional streaming for real-time subscriptions + async fn do_exchange( + &self, + request: Request>, + ) -> Result> + Send>>>, Status>; + + /// Check health/readiness of the bridge + async fn health_check(&self) -> Result; + + /// Get protocol-agnostic discovery capabilities + /// + /// Returns information about available tables, position semantics, + /// and supported filters. Clients use this to query without + /// protocol-specific knowledge. + /// + /// Default implementation returns an error. Bridges should override + /// this to enable protocol-agnostic discovery. + async fn get_discovery_capabilities(&self) -> Result { + Err(Status::unimplemented( + "Discovery not implemented for this bridge", + )) + } + + /// Handle a Flight Action + /// + /// Built-in actions: + /// - "describe": Returns DiscoveryCapabilities as JSON + /// + /// Bridges can override to add custom actions. + async fn do_action( + &self, + request: Request, + ) -> Result< + Response> + Send>>>, + Status, + > { + use futures::stream; + + let action = request.into_inner(); + + match action.r#type.as_str() { + ACTION_DESCRIBE => { + let capabilities = self.get_discovery_capabilities().await?; + let json = serde_json::to_vec(&capabilities).map_err(|e| { + Status::internal(format!("Failed to serialize capabilities: {e}")) + })?; + + let result = arrow_flight::Result { body: json.into() }; + Ok(Response::new(Box::pin(stream::once(async { Ok(result) })))) + } + other => Err(Status::unimplemented(format!("Unknown action: {other}"))), + } + } + + /// List available actions + /// + /// Default implementation lists built-in actions. + /// Bridges can override to add custom actions. + async fn list_actions(&self) -> Result, Status> { + Ok(vec![arrow_flight::ActionType { + r#type: ACTION_DESCRIBE.to_string(), + description: "Get protocol-agnostic bridge capabilities".to_string(), + }]) + } +} diff --git a/crates/phaser-bridge/src/error.rs b/crates/phaser-server/src/error.rs similarity index 93% rename from crates/phaser-bridge/src/error.rs rename to crates/phaser-server/src/error.rs index f9b0c67..4ee25c8 100644 --- a/crates/phaser-bridge/src/error.rs +++ b/crates/phaser-server/src/error.rs @@ -1,3 +1,5 @@ +//! Error types for bridge server operations + use thiserror::Error; /// Errors that can occur in bridge stream operations diff --git a/crates/phaser-server/src/lib.rs b/crates/phaser-server/src/lib.rs new file mode 100644 index 0000000..969a337 --- /dev/null +++ b/crates/phaser-server/src/lib.rs @@ -0,0 +1,63 @@ +//! Flight server implementation for phaser bridges +//! +//! This crate provides the server-side implementation for bridges. +//! Use this in bridge implementations like erigon-bridge. +//! +//! # Example +//! +//! ```ignore +//! use phaser_server::{FlightBridge, FlightBridgeServer, BridgeCapabilities}; +//! use std::sync::Arc; +//! +//! struct MyBridge { /* ... */ } +//! +//! #[async_trait::async_trait] +//! impl FlightBridge for MyBridge { +//! // implement trait methods... +//! } +//! +//! async fn run_server(bridge: MyBridge) { +//! let server = FlightBridgeServer::new(Arc::new(bridge)); +//! // Use server.into_service() with tonic +//! } +//! ``` + +mod bridge; +mod error; +mod server; + +pub use bridge::{BridgeCapabilities, FlightBridge}; +pub use error::StreamError; +pub use server::FlightBridgeServer; + +// Re-export core types for convenience +pub use phaser_types::{ + // Subscription + BackpressureStrategy, + // Batch metadata + BatchMetadata, + BatchWithRange, + BlockRange, + // Descriptors + BlockchainDescriptor, + BridgeInfo, + Compression, + ControlAction, + DataAvailability, + DataSource, + // Discovery + DiscoveryCapabilities, + FilterDescriptor, + FilterSpec, + GenericQuery, + GenericQueryMode, + QueryMode, + ResponsibilityRange, + StreamPreferences, + StreamType, + SubscriptionInfo, + SubscriptionOptions, + TableDescriptor, + ValidationStage, + ACTION_DESCRIBE, +}; diff --git a/crates/phaser-bridge/src/server.rs b/crates/phaser-server/src/server.rs similarity index 92% rename from crates/phaser-bridge/src/server.rs rename to crates/phaser-server/src/server.rs index b677fe4..29c0639 100644 --- a/crates/phaser-bridge/src/server.rs +++ b/crates/phaser-server/src/server.rs @@ -1,3 +1,5 @@ +//! Arrow Flight server implementation for blockchain bridges + use arrow_flight::{ flight_service_server::{FlightService, FlightServiceServer}, Action, ActionType, Criteria, Empty, FlightData, FlightDescriptor, FlightInfo, @@ -105,15 +107,17 @@ impl FlightService for FlightBridgeServer { async fn do_action( &self, - _request: Request, + request: Request, ) -> Result, Status> { - Err(Status::unimplemented("do_action not implemented")) + self.bridge.do_action(request).await } async fn list_actions( &self, _request: Request, ) -> Result, Status> { - Err(Status::unimplemented("list_actions not implemented")) + let actions = self.bridge.list_actions().await?; + let stream = futures::stream::iter(actions.into_iter().map(Ok)); + Ok(Response::new(Box::pin(stream))) } } diff --git a/crates/phaser-types/Cargo.toml b/crates/phaser-types/Cargo.toml new file mode 100644 index 0000000..54e68bf --- /dev/null +++ b/crates/phaser-types/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "phaser-types" +version = "0.1.0" +edition = "2021" +description = "Shared protocol types for phaser (discovery, descriptors, subscription)" + +[dependencies] +# Arrow (minimal - just for Ticket/FlightDescriptor types) +arrow-flight = { workspace = true } +arrow-array = { workspace = true } + +# Async +futures = "0.3" + +# Serialization +serde = { workspace = true } +serde_json = { workspace = true } +bincode = "1.3" + +# Error handling +thiserror = { workspace = true } +anyhow = { workspace = true } diff --git a/crates/phaser-bridge/src/descriptors.rs b/crates/phaser-types/src/descriptors.rs similarity index 96% rename from crates/phaser-bridge/src/descriptors.rs rename to crates/phaser-types/src/descriptors.rs index 1096581..37dc0d5 100644 --- a/crates/phaser-bridge/src/descriptors.rs +++ b/crates/phaser-types/src/descriptors.rs @@ -1,8 +1,13 @@ +//! Blockchain-specific descriptors for data requests +//! +//! These types are used for EVM-style bridges with block-based data. + use crate::subscription::{QueryMode, SubscriptionOptions}; -use anyhow; use arrow_flight::{FlightDescriptor, Ticket}; use serde::{Deserialize, Serialize}; +use crate::discovery::ParseError; + /// Types of blockchain data streams #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] pub enum StreamType { @@ -183,11 +188,11 @@ impl BlockchainDescriptor { FlightDescriptor::new_path(vec![json]) } - pub fn from_flight_descriptor(desc: &FlightDescriptor) -> Result { + pub fn from_flight_descriptor(desc: &FlightDescriptor) -> Result { if let Some(path) = desc.path.first() { Ok(serde_json::from_str(path)?) } else { - Err(anyhow::anyhow!("No path in descriptor")) + Err(ParseError::NoPath) } } diff --git a/crates/phaser-types/src/discovery.rs b/crates/phaser-types/src/discovery.rs new file mode 100644 index 0000000..da2b95a --- /dev/null +++ b/crates/phaser-types/src/discovery.rs @@ -0,0 +1,312 @@ +//! Protocol-agnostic bridge discovery types +//! +//! These types allow clients to discover bridge capabilities without +//! hardcoded knowledge of specific protocols (EVM, Canton, etc.). + +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; + +/// Action name for the describe endpoint +pub const ACTION_DESCRIBE: &str = "describe"; + +/// Protocol-agnostic bridge capabilities +/// +/// Bridges return this from the "describe" Flight Action. +/// Clients use this to understand what tables are available +/// and how to query them. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct DiscoveryCapabilities { + /// Human-readable bridge name (e.g., "erigon-bridge", "canton-bridge") + pub name: String, + + /// Bridge version + pub version: String, + + /// Protocol type for routing (e.g., "evm", "canton", "solana") + /// Clients may use this to select appropriate schema crates + pub protocol: String, + + /// Label for the position dimension (e.g., "block_number", "offset") + /// This is purely descriptive - clients should treat positions as opaque u64 + pub position_label: String, + + /// Current position (e.g., latest block number, current offset) + pub current_position: u64, + + /// Oldest available position + pub oldest_position: u64, + + /// Available tables/streams + pub tables: Vec, + + /// Optional protocol-specific metadata + /// Clients can inspect this if they know the protocol + #[serde(default, skip_serializing_if = "HashMap::is_empty")] + pub metadata: HashMap, +} + +/// Description of a table available from the bridge +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct TableDescriptor { + /// Table name (used in queries) + pub name: String, + + /// Column name containing the position (e.g., "_block_num", "_offset") + pub position_column: String, + + /// Columns the data is sorted by (for client optimization hints) + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub sorted_by: Vec, + + /// Supported query modes for this table + #[serde(default)] + pub supported_modes: Vec, + + /// Filters that must be provided + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub required_filters: Vec, + + /// Optional filters the bridge can apply + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub optional_filters: Vec, +} + +impl TableDescriptor { + /// Create a simple table descriptor with minimal metadata + pub fn new(name: impl Into, position_column: impl Into) -> Self { + Self { + name: name.into(), + position_column: position_column.into(), + sorted_by: Vec::new(), + supported_modes: vec!["historical".to_string(), "live".to_string()], + required_filters: Vec::new(), + optional_filters: Vec::new(), + } + } + + /// Add supported modes + pub fn with_modes(mut self, modes: Vec<&str>) -> Self { + self.supported_modes = modes.into_iter().map(String::from).collect(); + self + } + + /// Add sort columns + pub fn with_sorted_by(mut self, columns: Vec<&str>) -> Self { + self.sorted_by = columns.into_iter().map(String::from).collect(); + self + } +} + +/// Description of a filter parameter +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct FilterDescriptor { + /// Filter name (e.g., "party_id", "addresses") + pub name: String, + + /// Human-readable description + #[serde(default, skip_serializing_if = "Option::is_none")] + pub description: Option, + + /// JSON schema type (e.g., "string", "array", "number") + #[serde(default = "default_filter_type")] + pub value_type: String, +} + +fn default_filter_type() -> String { + "string".to_string() +} + +impl FilterDescriptor { + pub fn new(name: impl Into) -> Self { + Self { + name: name.into(), + description: None, + value_type: "string".to_string(), + } + } + + pub fn with_description(mut self, desc: impl Into) -> Self { + self.description = Some(desc.into()); + self + } + + pub fn with_type(mut self, t: impl Into) -> Self { + self.value_type = t.into(); + self + } +} + +/// Protocol-agnostic query format +/// +/// Clients use this to request data from any bridge. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct GenericQuery { + /// Table/stream name (from TableDescriptor.name) + pub table: String, + + /// Query mode + pub mode: GenericQueryMode, + + /// Pass-through filters (bridge validates against TableDescriptor) + #[serde(default, skip_serializing_if = "HashMap::is_empty")] + pub filters: HashMap, +} + +impl GenericQuery { + /// Create a historical range query + pub fn historical(table: impl Into, start: u64, end: u64) -> Self { + Self { + table: table.into(), + mode: GenericQueryMode::Range { start, end }, + filters: HashMap::new(), + } + } + + /// Create a live subscription query + pub fn live(table: impl Into) -> Self { + Self { + table: table.into(), + mode: GenericQueryMode::Live, + filters: HashMap::new(), + } + } + + /// Create a snapshot query at a specific position + pub fn snapshot(table: impl Into, at: u64) -> Self { + Self { + table: table.into(), + mode: GenericQueryMode::Snapshot { at }, + filters: HashMap::new(), + } + } + + /// Add a filter + pub fn with_filter(mut self, key: impl Into, value: serde_json::Value) -> Self { + self.filters.insert(key.into(), value); + self + } + + /// Convert to Flight Ticket + pub fn to_ticket(&self) -> arrow_flight::Ticket { + let json = serde_json::to_vec(self).expect("GenericQuery serialization should not fail"); + arrow_flight::Ticket::new(json) + } + + /// Parse from Flight Ticket + pub fn from_ticket(ticket: &arrow_flight::Ticket) -> Result { + serde_json::from_slice(&ticket.ticket) + } + + /// Convert to FlightDescriptor for GetSchema + pub fn to_flight_descriptor(&self) -> arrow_flight::FlightDescriptor { + let json = serde_json::to_string(self).expect("GenericQuery serialization should not fail"); + arrow_flight::FlightDescriptor::new_path(vec![json]) + } + + /// Parse from FlightDescriptor + pub fn from_flight_descriptor( + desc: &arrow_flight::FlightDescriptor, + ) -> Result { + if let Some(path) = desc.path.first() { + Ok(serde_json::from_str(path)?) + } else { + Err(ParseError::NoPath) + } + } +} + +/// Query mode for protocol-agnostic queries +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(tag = "type", rename_all = "lowercase")] +pub enum GenericQueryMode { + /// Query a range of positions + Range { + /// Start position (inclusive) + start: u64, + /// End position (inclusive) + end: u64, + }, + + /// Subscribe to live data from current head + Live, + + /// Snapshot at a specific position + Snapshot { + /// Position to snapshot at + at: u64, + }, +} + +/// Errors that can occur when parsing descriptors +#[derive(Debug, thiserror::Error)] +pub enum ParseError { + #[error("No path in descriptor")] + NoPath, + + #[error("JSON parse error: {0}")] + Json(#[from] serde_json::Error), +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_discovery_capabilities_serde() { + let caps = DiscoveryCapabilities { + name: "erigon-bridge".to_string(), + version: "0.1.0".to_string(), + protocol: "evm".to_string(), + position_label: "block_number".to_string(), + current_position: 19_000_000, + oldest_position: 0, + tables: vec![ + TableDescriptor::new("blocks", "_block_num"), + TableDescriptor::new("transactions", "_block_num") + .with_sorted_by(vec!["_block_num", "_tx_idx"]), + TableDescriptor::new("logs", "_block_num").with_sorted_by(vec![ + "_block_num", + "_tx_idx", + "_log_idx", + ]), + ], + metadata: HashMap::new(), + }; + + let json = serde_json::to_string_pretty(&caps).unwrap(); + let parsed: DiscoveryCapabilities = serde_json::from_str(&json).unwrap(); + + assert_eq!(parsed.name, "erigon-bridge"); + assert_eq!(parsed.tables.len(), 3); + assert_eq!(parsed.tables[1].sorted_by, vec!["_block_num", "_tx_idx"]); + } + + #[test] + fn test_generic_query_serde() { + let query = GenericQuery::historical("transactions", 1000, 2000) + .with_filter("addresses", serde_json::json!(["0xabc", "0xdef"])); + + let json = serde_json::to_string(&query).unwrap(); + let parsed: GenericQuery = serde_json::from_str(&json).unwrap(); + + assert_eq!(parsed.table, "transactions"); + assert!(matches!( + parsed.mode, + GenericQueryMode::Range { + start: 1000, + end: 2000 + } + )); + assert!(parsed.filters.contains_key("addresses")); + } + + #[test] + fn test_generic_query_to_ticket() { + let query = GenericQuery::live("logs"); + let ticket = query.to_ticket(); + let parsed = GenericQuery::from_ticket(&ticket).unwrap(); + + assert_eq!(parsed.table, "logs"); + assert!(matches!(parsed.mode, GenericQueryMode::Live)); + } +} diff --git a/crates/phaser-bridge/src/lib.rs b/crates/phaser-types/src/lib.rs similarity index 87% rename from crates/phaser-bridge/src/lib.rs rename to crates/phaser-types/src/lib.rs index 7fe04b6..6001899 100644 --- a/crates/phaser-bridge/src/lib.rs +++ b/crates/phaser-types/src/lib.rs @@ -1,18 +1,26 @@ -pub mod bridge; -pub mod client; +//! Shared protocol types for phaser +//! +//! This crate contains protocol types shared between phaser-client and phaser-server: +//! - Discovery types (DiscoveryCapabilities, GenericQuery, TableDescriptor) +//! - Descriptors (BlockchainDescriptor, StreamType) +//! - Subscription types (QueryMode, SubscriptionOptions) +//! - Batch metadata (BatchMetadata, ResponsibilityRange) + pub mod descriptors; -pub mod error; -pub mod server; +pub mod discovery; pub mod subscription; use arrow_array::RecordBatch; use serde::{Deserialize, Serialize}; -pub use bridge::{BridgeCapabilities, FlightBridge}; -pub use client::FlightBridgeClient; -pub use descriptors::{BlockchainDescriptor, StreamType, ValidationStage}; -pub use error::StreamError; -pub use server::FlightBridgeServer; +pub use descriptors::{ + BlockchainDescriptor, BridgeInfo, Compression, EndpointInfo, StreamPreferences, StreamType, + ValidationStage, +}; +pub use discovery::{ + DiscoveryCapabilities, FilterDescriptor, GenericQuery, GenericQueryMode, TableDescriptor, + ACTION_DESCRIBE, +}; pub use subscription::{ BackpressureStrategy, BlockRange, ControlAction, DataAvailability, DataSource, FilterSpec, QueryMode, SubscriptionHandle, SubscriptionInfo, SubscriptionOptions, @@ -82,7 +90,7 @@ impl BatchMetadata { /// Returns an error if metadata is missing, empty, or invalid. /// This enforces that all batches from subscribe_with_metadata() must /// include proper metadata. - pub fn decode(metadata: &[u8]) -> Result> { + pub fn decode(metadata: &[u8]) -> Result> { if metadata.is_empty() { return Err("FlightData.app_metadata is empty - bridge must send BatchMetadata".into()); } diff --git a/crates/phaser-bridge/src/subscription.rs b/crates/phaser-types/src/subscription.rs similarity index 86% rename from crates/phaser-bridge/src/subscription.rs rename to crates/phaser-types/src/subscription.rs index cd49578..3bbedae 100644 --- a/crates/phaser-bridge/src/subscription.rs +++ b/crates/phaser-types/src/subscription.rs @@ -1,9 +1,6 @@ -use arrow_array::RecordBatch; -use futures::Stream; +//! Subscription types for streaming data from bridges + use serde::{Deserialize, Serialize}; -use std::pin::Pin; -use std::sync::atomic::AtomicU64; -use std::sync::Arc; /// How to handle backpressure when consumer is slow #[derive(Debug, Clone, Serialize, Deserialize)] @@ -45,13 +42,18 @@ pub struct FilterSpec { } /// Handle to an active subscription +/// +/// Note: This type contains a Stream which is not Send+Sync by default. +/// It's defined here but typically constructed in the client crate. pub struct SubscriptionHandle { /// Unique subscription ID pub id: String, - /// Stream of record batches - pub stream: Pin> + Send>>, + /// Stream of record batches (boxed to avoid generic complexity) + pub stream: std::pin::Pin< + Box> + Send>, + >, /// Last delivered block number (for resumption) - pub checkpoint: Arc, + pub checkpoint: std::sync::Arc, } /// Information about an active subscription diff --git a/crates/schemas/evm/common/src/rpc_conversions.rs b/crates/schemas/evm/common/src/rpc_conversions.rs index 8c9c392..617996b 100644 --- a/crates/schemas/evm/common/src/rpc_conversions.rs +++ b/crates/schemas/evm/common/src/rpc_conversions.rs @@ -122,7 +122,11 @@ pub fn convert_rpc_transactions(block: &AnyRpcBlock) -> Result { Ok(arrays.into_record_batch()) } -/// Convert RPC logs to RecordBatch +/// Convert RPC logs to RecordBatch (single block version) +/// +/// For logs from a single block where you already know the block context. +/// For logs from multiple blocks (e.g., eth_getLogs with block range), use +/// `convert_rpc_logs_multi_block` instead. pub fn convert_rpc_logs( logs: &[RpcLog], block_num: u64, @@ -163,6 +167,53 @@ pub fn convert_rpc_logs( Ok(arrays.into_record_batch()) } +/// Convert RPC logs to RecordBatch (multi-block version) +/// +/// For logs from eth_getLogs spanning multiple blocks. Extracts block_num, +/// block_hash, and timestamp from each log individually. This is more efficient +/// than calling `convert_rpc_logs` per-block because it reduces RPC round-trips. +pub fn convert_rpc_logs_multi_block(logs: &[RpcLog]) -> Result { + let mut builders = LogRecord::new_builders(logs.len()); + + for log in logs { + // Extract block context from the log itself + let block_num = log.block_number.unwrap_or(0); + let block_hash = log.block_hash.unwrap_or_default(); + // Note: block_timestamp is not always present in all nodes + // If missing, use 0 and let downstream handle it + let timestamp = log.block_timestamp.unwrap_or(0); + + // Extract topics (up to 4) + let topics = log.topics(); + let topic0 = topics.first().map(|t| (*t).into()); + let topic1 = topics.get(1).map(|t| (*t).into()); + let topic2 = topics.get(2).map(|t| (*t).into()); + let topic3 = topics.get(3).map(|t| (*t).into()); + + let record = LogRecord { + _block_num: block_num, + block_num, + block_hash: block_hash.into(), + timestamp: timestamp as i64 * 1_000_000_000, + tx_index: log.transaction_index.unwrap_or(0) as u32, + tx_hash: log.transaction_hash.unwrap_or_default().into(), + log_index: log.log_index.unwrap_or(0) as u32, + address: log.address().into(), + data: log.data().data.to_vec(), + topic0, + topic1, + topic2, + topic3, + removed: log.removed, + }; + + builders.append_row(record); + } + + let arrays = builders.finish(); + Ok(arrays.into_record_batch()) +} + /// Convert an AnyHeader to a BlockRecord pub fn convert_any_header_to_record(header: &AnyHeader) -> BlockRecord { // AnyHeader derefs to the inner consensus header diff --git a/crates/tools/bridge-test/Cargo.toml b/crates/tools/bridge-test/Cargo.toml new file mode 100644 index 0000000..781d90b --- /dev/null +++ b/crates/tools/bridge-test/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "bridge-test" +version = "0.1.0" +edition = "2024" + +[[bin]] +name = "bridge-test" +path = "src/main.rs" + +[dependencies] +phaser-client = { path = "../../phaser-client" } +phaser-types = { path = "../../phaser-types" } +tokio = { workspace = true } +clap = { version = "4.0", features = ["derive"] } +tracing = { workspace = true } +tracing-subscriber = { workspace = true } +arrow = { workspace = true } +futures = { workspace = true } +serde_json = { workspace = true } diff --git a/crates/tools/bridge-test/src/main.rs b/crates/tools/bridge-test/src/main.rs new file mode 100644 index 0000000..51ed0d3 --- /dev/null +++ b/crates/tools/bridge-test/src/main.rs @@ -0,0 +1,428 @@ +//! Bridge comparison test tool +//! +//! Tests that phaser-query can work with any bridge implementation by comparing +//! discovery metadata, schemas, and data between two bridge endpoints. + +use clap::{Parser, Subcommand}; +use futures::StreamExt; +use phaser_client::{GenericQuery, PhaserClient}; +use std::path::PathBuf; +use tracing::{error, info}; + +#[derive(Parser)] +#[command(name = "bridge-test")] +#[command(about = "Test and compare bridge implementations")] +struct Cli { + #[command(subcommand)] + command: Commands, +} + +#[derive(Subcommand)] +enum Commands { + /// Discover bridge capabilities and print metadata + Discover { + /// Bridge endpoint (e.g., http://127.0.0.1:8090) + #[arg(short, long)] + endpoint: String, + }, + + /// Compare discovery metadata between two bridges + Compare { + /// First bridge endpoint + #[arg(long)] + bridge1: String, + /// Second bridge endpoint + #[arg(long)] + bridge2: String, + /// Output file for comparison results (JSON) + #[arg(short, long)] + output: Option, + }, + + /// Fetch data from a bridge and validate schema + Fetch { + /// Bridge endpoint + #[arg(short, long)] + endpoint: String, + /// Table to fetch (blocks, transactions, logs) + #[arg(short, long)] + table: String, + /// Start block + #[arg(long)] + start: u64, + /// End block + #[arg(long)] + end: u64, + /// Output file for data (Parquet) + #[arg(short, long)] + output: Option, + }, + + /// Compare data between two bridges for the same block range + CompareData { + /// First bridge endpoint + #[arg(long)] + bridge1: String, + /// Second bridge endpoint + #[arg(long)] + bridge2: String, + /// Table to compare + #[arg(short, long)] + table: String, + /// Start block + #[arg(long)] + start: u64, + /// End block + #[arg(long)] + end: u64, + /// Output directory for comparison results + #[arg(short, long)] + output: Option, + }, +} + +#[tokio::main] +async fn main() -> Result<(), Box> { + // Initialize tracing + tracing_subscriber::fmt() + .with_env_filter( + tracing_subscriber::EnvFilter::try_from_default_env() + .unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("info")), + ) + .init(); + + let cli = Cli::parse(); + + match cli.command { + Commands::Discover { endpoint } => { + discover_bridge(&endpoint).await?; + } + Commands::Compare { + bridge1, + bridge2, + output, + } => { + compare_bridges(&bridge1, &bridge2, output).await?; + } + Commands::Fetch { + endpoint, + table, + start, + end, + output, + } => { + fetch_data(&endpoint, &table, start, end, output).await?; + } + Commands::CompareData { + bridge1, + bridge2, + table, + start, + end, + output, + } => { + compare_data(&bridge1, &bridge2, &table, start, end, output).await?; + } + } + + Ok(()) +} + +async fn discover_bridge(endpoint: &str) -> Result<(), Box> { + info!("Discovering bridge at: {}", endpoint); + + let mut client = PhaserClient::connect(endpoint.to_string()).await?; + let capabilities = client.discover().await?; + + println!("Bridge Discovery Results:"); + println!("========================"); + println!("Name: {}", capabilities.name); + println!("Version: {}", capabilities.version); + println!("Protocol: {}", capabilities.protocol); + println!("Position Label: {}", capabilities.position_label); + println!( + "Position Range: {} - {}", + capabilities.oldest_position, capabilities.current_position + ); + println!(); + println!("Tables:"); + for table in &capabilities.tables { + println!(" - {}", table.name); + println!(" Position column: {}", table.position_column); + println!(" Supported modes: {:?}", table.supported_modes); + if !table.sorted_by.is_empty() { + println!(" Sorted by: {:?}", table.sorted_by); + } + } + println!(); + println!("Metadata:"); + println!("{}", serde_json::to_string_pretty(&capabilities.metadata)?); + + Ok(()) +} + +async fn compare_bridges( + bridge1: &str, + bridge2: &str, + output: Option, +) -> Result<(), Box> { + info!("Comparing bridges: {} vs {}", bridge1, bridge2); + + let mut client1 = PhaserClient::connect(bridge1.to_string()).await?; + let mut client2 = PhaserClient::connect(bridge2.to_string()).await?; + + let caps1 = client1.discover().await?; + let caps2 = client2.discover().await?; + + let mut issues: Vec = vec![]; + + // Compare protocol + if caps1.protocol != caps2.protocol { + issues.push(format!( + "Protocol mismatch: {} vs {}", + caps1.protocol, caps2.protocol + )); + } + + // Compare tables + let tables1: std::collections::HashSet<_> = caps1.tables.iter().map(|t| &t.name).collect(); + let tables2: std::collections::HashSet<_> = caps2.tables.iter().map(|t| &t.name).collect(); + + for table in tables1.difference(&tables2) { + issues.push(format!("Table '{table}' only in bridge1")); + } + for table in tables2.difference(&tables1) { + issues.push(format!("Table '{table}' only in bridge2")); + } + + // Compare common tables + for table_name in tables1.intersection(&tables2) { + let t1 = caps1.tables.iter().find(|t| &t.name == *table_name); + let t2 = caps2.tables.iter().find(|t| &t.name == *table_name); + + if let (Some(t1), Some(t2)) = (t1, t2) + && t1.position_column != t2.position_column + { + issues.push(format!( + "Table '{}' position column mismatch: {} vs {}", + table_name, t1.position_column, t2.position_column + )); + } + } + + // Print results + println!("Bridge Comparison Results:"); + println!("=========================="); + println!("Bridge 1: {} v{} ({})", caps1.name, caps1.version, bridge1); + println!("Bridge 2: {} v{} ({})", caps2.name, caps2.version, bridge2); + println!(); + + if issues.is_empty() { + println!("PASS: No differences found in discovery metadata"); + } else { + println!("DIFFERENCES FOUND ({}):", issues.len()); + for issue in &issues { + println!(" - {issue}"); + } + } + + // Save to file if requested + if let Some(output_path) = output { + let result = serde_json::json!({ + "bridge1": { + "endpoint": bridge1, + "name": caps1.name, + "version": caps1.version, + "protocol": caps1.protocol, + "tables": caps1.tables.iter().map(|t| &t.name).collect::>(), + }, + "bridge2": { + "endpoint": bridge2, + "name": caps2.name, + "version": caps2.version, + "protocol": caps2.protocol, + "tables": caps2.tables.iter().map(|t| &t.name).collect::>(), + }, + "issues": issues, + "pass": issues.is_empty(), + }); + std::fs::write(&output_path, serde_json::to_string_pretty(&result)?)?; + info!("Results written to: {:?}", output_path); + } + + Ok(()) +} + +async fn fetch_data( + endpoint: &str, + table: &str, + start: u64, + end: u64, + _output: Option, +) -> Result<(), Box> { + info!( + "Fetching {} from {} (blocks {}-{})", + table, endpoint, start, end + ); + + let mut client = PhaserClient::connect(endpoint.to_string()).await?; + + let query = GenericQuery::historical(table, start, end); + + let mut stream = client.query(query).await?; + + let mut total_rows = 0u64; + let mut total_batches = 0u64; + + while let Some(result) = stream.next().await { + match result { + Ok(batch) => { + total_batches += 1; + total_rows += batch.num_rows() as u64; + info!( + "Batch {}: {} rows, {} columns", + total_batches, + batch.num_rows(), + batch.num_columns() + ); + } + Err(e) => { + error!("Error fetching batch: {}", e); + return Err(e.into()); + } + } + } + + println!(); + println!("Fetch Results:"); + println!("=============="); + println!("Total batches: {total_batches}"); + println!("Total rows: {total_rows}"); + + Ok(()) +} + +async fn compare_data( + bridge1: &str, + bridge2: &str, + table: &str, + start: u64, + end: u64, + output: Option, +) -> Result<(), Box> { + info!( + "Comparing {} data between bridges (blocks {}-{})", + table, start, end + ); + + let mut client1 = PhaserClient::connect(bridge1.to_string()).await?; + let mut client2 = PhaserClient::connect(bridge2.to_string()).await?; + + let query = GenericQuery::historical(table, start, end); + + // Fetch from both bridges + let stream1 = client1.query(query.clone()).await?; + let stream2 = client2.query(query).await?; + + let batches1: Vec<_> = stream1 + .collect::>() + .await + .into_iter() + .filter_map(|r| r.ok()) + .collect(); + let batches2: Vec<_> = stream2 + .collect::>() + .await + .into_iter() + .filter_map(|r| r.ok()) + .collect(); + + let rows1: u64 = batches1.iter().map(|b| b.num_rows() as u64).sum(); + let rows2: u64 = batches2.iter().map(|b| b.num_rows() as u64).sum(); + + println!(); + println!("Data Comparison Results:"); + println!("========================"); + println!("Table: {table}"); + println!("Block range: {start} - {end}"); + println!(); + println!("Bridge 1: {} batches, {} rows", batches1.len(), rows1); + println!("Bridge 2: {} batches, {} rows", batches2.len(), rows2); + + let mut issues: Vec = vec![]; + + if rows1 != rows2 { + issues.push(format!("Row count mismatch: {rows1} vs {rows2}")); + } + + // Compare schemas from first batch + if let (Some(b1), Some(b2)) = (batches1.first(), batches2.first()) { + let schema1 = b1.schema(); + let schema2 = b2.schema(); + + let fields1: std::collections::HashSet<_> = + schema1.fields().iter().map(|f| f.name()).collect(); + let fields2: std::collections::HashSet<_> = + schema2.fields().iter().map(|f| f.name()).collect(); + + for field in fields1.difference(&fields2) { + issues.push(format!("Field '{field}' only in bridge1")); + } + for field in fields2.difference(&fields1) { + issues.push(format!("Field '{field}' only in bridge2")); + } + + // Check types for common fields + for field_name in fields1.intersection(&fields2) { + let f1 = schema1.field_with_name(field_name).ok(); + let f2 = schema2.field_with_name(field_name).ok(); + + if let (Some(f1), Some(f2)) = (f1, f2) + && f1.data_type() != f2.data_type() + { + issues.push(format!( + "Field '{}' type mismatch: {:?} vs {:?}", + field_name, + f1.data_type(), + f2.data_type() + )); + } + } + } + + if issues.is_empty() { + println!(); + println!("PASS: Schema and row counts match"); + } else { + println!(); + println!("DIFFERENCES FOUND:"); + for issue in &issues { + println!(" - {issue}"); + } + } + + // Save to file if requested + if let Some(output_path) = output { + let result = serde_json::json!({ + "table": table, + "start_block": start, + "end_block": end, + "bridge1": { + "endpoint": bridge1, + "batches": batches1.len(), + "rows": rows1, + }, + "bridge2": { + "endpoint": bridge2, + "batches": batches2.len(), + "rows": rows2, + }, + "issues": issues, + "pass": issues.is_empty(), + }); + std::fs::write(&output_path, serde_json::to_string_pretty(&result)?)?; + info!("Results written to: {:?}", output_path); + } + + Ok(()) +} diff --git a/docs/METRICS_PLANNING.md b/docs/METRICS_PLANNING.md new file mode 100644 index 0000000..6b10bc5 --- /dev/null +++ b/docs/METRICS_PLANNING.md @@ -0,0 +1,197 @@ +# Phaser Metrics Planning + +## Overview + +This document outlines the metrics architecture for phaser bridges and query services, with focus on multi-chain support and protocol-agnostic design. + +## Problem Statement + +Bridges (jsonrpc-bridge, erigon-bridge) can connect to different chains: +- Ethereum mainnet (chain_id: 1) +- Arbitrum One (chain_id: 42161) +- Base (chain_id: 8453) +- Gnosis (chain_id: 100) +- Polygon (chain_id: 137) +- etc. + +Erigon is adding Arbitrum support. A single jsonrpc-bridge binary can point at any EVM-compatible JSON-RPC endpoint. Metrics need to cleanly disambiguate which chain is being synced. + +## Current State + +### Labels in Use +``` +chain_id="1" # Chain identifier +bridge_name="jsonrpc" # Bridge instance name +service_name="jsonrpc_bridge" # Metric prefix +``` + +### Issues +1. **Service name collisions**: Running `jsonrpc-bridge` against mainnet and arbitrum creates duplicate metric registrations +2. **Metric prefix baked in**: `jsonrpc_bridge_*` doesn't indicate which chain +3. **No standard chain labeling**: Each bridge handles chain_id differently + +## Proposed Design + +### 1. Chain-Aware Metric Naming + +**Option A: Chain in service name** +``` +jsonrpc_bridge_1_segment_duration_seconds{...} # Mainnet +jsonrpc_bridge_42161_segment_duration_seconds{...} # Arbitrum +``` +- Pro: Clear separation, no collision +- Con: Many metrics per chain, harder to aggregate + +**Option B: Chain as primary label (recommended)** +``` +bridge_segment_duration_seconds{bridge_type="jsonrpc", chain_id="1", ...} +bridge_segment_duration_seconds{bridge_type="jsonrpc", chain_id="42161", ...} +``` +- Pro: Easy to aggregate across chains, filter by chain +- Con: Requires changing existing metric names + +**Option C: Instance label** +``` +jsonrpc_bridge_segment_duration_seconds{instance="mainnet", chain_id="1", ...} +jsonrpc_bridge_segment_duration_seconds{instance="arbitrum", chain_id="42161", ...} +``` +- Pro: Backward compatible +- Con: Instance name is arbitrary, chain_id is authoritative + +### 2. Standard Labels + +All bridge metrics should include: +``` +chain_id # Numeric chain ID (1, 42161, etc.) +chain_name # Human-readable (mainnet, arbitrum, base) - optional +bridge_type # jsonrpc, erigon, canton +bridge_name # Instance name for disambiguation +``` + +### 3. Generic Segment Metrics + +Protocol-agnostic metrics that work for any data source: + +```rust +pub struct SegmentMetricsConfig { + /// Metric name prefix + pub prefix: String, + + /// Chain identifier + pub chain_id: u64, + + /// Optional chain name for labels + pub chain_name: Option, + + /// Bridge type (jsonrpc, erigon, etc.) + pub bridge_type: String, + + /// Bridge instance name + pub bridge_name: String, + + /// Data types/phases to track (e.g., ["blocks", "transactions", "logs"]) + pub data_types: Vec, +} +``` + +### 4. Metrics to Implement + +#### Segment Lifecycle +``` +segment_start_timestamp{segment_num, chain_id} +segment_complete_timestamp{segment_num, chain_id} +segment_duration_total_seconds{segment_num, chain_id} # Wall-clock time +segment_attempts_total{chain_id, result} # success/failure +segment_retry_count{segment_num, chain_id} +``` + +#### Data Progress +``` +items_processed_total{chain_id, data_type, segment_num} +items_per_second{chain_id, data_type} # Gauge, current rate +bytes_processed_total{chain_id, data_type} +``` + +#### RPC/Transport +``` +rpc_requests_total{chain_id, method, status} +rpc_request_duration_seconds{chain_id, method} +rpc_errors_total{chain_id, error_type} +rate_limit_events_total{chain_id} +``` + +#### Consumer Lag +``` +consumer_lag_blocks{chain_id, consumer} +consumer_lag_seconds{chain_id, consumer} +pending_batches{chain_id} +``` + +## Migration Path + +1. **Phase 1**: Add `chain_name` label to existing metrics (non-breaking) +2. **Phase 2**: Create new unified `bridge_*` metrics alongside existing +3. **Phase 3**: Deprecate old metrics, update dashboards +4. **Phase 4**: Remove deprecated metrics + +## Dashboard Considerations + +With chain-aware metrics, dashboards can: +- Show all chains on one view with chain selector +- Compare sync progress across chains +- Aggregate error rates across deployments +- Alert on chain-specific issues + +Example Grafana variables: +``` +$chain_id = label_values(bridge_segment_duration_seconds, chain_id) +$chain_name = label_values(bridge_segment_duration_seconds, chain_name) +``` + +## Implementation Notes + +### BridgeMetrics Constructor +```rust +impl BridgeMetrics { + pub fn new(config: SegmentMetricsConfig) -> Self { + // Use config.prefix for metric names + // Include chain_id, chain_name, bridge_type in all labels + } +} +``` + +### CLI Configuration +```bash +jsonrpc-bridge \ + --jsonrpc-url http://127.0.0.1:8545 \ + --chain-name mainnet \ # Optional, derived from chain_id if not set + --metrics-prefix jsonrpc \ # Or use unified "bridge" prefix +``` + +### Chain Name Resolution +```rust +fn chain_name(chain_id: u64) -> &'static str { + match chain_id { + 1 => "mainnet", + 42161 => "arbitrum", + 8453 => "base", + 100 => "gnosis", + 137 => "polygon", + 10 => "optimism", + _ => "unknown", + } +} +``` + +## Open Questions + +1. Should we use a unified `bridge_*` prefix or keep `jsonrpc_bridge_*` / `erigon_bridge_*`? +2. How to handle metrics for bridges serving multiple chains (if ever)? +3. Should chain_name be required or auto-derived from chain_id? +4. Cardinality concerns with segment_num labels for long-running syncs? + +## Related Files + +- `crates/phaser-metrics/src/segment_metrics.rs` - Current implementation +- `crates/phaser-metrics/src/lib.rs` - QueryMetrics, BridgeMetrics +- `test-data/jsonrpc-bridge-tests/IMPROVEMENTS.md` - Testing observations diff --git a/docs/example-grafana-dash.json b/docs/example-grafana-dash.json new file mode 100644 index 0000000..e6489ec --- /dev/null +++ b/docs/example-grafana-dash.json @@ -0,0 +1,800 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 7, + "links": [], + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 10, + "options": { + "calculate": false, + "cellGap": 1, + "cellValues": { + "unit": "ms" + }, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdYlGn", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false, + "unit": "" + } + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "editorMode": "code", + "expr": " sum by (segment_num) (\n rate(erigon_bridge_grpc_request_duration_blocks_milliseconds_sum[1m])\n ) > 0\n", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Active Blocks Request times (ms)", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 7 + }, + "id": 9, + "options": { + "calculate": false, + "cellGap": 1, + "cellValues": { + "unit": "ms" + }, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Rainbow", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false, + "unit": "" + } + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "editorMode": "code", + "expr": " sum by (segment_num) (\n rate(erigon_bridge_grpc_request_duration_transactions_milliseconds_sum[1m])\n ) > 0\n", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Active Transaction Request times (ms)", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 14 + }, + "id": 8, + "options": { + "calculate": false, + "cellGap": 1, + "cellValues": { + "unit": "ms" + }, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Sinebow", + "steps": 128 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false, + "unit": "" + } + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "editorMode": "code", + "expr": " sum by (segment_num) (\n rate(erigon_bridge_grpc_request_duration_logs_milliseconds_sum[1m])\n ) > 0\n", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Active Logs Request times (ms)", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 21 + }, + "id": 4, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "editorMode": "builder", + "expr": "erigon_bridge_active_workers", + "legendFormat": "{{phase}}", + "range": true, + "refId": "A" + } + ], + "title": "Bridge Workers", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 21 + }, + "id": 3, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "editorMode": "builder", + "expr": "erigon_bridge_grpc_streams_active", + "legendFormat": "{{stream_type}}", + "range": true, + "refId": "A" + } + ], + "title": "Bridge GRPC streams", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 30 + }, + "id": 1, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "builder", + "expr": "phaser_query_active_workers", + "legendFormat": "{{phase}} workers", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "builder", + "expr": "phaser_query_segment_retry_count", + "hide": false, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "builder", + "expr": "phaser_query_sync_queue_depth", + "hide": false, + "instant": false, + "legendFormat": "{{phase}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "builder", + "expr": "phaser_query_errors_total", + "hide": false, + "instant": false, + "legendFormat": "{{phase}}", + "range": true, + "refId": "D" + } + ], + "title": "Query sync workers and queue", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 30 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "editorMode": "builder", + "expr": "erigon_bridge_segment_attempts_total", + "legendFormat": "__auto", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "builder", + "expr": "erigon_bridge_segment_duration_seconds_count", + "hide": false, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "B" + } + ], + "title": "bridge segments", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 8, + "y": 39 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "editorMode": "builder", + "expr": "erigon_bridge_errors_total", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Bridge Errors", + "type": "timeseries" + } + ], + "preload": false, + "schemaVersion": 42, + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "2025-11-07T15:20:49.622Z", + "to": "2025-11-08T06:15:30.095Z" + }, + "timepicker": {}, + "timezone": "browser", + "title": "phaser ", + "uid": "adrx7z7", + "version": 34 +} \ No newline at end of file diff --git a/docs/phaser-cli.md b/docs/phaser-cli.md new file mode 100644 index 0000000..342eecc --- /dev/null +++ b/docs/phaser-cli.md @@ -0,0 +1,308 @@ +# phaser-cli - Historical Sync Administration + +`phaser-cli` is a command-line tool for managing historical blockchain data synchronization with phaser-query. + +## Installation + +Build from source: + +```bash +cargo build -p phaser-query --bin phaser-cli +``` + +The binary will be at `./target/debug/phaser-cli` (or `./target/release/phaser-cli` with `--release`). + +## Prerequisites + +Before using the CLI, you need: + +1. **phaser-query running** with sync admin enabled (default port 9093) +2. **erigon-bridge running** connected to a custom Erigon node with BlockDataBackend + +Example startup: + +```bash +# Start erigon-bridge (connects to custom Erigon) +./target/debug/erigon-bridge \ + --erigon-grpc 192.168.0.174:9090 \ + --flight-addr 0.0.0.0:8090 + +# Start phaser-query (without live streaming for historical sync) +./target/debug/phaser-query \ + -c config.yaml \ + --disable-streaming \ + --metrics-port 9092 +``` + +## Commands + +### sync - Start a Historical Sync Job + +Start syncing blockchain data for a range of blocks. + +```bash +phaser-cli -e http://127.0.0.1:9093 sync \ + --chain-id 1 \ + --bridge erigon \ + --from 0 \ + --to 24628000 +``` + +**Options:** +- `-c, --chain-id` - Chain ID (must match a configured bridge) +- `-b, --bridge` - Bridge name (as defined in config.yaml) +- `-f, --from` - Starting block number (inclusive) +- `-t, --to` - Ending block number (inclusive) + +**Example output:** + +``` +✓ Sync job started + Job ID: 9fe2cf40-9f24-4b8c-a866-f57844bfdea4 + Sync job created for blocks 0-24628000 on chain 1 via bridge 'erigon' + +Gap Analysis: + Total segments: 50 + Complete: 0 (0.0%) + Missing: 50 + + 50 incomplete segments (showing first 5): + Segment 0 (blocks 0-499999): missing transactions, logs + Segment 1 (blocks 500000-999999): missing transactions, logs + ... +``` + +### status - Check Sync Job Status + +View status of all jobs or a specific job. + +**List all jobs:** + +```bash +phaser-cli -e http://127.0.0.1:9093 status +``` + +**View specific job:** + +```bash +phaser-cli -e http://127.0.0.1:9093 status 9fe2cf40-9f24-4b8c-a866-f57844bfdea4 +``` + +**Filter by status:** + +```bash +phaser-cli -e http://127.0.0.1:9093 status --status RUNNING +``` + +Valid status filters: `PENDING`, `RUNNING`, `COMPLETED`, `FAILED`, `CANCELLED` + +**Example output:** + +``` +Found 1 sync job(s): + +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +Job ID: 9fe2cf40-9f24-4b8c-a866-f57844bfdea4 +Status: RUNNING +Chain: 1 / Bridge: erigon +Blocks: 0-24628000 + +Data Progress (by segment): + Blocks: 7/50 segments - 8 files, 907.3 MB (2 gaps) + Transactions: 0/50 segments - 1 files, 12.5 MB (2 gaps) + Logs: 0/50 segments - 1 files, 7.6 MB (2 gaps) + +Total Files: 10 (927.4 MB) + +Complete Segments: 0/50 (0.0% of segments) +Incomplete Segments: 50 + - 43 segments missing blocks + - 50 segments missing transactions + - 50 segments missing logs +Active workers: 4 +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +``` + +### cancel - Cancel a Running Sync Job + +Stop a sync job in progress. + +```bash +phaser-cli -e http://127.0.0.1:9093 cancel 9fe2cf40-9f24-4b8c-a866-f57844bfdea4 +``` + +**Example output:** + +``` +✓ Sync job cancelled +``` + +### analyze - Analyze Data Gaps + +Check what data is missing without starting a sync. + +```bash +phaser-cli -e http://127.0.0.1:9093 analyze \ + --chain-id 1 \ + --bridge erigon \ + --from 0 \ + --to 24628000 +``` + +**Example output:** + +``` +Analyzed blocks 0-24628000 for chain 1 via bridge 'erigon' + +Gap Analysis: + Total segments: 50 + Complete: 7 (14.0%) + Missing: 43 + + Incomplete segments: + Segment 0 (blocks 0-499999): + - blocks: + 0-499999 (500000 blocks) + - transactions: + 0-499999 (500000 blocks) + ... + + Segments to sync: [0, 1, 2, 3, 4, 5, ...] +``` + +## Concepts + +### Segments + +Data is organized into segments of 500,000 blocks each (aligned with Erigon snapshots): + +- Segment 0: blocks 0 - 499,999 +- Segment 1: blocks 500,000 - 999,999 +- Segment 2: blocks 1,000,000 - 1,499,999 +- etc. + +A segment is "complete" when it has all three data types: blocks, transactions, and logs. + +### Data Types + +Each segment contains three types of data: + +- **Blocks**: Block headers (hash, number, timestamp, gas, etc.) +- **Transactions**: Transaction data with sender addresses +- **Logs**: Event logs from contract execution + +### Workers + +Sync jobs run with multiple parallel workers (default: 4). Each worker syncs one segment at a time. + +## Configuration + +phaser-query reads configuration from a YAML file: + +```yaml +# config.yaml +rocksdb_path: ./data/rocksdb +data_root: ./data + +bridges: + - chain_id: 1 + endpoint: http://127.0.0.1:8090 + name: erigon + +segment_size: 500000 +sync_admin_port: 9093 +sync_parallelism: 4 +``` + +## Monitoring + +### Prometheus Metrics + +phaser-query exposes metrics on port 9092 (configurable): + +```bash +curl http://localhost:9092/metrics +``` + +### Log Files + +When running with output redirection: + +```bash +./target/debug/phaser-query -c config.yaml > ./logs/phaser-query.log 2>&1 & +``` + +Monitor sync progress: + +```bash +tail -f ./logs/phaser-query.log | grep -E "segment|sync|worker" +``` + +## Troubleshooting + +### "Live streaming is enabled, waiting for boundary..." + +This happens when phaser-query is started with live streaming enabled but the Erigon node isn't receiving new blocks. Either: + +1. Wait for a new block (~12 seconds on mainnet) +2. Restart phaser-query with `--disable-streaming` + +### Connection refused + +Make sure phaser-query's sync admin gRPC server is running: + +```bash +# Check port 9093 is listening +lsof -i :9093 +``` + +### No sync jobs found + +The sync job may have completed or failed. Check phaser-query logs: + +```bash +tail -100 ./logs/phaser-query.log | grep -E "sync|error|fail" +``` + +## Examples + +### Full Mainnet Sync + +Sync the entire Ethereum mainnet history: + +```bash +# Get latest block number +LATEST=$(curl -s http://localhost:8545 -X POST \ + -H "Content-Type: application/json" \ + -d '{"jsonrpc":"2.0","method":"eth_blockNumber","params":[],"id":1}' \ + | jq -r '.result' | xargs printf "%d") + +# Start sync +phaser-cli -e http://127.0.0.1:9093 sync \ + --chain-id 1 \ + --bridge erigon \ + --from 0 \ + --to $LATEST +``` + +### Monitor Progress + +Watch sync progress in a loop: + +```bash +watch -n 5 './target/debug/phaser-cli -e http://127.0.0.1:9093 status' +``` + +### Resume After Failure + +If a sync job fails, just start a new one - it will automatically detect existing data and only sync missing segments: + +```bash +phaser-cli -e http://127.0.0.1:9093 sync \ + --chain-id 1 \ + --bridge erigon \ + --from 0 \ + --to 24628000 +``` + +The gap analysis will show which segments still need syncing. diff --git a/docs/supervisor-monitoring.md b/docs/supervisor-monitoring.md new file mode 100644 index 0000000..fcc1703 --- /dev/null +++ b/docs/supervisor-monitoring.md @@ -0,0 +1,221 @@ +# Using Balloons Supervisor for Phaser Monitoring + +The Balloons supervisor provides background process management with log capture, +ideal for long-running sync jobs. + +## Benefits Over Direct Execution + +| Feature | Direct (bash &) | Supervisor | +|---------|----------------|------------| +| Log capture | Manual redirection | Automatic ring buffer | +| Process tracking | By PID | By name/ID | +| Session awareness | None | Tied to session | +| Remote execution | Manual SSH | Built-in SSH support | +| Output querying | `tail -f` | Structured queries | + +## Starting Services via Supervisor + +### Local Execution + +Start erigon-bridge: +``` + +{"name": "supervisor_start", "args": { + "command": "./target/release/erigon-bridge --erigon-grpc 192.168.0.174:9091 --flight-addr 0.0.0.0:8090", + "name": "erigon-bridge", + "working_dir": "/home/dan/Development/en/phaser", + "env": {"RUST_BACKTRACE": "1"} +}} + +``` + +Start phaser-query: +``` + +{"name": "supervisor_start", "args": { + "command": "./target/release/phaser-query -c ./test-data/test-config.yaml --disable-streaming --metrics-port 9092", + "name": "phaser-query", + "working_dir": "/home/dan/Development/en/phaser" +}} + +``` + +### Remote Execution (via SSH) + +If you have hosts configured in `~/.balloons/supervisor.yaml`: + +```yaml +# ~/.balloons/supervisor.yaml +hosts: + superserver: + type: ssh + host: 192.168.0.174 + user: dan + tags: [erigon, ethereum] +``` + +Then execute remotely: +``` + +{"name": "supervisor_start", "args": { + "command": "/home/dan/bin/erigon --datadir=/mnt/nvme-raid/erigon-archive --prune.mode=archive ...", + "name": "erigon-archive", + "host": "superserver" +}} + +``` + +## Monitoring Processes + +### List All Processes +``` + +{"name": "supervisor_list", "args": {}} + +``` + +Returns: +```json +[ + { + "process_id": "abc123", + "name": "erigon-bridge", + "command": "./target/release/erigon-bridge ...", + "status": {"state": "running", "pid": 12345}, + "session_id": "current-session" + }, + ... +] +``` + +### Get Process Output +``` + +{"name": "supervisor_output", "args": { + "process_id": "abc123", + "limit": 50 +}} + +``` + +Returns last 50 log entries with timestamps and source (stdout/stderr). + +### Filter by Session +``` + +{"name": "supervisor_list", "args": {"all_sessions": false}} + +``` + +Only shows processes from current session. + +### Check Specific Host +``` + +{"name": "supervisor_host_status", "args": {"host": "superserver"}} + +``` + +## Stopping Processes + +``` + +{"name": "supervisor_stop", "args": {"process_id": "abc123"}} + +``` + +## Example Workflow: Full Sync Monitoring + +### 1. Start Services + +``` +# Start bridge + +{"name": "supervisor_start", "args": { + "command": "./target/release/erigon-bridge --erigon-grpc 192.168.0.174:9091 --flight-addr 0.0.0.0:8090", + "name": "bridge", + "working_dir": "/home/dan/Development/en/phaser" +}} + + +# Start query service + +{"name": "supervisor_start", "args": { + "command": "./target/release/phaser-query -c ./test-data/test-config.yaml --disable-streaming", + "name": "query", + "working_dir": "/home/dan/Development/en/phaser" +}} + +``` + +### 2. Start Sync Job (via bash, not supervisor) + +```bash +./target/debug/phaser-cli -e http://127.0.0.1:9093 sync \ + --chain-id 1 --bridge erigon --from 0 --to 24600000 +``` + +### 3. Periodic Status Checks + +Ask Claude to check status: +``` +Check on the sync - list supervised processes and show recent output from phaser-query +``` + +Claude will: +``` + +{"name": "supervisor_list", "args": {}} + + + +{"name": "supervisor_output", "args": {"process_id": "query-id", "limit": 30}} + +``` + +### 4. Error Investigation + +If errors occur: +``` + +{"name": "supervisor_output", "args": { + "process_id": "bridge-id", + "limit": 100 +}} + +``` + +Look for ERROR/WARN entries in the output. + +## Log Entry Format + +Each log entry contains: +- `timestamp`: When the log was captured +- `source`: "stdout", "stderr", or "system" +- `content`: The log line content + +Example: +```json +{ + "timestamp": "2026-03-11T06:30:00Z", + "source": "stdout", + "content": "2026-03-11T06:30:00.123Z INFO phaser_query::sync: Segment 5 complete" +} +``` + +## Integration with Metrics + +While supervisor handles logs, combine with Prometheus metrics for full observability: + +```bash +# Metrics scraping alongside supervisor monitoring +curl -s http://localhost:9092/metrics | grep -E "segment|error|worker" +``` + +## Best Practices + +1. **Name processes clearly** - Use descriptive names like "bridge-archive" not "proc1" +2. **Check output periodically** - Don't wait for failures; proactively monitor +3. **Use working_dir** - Always specify to ensure correct relative paths +4. **Set env vars** - Include RUST_BACKTRACE=1 for debugging +5. **Monitor both services** - Bridge and query are interdependent diff --git a/docs/testing-full-sync.md b/docs/testing-full-sync.md new file mode 100644 index 0000000..8816a12 --- /dev/null +++ b/docs/testing-full-sync.md @@ -0,0 +1,252 @@ +# Full Chain Sync Testing Guide + +This guide documents how to test phaser-query with a full Ethereum mainnet archive sync. + +## Infrastructure Requirements + +### Storage Requirements + +Erigon archive data requires ~2 TB of storage. + +### Server Setup + +The target server needs: +- High I/O throughput (NVMe recommended) +- 3+ TB available storage +- Lighthouse beacon node for consensus layer (erigon runs with `--externalcl`) + +**Directory layout on target server:** +``` +$REMOTE_ROOT/ # Default: /mnt/nvme-raid +├── erigon-customized/ +│ ├── bin/ +│ │ └── erigon # Custom erigon with BlockDataBackend +│ ├── data/ # Erigon archive data (~2 TB) +│ │ └── jwt.hex +│ ├── logs/ +│ │ └── erigon.log +│ └── scripts/ +│ └── start-erigon.sh +│ +├── phaser/ +│ ├── bin/ +│ │ ├── erigon-bridge-{debug,release} +│ │ ├── phaser-query-{debug,release} +│ │ └── phaser-cli-{debug,release} +│ ├── config/ +│ │ └── config.yaml +│ ├── data/ +│ │ └── 1/erigon/ # Parquet output +│ ├── logs/ +│ │ ├── erigon-bridge.log +│ │ └── phaser-query.log +│ └── scripts/ +│ └── run-full-sync.sh +│ +└── lighthouse-beacon/ # Consensus layer (for erigon) +``` + +## Prerequisites + +### 1. Build Binaries + +Build both debug and release versions locally: + +```bash +# Release builds (for performance testing) +cargo build --release -p erigon-bridge -p phaser-query + +# Debug builds (for debugging) +cargo build -p erigon-bridge -p phaser-query +cargo build -p phaser-query --bin phaser-cli + +# Verify debug info present in release builds +file ./target/release/erigon-bridge +# Should show: "with debug_info, not stripped" +``` + +### 2. Deploy to Server + +Use the deploy script: + +```bash +# Deploy everything to a host +./scripts/deploy-superserver.sh myserver + +# Or use environment variable +DEPLOY_HOST=myserver ./scripts/deploy-superserver.sh --all + +# Deploy selectively +./scripts/deploy-superserver.sh myserver --phaser # Just phaser binaries +./scripts/deploy-superserver.sh myserver --erigon # Just custom erigon +./scripts/deploy-superserver.sh myserver --scripts # Just scripts +./scripts/deploy-superserver.sh myserver --config # Just config + +# Custom remote root (default: /mnt/nvme-raid) +REMOTE_ROOT=/data ./scripts/deploy-superserver.sh myserver --all +``` + +### 3. Migrate Existing Erigon Data (if needed) + +If you have existing erigon archive data: + +```bash +# Option 1: Symlink existing data +ssh $HOST "ln -s /path/to/existing/erigon /mnt/nvme-raid/erigon-customized/data" + +# Option 2: Copy JWT if using existing data +ssh $HOST "cp /path/to/existing/jwt.hex /mnt/nvme-raid/erigon-customized/data/" +``` + +## Running the Sync + +### Step 1: Start Erigon (separate process) + +Erigon runs independently from the phaser sync. Start it first: + +```bash +ssh $HOST + +# Start erigon (default path: /mnt/nvme-raid/erigon-customized) +/mnt/nvme-raid/erigon-customized/scripts/start-erigon.sh + +# Or with custom path +ERIGON_ROOT=/data/erigon-customized ./scripts/start-erigon.sh + +# Check status +/mnt/nvme-raid/erigon-customized/scripts/start-erigon.sh --status + +# View logs +tail -f /mnt/nvme-raid/erigon-customized/logs/erigon.log + +# Stop (when done) +/mnt/nvme-raid/erigon-customized/scripts/start-erigon.sh --stop +``` + +The erigon start script: +- Checks binary exists +- Creates JWT secret if missing +- Rotates logs (keeps last 3) +- Runs with archive mode and BlockDataBackend gRPC on port 9091 + +### Step 2: Start Phaser Sync + +Once erigon is running: + +```bash +ssh $HOST +cd /mnt/nvme-raid/phaser + +# Fresh sync with release builds (default) +./scripts/run-full-sync.sh --clean + +# Or with debug builds +./scripts/run-full-sync.sh --clean --debug + +# Resume existing sync +./scripts/run-full-sync.sh --release + +# Custom block range +./scripts/run-full-sync.sh --from 0 --to 10000000 --debug + +# Custom paths via environment +PHASER_ROOT=/data/phaser ERIGON_ROOT=/data/erigon ./scripts/run-full-sync.sh --clean + +# Show help +./scripts/run-full-sync.sh --help +``` + +The sync script handles: +- Log rotation (keeps last 5 runs) +- Data cleanup with `--clean` +- Prerequisite checks (binaries, config, erigon connectivity, disk space) +- Service startup with `tee` for logging +- Optional monitoring loop + +### Stopping Services + +```bash +# Stop phaser (Ctrl+C or) +pkill -f 'erigon-bridge-' && pkill -f 'phaser-query-' + +# Stop erigon (separately) +/mnt/nvme-raid/erigon-customized/scripts/start-erigon.sh --stop +``` + +## Monitoring + +### From Local Machine (via SSH) + +```bash +HOST=myserver + +# Check sync status +ssh $HOST "/mnt/nvme-raid/phaser/bin/phaser-cli-release -e http://127.0.0.1:9093 status" + +# Watch logs +ssh $HOST "tail -f /mnt/nvme-raid/phaser/logs/phaser-query.log" + +# Check erigon status +ssh $HOST "/mnt/nvme-raid/erigon-customized/scripts/start-erigon.sh --status" + +# Check data growth +ssh $HOST "du -sh /mnt/nvme-raid/phaser/data/1/erigon/" + +# Count parquet files +ssh $HOST "ls /mnt/nvme-raid/phaser/data/1/erigon/*.parquet 2>/dev/null | wc -l" + +# Check disk space +ssh $HOST "df -h /mnt/nvme-raid" +``` + +### Prometheus Metrics + +```bash +ssh $HOST "curl -s http://localhost:9092/metrics | grep phaser_query" +``` + +## Troubleshooting + +### "Erigon not reachable" +Erigon must be started separately before running the sync: +```bash +/mnt/nvme-raid/erigon-customized/scripts/start-erigon.sh +``` + +### "no transactions snapshot file" +Erigon is pruned. Need archive node with `--prune.mode=archive`. + +### "erigon-bridge-release not found" +Deploy the binaries with proper suffixes: +```bash +./scripts/deploy-superserver.sh $HOST --phaser +``` + +### Connection refused to bridge +Check erigon-bridge is running: `pgrep -a erigon-bridge` + +### High memory usage +- Reduce `sync_parallelism` in config (default 4) +- Check for stuck workers + +### Disk full +Monitor with `df -h` - need 3+ TB free + +### Logs filling up +Rotate logs: `mv logs/phaser-query.log logs/phaser-query.log.1` + +## Verifying Data Quality + +After sync, verify early blocks have transaction data: + +```bash +ssh $HOST "python3 << 'EOF' +import pyarrow.parquet as pq + +# Check transactions for early blocks +txs = pq.read_table('/mnt/nvme-raid/phaser/data/1/erigon/transactions_from_0_to_499999_0.parquet') +print(f'Transactions: {len(txs)} rows') +print(f'First block with tx: {min(txs[\"block_num\"].to_pylist())}') +# Should be block 46147 (first tx on mainnet) +EOF" +``` diff --git a/scripts/deploy.sh b/scripts/deploy.sh new file mode 100755 index 0000000..b746bc3 --- /dev/null +++ b/scripts/deploy.sh @@ -0,0 +1,275 @@ +#!/bin/bash +# Deploy phaser and custom erigon binaries to a remote server +# +# Usage: ./scripts/deploy.sh [OPTIONS] HOST +# HOST SSH host to deploy to (required) +# --env FILE env.sh file to deploy (required for --config) +# --all Deploy everything +# --phaser Deploy only phaser binaries +# --erigon Deploy only custom erigon +# --scripts Deploy only scripts +# --config Deploy config files (requires --env) +# --help Show this help +# +# Example: +# # Generate env.sh first +# ./scripts/generate-env.sh --phaser-root /mnt/nvme-raid/phaser \ +# --erigon-root /mnt/nvme-raid/erigon-customized -o env.sh +# +# # Then deploy +# ./scripts/deploy.sh myserver --env env.sh --all +set -e + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +REPO_DIR="$(dirname "$SCRIPT_DIR")" +ERIGON_REPO="$REPO_DIR/../erigon" + +# Settings +REMOTE="" +ENV_FILE="" + +# Colors +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' + +log() { echo -e "${GREEN}[deploy]${NC} $1"; } +warn() { echo -e "${YELLOW}[deploy]${NC} $1"; } +error() { echo -e "${RED}[deploy]${NC} $1"; exit 1; } + +# What to deploy +DEPLOY_PHASER=false +DEPLOY_ERIGON=false +DEPLOY_SCRIPTS=false +DEPLOY_CONFIG=false + +# Parse args +POSITIONAL_ARGS=() +while [[ $# -gt 0 ]]; do + case $1 in + --all) + DEPLOY_PHASER=true + DEPLOY_ERIGON=true + DEPLOY_SCRIPTS=true + DEPLOY_CONFIG=true + shift ;; + --phaser) DEPLOY_PHASER=true; shift ;; + --erigon) DEPLOY_ERIGON=true; shift ;; + --scripts) DEPLOY_SCRIPTS=true; shift ;; + --config) DEPLOY_CONFIG=true; shift ;; + --env) ENV_FILE="$2"; shift 2 ;; + --help) + echo "Usage: $0 [OPTIONS] HOST" + echo "" + echo "Arguments:" + echo " HOST SSH host to deploy to (required)" + echo "" + echo "Options:" + echo " --env FILE env.sh file to deploy (required for --config)" + echo " --all Deploy everything" + echo " --phaser Deploy only phaser binaries" + echo " --erigon Deploy only custom erigon" + echo " --scripts Deploy only scripts" + echo " --config Deploy config files (requires --env)" + echo "" + echo "Example:" + echo " # Generate env.sh first" + echo " ./scripts/generate-env.sh --phaser-root /mnt/nvme-raid/phaser \\" + echo " --erigon-root /mnt/nvme-raid/erigon-customized -o env.sh" + echo "" + echo " # Then deploy" + echo " $0 myserver --env env.sh --all" + exit 0 ;; + -*) error "Unknown option: $1" ;; + *) POSITIONAL_ARGS+=("$1"); shift ;; + esac +done + +# Get host from positional arg +if [[ ${#POSITIONAL_ARGS[@]} -gt 0 ]]; then + REMOTE="${POSITIONAL_ARGS[0]}" +fi + +# Require host +if [[ -z "$REMOTE" ]]; then + error "No host specified. Usage: $0 [OPTIONS] HOST" +fi + +# Default to deploying phaser + erigon + scripts if nothing specified +if [[ "$DEPLOY_PHASER" == false && "$DEPLOY_ERIGON" == false && "$DEPLOY_SCRIPTS" == false && "$DEPLOY_CONFIG" == false ]]; then + DEPLOY_PHASER=true + DEPLOY_ERIGON=true + DEPLOY_SCRIPTS=true +fi + +# If deploying config, require env file +if [[ "$DEPLOY_CONFIG" == true && -z "$ENV_FILE" ]]; then + error "--config requires --env FILE. Generate one with: ./scripts/generate-env.sh --help" +fi + +# Load paths from env file +if [[ -n "$ENV_FILE" ]]; then + if [[ ! -f "$ENV_FILE" ]]; then + error "env file not found: $ENV_FILE" + fi + source "$ENV_FILE" +fi + +# Remote paths (from env file or defaults) +PHASER_DIR="${PHASER_ROOT:-/mnt/nvme-raid/phaser}" +ERIGON_DIR="${ERIGON_ROOT:-/mnt/nvme-raid/erigon-customized}" + +# Create remote directories +create_dirs() { + log "Creating remote directories on $REMOTE..." + ssh "$REMOTE" "mkdir -p $ERIGON_DIR/{bin,config,data,logs,scripts}" + ssh "$REMOTE" "mkdir -p $PHASER_DIR/{bin,config,data/1/erigon,logs,scripts}" +} + +# Deploy phaser binaries +deploy_phaser() { + log "Deploying phaser binaries..." + + # Check release builds exist + if [[ ! -f "$REPO_DIR/target/release/erigon-bridge" ]]; then + warn "Release builds not found. Building..." + (cd "$REPO_DIR" && cargo build --release -p erigon-bridge -p phaser-query) + fi + + # Check debug builds exist + if [[ ! -f "$REPO_DIR/target/debug/erigon-bridge" ]]; then + warn "Debug builds not found. Building..." + (cd "$REPO_DIR" && cargo build -p erigon-bridge -p phaser-query) + fi + + # Check CLI + if [[ ! -f "$REPO_DIR/target/debug/phaser-cli" ]]; then + warn "phaser-cli not found. Building..." + (cd "$REPO_DIR" && cargo build -p phaser-query --bin phaser-cli) + fi + + # Copy release binaries + log " erigon-bridge-release" + scp "$REPO_DIR/target/release/erigon-bridge" "$REMOTE:$PHASER_DIR/bin/erigon-bridge-release" + log " phaser-query-release" + scp "$REPO_DIR/target/release/phaser-query" "$REMOTE:$PHASER_DIR/bin/phaser-query-release" + + # Copy debug binaries + log " erigon-bridge-debug" + scp "$REPO_DIR/target/debug/erigon-bridge" "$REMOTE:$PHASER_DIR/bin/erigon-bridge-debug" + log " phaser-query-debug" + scp "$REPO_DIR/target/debug/phaser-query" "$REMOTE:$PHASER_DIR/bin/phaser-query-debug" + + # CLI (same for both, use debug build) + log " phaser-cli-{debug,release}" + scp "$REPO_DIR/target/debug/phaser-cli" "$REMOTE:$PHASER_DIR/bin/phaser-cli-debug" + scp "$REPO_DIR/target/debug/phaser-cli" "$REMOTE:$PHASER_DIR/bin/phaser-cli-release" + + log "Phaser binaries deployed" +} + +# Deploy custom erigon +deploy_erigon() { + log "Deploying custom erigon..." + + if [[ ! -f "$ERIGON_REPO/build/bin/erigon" ]]; then + error "Custom erigon not found at $ERIGON_REPO/build/bin/erigon" + fi + + log " erigon" + scp "$ERIGON_REPO/build/bin/erigon" "$REMOTE:$ERIGON_DIR/bin/" + ssh "$REMOTE" "chmod +x $ERIGON_DIR/bin/erigon" + + log "Custom erigon deployed" +} + +# Deploy scripts +deploy_scripts() { + log "Deploying scripts..." + + # Phaser scripts + log " run-full-sync.sh" + scp "$REPO_DIR/scripts/run-full-sync.sh" "$REMOTE:$PHASER_DIR/scripts/" + ssh "$REMOTE" "chmod +x $PHASER_DIR/scripts/*.sh" + + # Erigon scripts + log " start-erigon.sh" + scp "$REPO_DIR/scripts/erigon/start-erigon.sh" "$REMOTE:$ERIGON_DIR/scripts/" + ssh "$REMOTE" "chmod +x $ERIGON_DIR/scripts/*.sh" + + log "Scripts deployed" +} + +# Deploy config +deploy_config() { + log "Deploying config..." + + # Deploy env.sh to both locations + log " env.sh -> phaser/config/" + scp "$ENV_FILE" "$REMOTE:$PHASER_DIR/config/env.sh" + log " env.sh -> erigon-customized/config/" + scp "$ENV_FILE" "$REMOTE:$ERIGON_DIR/config/env.sh" + + # Deploy phaser config.yaml + log " config.yaml" + ssh "$REMOTE" "cat > $PHASER_DIR/config/config.yaml << EOF +# Phaser Query Configuration +rocksdb_path: $PHASER_DIR/data/rocksdb +data_root: $PHASER_DIR/data + +bridges: + - chain_id: 1 + endpoint: http://127.0.0.1:8090 + name: erigon + +segment_size: 500000 +max_file_size_mb: 1024 +buffer_timeout_secs: 60 + +rpc_port: 8545 +sql_port: 0 +sync_admin_port: 9093 + +sync_parallelism: 4 +EOF" + + log "Config deployed" +} + +# Main +main() { + log "Deploying to $REMOTE..." + log " Phaser: $PHASER_DIR" + log " Erigon: $ERIGON_DIR" + echo "" + + create_dirs + + if [[ "$DEPLOY_ERIGON" == true ]]; then + deploy_erigon + echo "" + fi + + if [[ "$DEPLOY_PHASER" == true ]]; then + deploy_phaser + echo "" + fi + + if [[ "$DEPLOY_SCRIPTS" == true ]]; then + deploy_scripts + echo "" + fi + + if [[ "$DEPLOY_CONFIG" == true ]]; then + deploy_config + echo "" + fi + + log "Deployment complete!" + echo "" + echo "Remote layout:" + ssh "$REMOTE" "ls -la $ERIGON_DIR/bin/ $PHASER_DIR/bin/ 2>/dev/null | head -20" +} + +main "$@" diff --git a/scripts/erigon/start-erigon.sh b/scripts/erigon/start-erigon.sh new file mode 100755 index 0000000..afb0bf8 --- /dev/null +++ b/scripts/erigon/start-erigon.sh @@ -0,0 +1,214 @@ +#!/bin/bash +# Start custom Erigon archive node for phaser sync testing +# Logs to file with rotation +# +# Usage: ./start-erigon.sh [OPTIONS] +# --stop Stop running erigon +# --status Check if running +# --help Show this help +# +# Environment (auto-loaded from config/env.sh if present): +# ERIGON_ROOT Base path (default: /mnt/nvme-raid/erigon-customized) +set -e + +# Find script directory and source env.sh if present +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +if [[ -f "$SCRIPT_DIR/../config/env.sh" ]]; then + source "$SCRIPT_DIR/../config/env.sh" +fi + +# Configurable paths (override via environment or env.sh) +ERIGON_ROOT="${ERIGON_ROOT:-/mnt/nvme-raid/erigon-customized}" + +# Derived paths +BIN_DIR="$ERIGON_ROOT/bin" +LIB_DIR="$ERIGON_ROOT/lib" +DATA_DIR="$ERIGON_ROOT/data" +LOG_DIR="$ERIGON_ROOT/logs" + +# Add lib dir to library path (for libsilkworm_capi.so) +export LD_LIBRARY_PATH="${LIB_DIR}:${LD_LIBRARY_PATH:-}" + +# Ports (avoid conflict with any systemd erigon services) +HTTP_PORT="8546" +WS_PORT="8547" +GRPC_PORT="9091" +AUTH_PORT="8552" +METRICS_PORT="6061" + +# Colors +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' + +log() { echo -e "${GREEN}[$(date +'%Y-%m-%d %H:%M:%S')]${NC} $1"; } +warn() { echo -e "${YELLOW}[$(date +'%Y-%m-%d %H:%M:%S')]${NC} $1"; } +error() { echo -e "${RED}[$(date +'%Y-%m-%d %H:%M:%S')]${NC} $1"; exit 1; } + +# Ensure directories exist +mkdir -p "$LOG_DIR" "$DATA_DIR" + +# Check if erigon is running (our custom one) +is_running() { + pgrep -f "$BIN_DIR/erigon.*--private.api.addr.*$GRPC_PORT" > /dev/null 2>&1 +} + +get_pid() { + pgrep -f "$BIN_DIR/erigon.*--private.api.addr.*$GRPC_PORT" 2>/dev/null || echo "" +} + +# Stop erigon +stop_erigon() { + if is_running; then + local pid=$(get_pid) + log "Stopping erigon (PID: $pid)..." + kill "$pid" 2>/dev/null || true + sleep 3 + if is_running; then + warn "Erigon didn't stop gracefully, forcing..." + kill -9 "$pid" 2>/dev/null || true + sleep 1 + fi + log "Erigon stopped" + else + log "Erigon not running" + fi +} + +# Show status +show_status() { + if is_running; then + local pid=$(get_pid) + echo -e "${GREEN}Erigon is running${NC} (PID: $pid)" + echo "" + echo "Ports:" + echo " HTTP RPC: http://127.0.0.1:$HTTP_PORT" + echo " WebSocket: ws://127.0.0.1:$WS_PORT" + echo " gRPC: 127.0.0.1:$GRPC_PORT" + echo " Metrics: http://127.0.0.1:$METRICS_PORT" + echo "" + echo "Logs: $LOG_DIR/erigon.log" + echo "" + # Show current block + local block=$(curl -s "http://127.0.0.1:$HTTP_PORT" -X POST \ + -H "Content-Type: application/json" \ + -d '{"jsonrpc":"2.0","method":"eth_blockNumber","params":[],"id":1}' 2>/dev/null \ + | python3 -c "import sys,json; print(int(json.load(sys.stdin)['result'],16))" 2>/dev/null || echo "unknown") + echo "Current block: $block" + else + echo -e "${YELLOW}Erigon is not running${NC}" + return 1 + fi +} + +# Rotate logs +rotate_logs() { + local max_rotations=3 + if [[ -f "$LOG_DIR/erigon.log" ]]; then + for i in $(seq $((max_rotations-1)) -1 1); do + if [[ -f "$LOG_DIR/erigon.log.$i" ]]; then + mv "$LOG_DIR/erigon.log.$i" "$LOG_DIR/erigon.log.$((i+1))" + fi + done + mv "$LOG_DIR/erigon.log" "$LOG_DIR/erigon.log.1" + rm -f "$LOG_DIR/erigon.log.$((max_rotations+1))" 2>/dev/null || true + fi +} + +# Start erigon +start_erigon() { + if is_running; then + warn "Erigon already running (PID: $(get_pid))" + show_status + return 0 + fi + + # Check binary exists + [[ -f "$BIN_DIR/erigon" ]] || error "Erigon binary not found at $BIN_DIR/erigon" + [[ -x "$BIN_DIR/erigon" ]] || chmod +x "$BIN_DIR/erigon" + + # Check/create JWT secret + if [[ ! -f "$DATA_DIR/jwt.hex" ]]; then + log "Generating JWT secret..." + openssl rand -hex 32 > "$DATA_DIR/jwt.hex" + fi + + rotate_logs + + log "Starting erigon archive node..." + log " Root: $ERIGON_ROOT" + log " Data dir: $DATA_DIR" + log " gRPC: 0.0.0.0:$GRPC_PORT" + log " HTTP: 0.0.0.0:$HTTP_PORT" + + # Start erigon with all args from systemd service + "$BIN_DIR/erigon" \ + --datadir="$DATA_DIR" \ + --chain=mainnet \ + --prune.mode=archive \ + --authrpc.jwtsecret="$DATA_DIR/jwt.hex" \ + --authrpc.addr=0.0.0.0 \ + --authrpc.port="$AUTH_PORT" \ + --http \ + --http.addr=0.0.0.0 \ + --http.port="$HTTP_PORT" \ + --http.vhosts='*' \ + --http.corsdomain='*' \ + --http.api=eth,debug,net,trace,web3,erigon \ + --ws \ + --ws.port="$WS_PORT" \ + --private.api.addr=0.0.0.0:"$GRPC_PORT" \ + --metrics \ + --metrics.addr=0.0.0.0 \ + --metrics.port="$METRICS_PORT" \ + --torrent.download.rate=62mb \ + --torrent.upload.rate=10mb \ + --maxpeers=50 \ + --db.size.limit=4TB \ + --externalcl \ + 2>&1 | tee "$LOG_DIR/erigon.log" & + + sleep 5 + + if is_running; then + log "Erigon started (PID: $(get_pid))" + echo "" + show_status + else + error "Erigon failed to start. Check $LOG_DIR/erigon.log" + fi +} + +# Parse args +case "${1:-start}" in + --stop|-s) + stop_erigon + ;; + --status|-t) + show_status + ;; + --help|-h) + echo "Usage: $0 [OPTIONS]" + echo "" + echo "Options:" + echo " --stop, -s Stop running erigon" + echo " --status, -t Check if running" + echo " --help, -h Show this help" + echo "" + echo "Environment:" + echo " ERIGON_ROOT Base path (default: /mnt/nvme-raid/erigon-customized)" + echo "" + echo "Paths:" + echo " Binary: $BIN_DIR/erigon" + echo " Data: $DATA_DIR" + echo " Logs: $LOG_DIR" + ;; + start|"") + start_erigon + ;; + *) + echo "Unknown option: $1" + exit 1 + ;; +esac diff --git a/scripts/generate-env.sh b/scripts/generate-env.sh new file mode 100755 index 0000000..2bf7a8f --- /dev/null +++ b/scripts/generate-env.sh @@ -0,0 +1,85 @@ +#!/bin/bash +# Generate env.sh configuration file for remote deployment +# +# Usage: ./scripts/generate-env.sh [OPTIONS] +# --phaser-root PATH Phaser installation path (required) +# --erigon-root PATH Erigon installation path (required) +# --output PATH Output file (default: stdout) +# --help Show this help +# +# Example: +# ./scripts/generate-env.sh \ +# --phaser-root /mnt/nvme-raid/phaser \ +# --erigon-root /mnt/nvme-raid/erigon-customized \ +# --output env.sh +# +# # Then copy to remote +# scp env.sh myserver:/mnt/nvme-raid/phaser/config/ +# ssh myserver "cp /mnt/nvme-raid/phaser/config/env.sh /mnt/nvme-raid/erigon-customized/config/" + +set -e + +PHASER_ROOT="" +ERIGON_ROOT="" +OUTPUT="" + +# Parse args +while [[ $# -gt 0 ]]; do + case $1 in + --phaser-root) PHASER_ROOT="$2"; shift 2 ;; + --erigon-root) ERIGON_ROOT="$2"; shift 2 ;; + --output|-o) OUTPUT="$2"; shift 2 ;; + --help|-h) + echo "Usage: $0 [OPTIONS]" + echo "" + echo "Options:" + echo " --phaser-root PATH Phaser installation path (required)" + echo " --erigon-root PATH Erigon installation path (required)" + echo " --output PATH Output file (default: stdout)" + echo "" + echo "Example:" + echo " $0 --phaser-root /mnt/nvme-raid/phaser --erigon-root /mnt/nvme-raid/erigon-customized -o env.sh" + echo "" + echo " # Then copy to remote" + echo " scp env.sh myserver:/mnt/nvme-raid/phaser/config/" + echo " ssh myserver \"mkdir -p /mnt/nvme-raid/erigon-customized/config && cp /mnt/nvme-raid/phaser/config/env.sh /mnt/nvme-raid/erigon-customized/config/\"" + exit 0 ;; + *) echo "Unknown option: $1"; exit 1 ;; + esac +done + +# Validate required args +if [[ -z "$PHASER_ROOT" ]]; then + echo "Error: --phaser-root is required" >&2 + exit 1 +fi + +if [[ -z "$ERIGON_ROOT" ]]; then + echo "Error: --erigon-root is required" >&2 + exit 1 +fi + +# Generate config +generate_config() { + cat << EOF +# Phaser environment configuration +# Generated by: scripts/generate-env.sh +# Date: $(date -Iseconds) +# +# This file is sourced by run-full-sync.sh and start-erigon.sh +# Place copies in: +# $PHASER_ROOT/config/env.sh +# $ERIGON_ROOT/config/env.sh + +export PHASER_ROOT="$PHASER_ROOT" +export ERIGON_ROOT="$ERIGON_ROOT" +EOF +} + +# Output +if [[ -n "$OUTPUT" ]]; then + generate_config > "$OUTPUT" + echo "Generated: $OUTPUT" >&2 +else + generate_config +fi diff --git a/scripts/run-full-sync.sh b/scripts/run-full-sync.sh new file mode 100755 index 0000000..7ea1909 --- /dev/null +++ b/scripts/run-full-sync.sh @@ -0,0 +1,336 @@ +#!/bin/bash +# Full chain sync automation script +# Runs on the target server, writes to configured data directory +# +# Usage: ./run-full-sync.sh [OPTIONS] +# --from BLOCK Starting block (default: 0) +# --to BLOCK Ending block (default: latest from erigon) +# --clean Remove existing data before starting +# --debug Use debug builds +# --release Use release builds (default) +# --help Show this help +# +# Environment (auto-loaded from config/env.sh if present): +# PHASER_ROOT Base path for phaser (default: /mnt/nvme-raid/phaser) +# ERIGON_ROOT Base path for erigon (default: /mnt/nvme-raid/erigon-customized) +# BUILD_TYPE "release" or "debug" (default: release) +# +# Prerequisites: +# Erigon must be running separately (use start-erigon.sh) +set -e + +# Find script directory and source env.sh if present +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +if [[ -f "$SCRIPT_DIR/../config/env.sh" ]]; then + source "$SCRIPT_DIR/../config/env.sh" +fi + +# Configurable paths (override via environment or env.sh) +PHASER_ROOT="${PHASER_ROOT:-/mnt/nvme-raid/phaser}" +ERIGON_ROOT="${ERIGON_ROOT:-/mnt/nvme-raid/erigon-customized}" + +# Derived paths +BIN_DIR="$PHASER_ROOT/bin" +DATA_DIR="$PHASER_ROOT/data" +CONFIG_FILE="$PHASER_ROOT/config/config.yaml" +LOG_DIR="$PHASER_ROOT/logs" + +# Build type: "release" or "debug" +BUILD_TYPE="${BUILD_TYPE:-release}" + +# Ports +ERIGON_GRPC="127.0.0.1:9091" +ERIGON_HTTP="127.0.0.1:8546" +BRIDGE_PORT="8090" +BRIDGE_METRICS_PORT="9094" +ADMIN_PORT="9093" +METRICS_PORT="9092" + +# Defaults +FROM_BLOCK="${FROM_BLOCK:-0}" +TO_BLOCK="${TO_BLOCK:-}" +CLEAN_DATA=false + +# Parse args +while [[ $# -gt 0 ]]; do + case $1 in + --from) FROM_BLOCK="$2"; shift 2 ;; + --to) TO_BLOCK="$2"; shift 2 ;; + --clean) CLEAN_DATA=true; shift ;; + --debug) BUILD_TYPE="debug"; shift ;; + --release) BUILD_TYPE="release"; shift ;; + --help) + echo "Usage: $0 [OPTIONS]" + echo "" + echo "Options:" + echo " --from BLOCK Starting block (default: 0)" + echo " --to BLOCK Ending block (default: latest from erigon)" + echo " --clean Remove existing data before starting" + echo " --debug Use debug builds" + echo " --release Use release builds (default)" + echo "" + echo "Environment:" + echo " PHASER_ROOT Base path (default: /mnt/nvme-raid/phaser)" + echo " ERIGON_ROOT Erigon path (default: /mnt/nvme-raid/erigon-customized)" + echo " BUILD_TYPE 'release' or 'debug' (default: release)" + echo "" + echo "Paths:" + echo " Binaries: $BIN_DIR/*-{debug,release}" + echo " Data: $DATA_DIR" + echo " Config: $CONFIG_FILE" + echo " Logs: $LOG_DIR" + echo "" + echo "Prerequisites:" + echo " Erigon must be running separately:" + echo " $ERIGON_ROOT/scripts/start-erigon.sh" + exit 0 ;; + *) echo "Unknown option: $1"; exit 1 ;; + esac +done + +# Binary names with build type suffix +ERIGON_BRIDGE="$BIN_DIR/erigon-bridge-$BUILD_TYPE" +PHASER_QUERY="$BIN_DIR/phaser-query-$BUILD_TYPE" +PHASER_CLI="$BIN_DIR/phaser-cli-$BUILD_TYPE" + +# Colors +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' + +log() { echo -e "${GREEN}[$(date +'%Y-%m-%d %H:%M:%S')]${NC} $1"; } +warn() { echo -e "${YELLOW}[$(date +'%Y-%m-%d %H:%M:%S')]${NC} $1"; } +error() { echo -e "${RED}[$(date +'%Y-%m-%d %H:%M:%S')]${NC} $1"; exit 1; } +info() { echo -e "${BLUE}[$(date +'%Y-%m-%d %H:%M:%S')]${NC} $1"; } + +# Ensure directories exist +mkdir -p "$LOG_DIR" "$DATA_DIR/1/erigon" + +# Cleanup function +cleanup() { + log "Stopping phaser processes..." + pkill -f "erigon-bridge-$BUILD_TYPE.*--flight-addr.*$BRIDGE_PORT" 2>/dev/null || true + pkill -f "phaser-query-$BUILD_TYPE.*--metrics-port.*$METRICS_PORT" 2>/dev/null || true +} + +handle_interrupt() { + warn "Interrupted by user" + cleanup + exit 130 +} + +trap handle_interrupt SIGINT SIGTERM + +# Clean data if requested +clean_data() { + log "Cleaning existing data..." + rm -rf "$DATA_DIR/1/erigon/"*.parquet 2>/dev/null || true + rm -rf "$DATA_DIR/1/erigon/"*.tmp 2>/dev/null || true + rm -rf "$DATA_DIR/rocksdb" 2>/dev/null || true + log "Data cleaned" +} + +# Rotate logs +rotate_logs() { + local max_rotations=5 + + for logfile in erigon-bridge.log phaser-query.log; do + if [[ -f "$LOG_DIR/$logfile" ]]; then + # Shift old logs + for i in $(seq $((max_rotations-1)) -1 1); do + if [[ -f "$LOG_DIR/${logfile}.$i" ]]; then + mv "$LOG_DIR/${logfile}.$i" "$LOG_DIR/${logfile}.$((i+1))" + fi + done + mv "$LOG_DIR/$logfile" "$LOG_DIR/${logfile}.1" + fi + done + + # Remove old rotations + for logfile in erigon-bridge.log phaser-query.log; do + rm -f "$LOG_DIR/${logfile}.$((max_rotations+1))" 2>/dev/null || true + done + + log "Rotated logs (keeping last $max_rotations)" +} + +# Check prerequisites +check_prerequisites() { + log "Checking prerequisites..." + log "Build type: $BUILD_TYPE" + + # Check binaries + [[ -f "$ERIGON_BRIDGE" ]] || error "erigon-bridge-$BUILD_TYPE not found at $BIN_DIR" + [[ -f "$PHASER_QUERY" ]] || error "phaser-query-$BUILD_TYPE not found at $BIN_DIR" + [[ -f "$PHASER_CLI" ]] || error "phaser-cli-$BUILD_TYPE not found at $BIN_DIR" + [[ -f "$CONFIG_FILE" ]] || error "Config not found at $CONFIG_FILE" + + # Check erigon is running (must be started separately) + if ! curl -s --connect-timeout 5 "http://$ERIGON_HTTP" -X POST \ + -H "Content-Type: application/json" \ + -d '{"jsonrpc":"2.0","method":"eth_blockNumber","params":[],"id":1}' \ + | grep -q "result"; then + error "Erigon not reachable at $ERIGON_HTTP. Start it first with: $ERIGON_ROOT/scripts/start-erigon.sh" + fi + + # Check disk space + local avail_gb=$(df -BG "$DATA_DIR" | tail -1 | awk '{print $4}' | tr -d 'G') + if [[ $avail_gb -lt 2000 ]]; then + warn "Low disk space: ${avail_gb}GB available (recommend 3TB+)" + fi + + log "Prerequisites OK (${avail_gb}GB available)" +} + +# Get latest block +get_latest_block() { + curl -s "http://$ERIGON_HTTP" -X POST \ + -H "Content-Type: application/json" \ + -d '{"jsonrpc":"2.0","method":"eth_blockNumber","params":[],"id":1}' \ + | python3 -c "import sys,json; print(int(json.load(sys.stdin)['result'],16))" +} + +# Start erigon-bridge with tee +start_bridge() { + log "Starting erigon-bridge-$BUILD_TYPE..." + + pkill -f "erigon-bridge-$BUILD_TYPE.*--flight-addr.*$BRIDGE_PORT" 2>/dev/null || true + sleep 1 + + "$ERIGON_BRIDGE" \ + --erigon-grpc "$ERIGON_GRPC" \ + --flight-addr "0.0.0.0:$BRIDGE_PORT" \ + --metrics-port "$BRIDGE_METRICS_PORT" \ + 2>&1 | tee "$LOG_DIR/erigon-bridge.log" & + BRIDGE_PID=$! + + sleep 5 + + if ! pgrep -f "erigon-bridge-$BUILD_TYPE.*--flight-addr.*$BRIDGE_PORT" > /dev/null; then + error "erigon-bridge failed to start" + fi + + if grep -q "BlockDataBackend service is available" "$LOG_DIR/erigon-bridge.log" 2>/dev/null; then + log "BlockDataBackend connection confirmed" + else + warn "BlockDataBackend not confirmed yet" + fi + + log "erigon-bridge started" +} + +# Start phaser-query with tee +start_query() { + log "Starting phaser-query-$BUILD_TYPE..." + + pkill -f "phaser-query-$BUILD_TYPE.*--metrics-port.*$METRICS_PORT" 2>/dev/null || true + sleep 1 + + rm -f "$DATA_DIR/rocksdb/LOCK" 2>/dev/null + + "$PHASER_QUERY" \ + -c "$CONFIG_FILE" \ + --disable-streaming \ + --metrics-port "$METRICS_PORT" \ + 2>&1 | tee "$LOG_DIR/phaser-query.log" & + QUERY_PID=$! + + sleep 3 + + if ! pgrep -f "phaser-query-$BUILD_TYPE.*--metrics-port.*$METRICS_PORT" > /dev/null; then + error "phaser-query failed to start" + fi + + log "phaser-query started" +} + +# Start sync +start_sync() { + local from=$1 + local to=$2 + + log "Starting sync (blocks $from to $to)..." + + "$PHASER_CLI" -e "http://127.0.0.1:$ADMIN_PORT" sync \ + --chain-id 1 \ + --bridge erigon \ + --from "$from" \ + --to "$to" +} + +# Show status +show_status() { + "$PHASER_CLI" -e "http://127.0.0.1:$ADMIN_PORT" status 2>/dev/null +} + +# Main +main() { + info "========================================" + info " Phaser Full Chain Sync" + info "========================================" + info "Build: $BUILD_TYPE" + info "Phaser root: $PHASER_ROOT" + info "Erigon gRPC: $ERIGON_GRPC" + info "Data dir: $DATA_DIR" + info "Logs: $LOG_DIR" + info "Clean: $CLEAN_DATA" + echo + + check_prerequisites + + if [[ "$CLEAN_DATA" == true ]]; then + clean_data + fi + + rotate_logs + + # Get latest block if not specified + if [[ -z "$TO_BLOCK" ]]; then + TO_BLOCK=$(get_latest_block) + fi + + log "Sync range: $FROM_BLOCK to $TO_BLOCK" + + start_bridge + start_query + + sleep 2 + + start_sync "$FROM_BLOCK" "$TO_BLOCK" + + echo + log "Sync job started!" + echo + info "Monitor with:" + echo " $PHASER_CLI -e http://127.0.0.1:$ADMIN_PORT status" + echo " tail -f $LOG_DIR/phaser-query.log" + echo " du -sh $DATA_DIR/1/erigon/" + echo + info "Running processes:" + pgrep -a "erigon-bridge\|phaser-query" | head -5 + echo + + # Monitoring loop + if [[ -t 0 ]]; then + read -p "Enter monitoring loop? [y/N] " -n 1 -r + echo + if [[ $REPLY =~ ^[Yy]$ ]]; then + while true; do + clear + echo "=== $(date) ===" + echo + show_status || echo "Status unavailable" + echo + echo "Data size: $(du -sh "$DATA_DIR/1/erigon/" 2>/dev/null | cut -f1)" + echo "Disk free: $(df -h "$DATA_DIR" | tail -1 | awk '{print $4}')" + echo + echo "Press Ctrl+C to exit (processes continue in background)" + sleep 30 + done + fi + fi +} + +main "$@"