diff --git a/Cargo.lock b/Cargo.lock index 528244c8..60af43bf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -25,7 +25,7 @@ checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" dependencies = [ "cfg-if", "cipher", - "cpufeatures", + "cpufeatures 0.2.17", ] [[package]] @@ -79,9 +79,9 @@ checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" [[package]] name = "anstream" -version = "0.6.21" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43d5b281e737544384e969a5ccad3f1cdd24b48086a0fc1b2a5262a26b8f4f4a" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" dependencies = [ "anstyle", "anstyle-parse", @@ -94,15 +94,15 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.13" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5192cca8006f1fd4f7237516f40fa183bb07f8fbdfedaa0036de5ea9b0b45e78" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" [[package]] name = "anstyle-parse" -version = "0.2.7" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e7644824f0aa2c7b9384579234ef10eb7efb6a0deb83f9630a49594dd9c15c2" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" dependencies = [ "utf8parse", ] @@ -145,6 +145,12 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" +[[package]] +name = "assert_matches" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b34d609dfbaf33d6889b2b7106d3ca345eacad44200913df5ba02bfd31d2ba9" + [[package]] name = "async-stream" version = "0.3.6" @@ -212,9 +218,9 @@ checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" [[package]] name = "aws-lc-rs" -version = "1.16.1" +version = "1.16.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94bffc006df10ac2a68c83692d734a465f8ee6c5b384d8545a636f81d858f4bf" +checksum = "a054912289d18629dc78375ba2c3726a3afe3ff71b4edba9dedfca0e3446d1fc" dependencies = [ "aws-lc-sys", "zeroize", @@ -222,9 +228,9 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.38.0" +version = "0.39.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4321e568ed89bb5a7d291a7f37997c2c0df89809d7b6d12062c81ddb54aa782e" +checksum = "83a25cf98105baa966497416dbd42565ce3a8cf8dbfd59803ec9ad46f3126399" dependencies = [ "cc", "cmake", @@ -336,6 +342,26 @@ dependencies = [ "num-traits", ] +[[package]] +name = "bincode" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36eaf5d7b090263e8150820482d5d93cd964a81e4019913c972f4edcc6edb740" +dependencies = [ + "bincode_derive", + "serde", + "unty", +] + +[[package]] +name = "bincode_derive" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf95709a440f45e986983918d0e8a1f30a9b1df04918fc828670606804ac3c09" +dependencies = [ + "virtue", +] + [[package]] name = "bitflags" version = "1.3.2" @@ -369,21 +395,21 @@ version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe" dependencies = [ - "digest", + "digest 0.10.7", ] [[package]] name = "blake3" -version = "1.8.3" +version = "1.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2468ef7d57b3fb7e16b576e8377cdbde2320c60e1491e961d11da40fc4f02a2d" +checksum = "4d2d5991425dfd0785aed03aedcf0b321d61975c9b5b3689c774a2610ae0b51e" dependencies = [ "arrayref", "arrayvec", "cc", "cfg-if", "constant_time_eq", - "cpufeatures", + "cpufeatures 0.3.0", ] [[package]] @@ -395,6 +421,15 @@ dependencies = [ "generic-array", ] +[[package]] +name = "block-buffer" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdd35008169921d80bc60d3d0ab416eecb028c4cd653352907921d95084790be" +dependencies = [ + "hybrid-array", +] + [[package]] name = "bumpalo" version = "3.20.2" @@ -446,9 +481,9 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "cc" -version = "1.2.56" +version = "1.2.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aebf35691d1bfb0ac386a69bac2fde4dd276fb618cf8bf4f5318fe285e821bb2" +checksum = "e1e928d4b69e3077709075a938a05ffbedfa53a84c8f766efbf8220bb1ff60e1" dependencies = [ "find-msvc-tools", "jobserver", @@ -468,6 +503,17 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" +[[package]] +name = "chacha20" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f8d983286843e49675a4b7a2d174efe136dc93a18d69130dd18198a6c167601" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "rand_core 0.10.0", +] + [[package]] name = "chrono" version = "0.4.44" @@ -515,15 +561,15 @@ version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" dependencies = [ - "crypto-common", + "crypto-common 0.1.7", "inout", ] [[package]] name = "clap" -version = "4.5.60" +version = "4.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2797f34da339ce31042b27d23607e051786132987f595b02ba4f6a6dffb7030a" +checksum = "b193af5b67834b676abd72466a96c1024e6a6ad978a1f484bd90b85c94041351" dependencies = [ "clap_builder", "clap_derive", @@ -531,9 +577,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.60" +version = "4.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24a241312cea5059b13574bb9b3861cabf758b879c15190b37b6d6fd63ab6876" +checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" dependencies = [ "anstream", "anstyle", @@ -543,9 +589,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.55" +version = "4.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a92793da1a46a5f2a02a6f4c46c6496b28c43638adea8306fcb0caa1634f24e5" +checksum = "1110bd8a634a1ab8cb04345d8d878267d57c3cf1b38d91b71af6686408bbca6a" dependencies = [ "heck", "proc-macro2", @@ -555,24 +601,30 @@ dependencies = [ [[package]] name = "clap_lex" -version = "1.0.0" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a822ea5bc7590f9d40f1ba12c0dc3c2760f3482c6984db1573ad11031420831" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" [[package]] name = "cmake" -version = "0.1.57" +version = "0.1.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75443c44cd6b379beb8c5b45d85d0773baf31cce901fe7bb252f4eff3008ef7d" +checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678" dependencies = [ "cc", ] +[[package]] +name = "cmov" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de0758edba32d61d1fd9f4d69491b47604b91ee2f7e6b33de7e54ca4ebe55dc3" + [[package]] name = "colorchoice" -version = "1.0.4" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" [[package]] name = "concurrent-queue" @@ -589,6 +641,12 @@ version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" +[[package]] +name = "const-oid" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" + [[package]] name = "const_format" version = "0.2.35" @@ -668,6 +726,15 @@ dependencies = [ "libc", ] +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + [[package]] name = "crc" version = "3.4.0" @@ -791,6 +858,15 @@ dependencies = [ "typenum", ] +[[package]] +name = "crypto-common" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77727bb15fa921304124b128af125e7e3b968275d1b108b379190264f4423710" +dependencies = [ + "hybrid-array", +] + [[package]] name = "ctr" version = "0.9.2" @@ -800,11 +876,20 @@ dependencies = [ "cipher", ] +[[package]] +name = "ctutils" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1005a6d4446f5120ef475ad3d2af2b30c49c2c9c6904258e3bb30219bebed5e4" +dependencies = [ + "cmov", +] + [[package]] name = "darling" -version = "0.21.3" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cdf337090841a411e2a7f3deb9187445851f91b309c0c0a29e05f74a00a48c0" +checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d" dependencies = [ "darling_core", "darling_macro", @@ -812,11 +897,10 @@ dependencies = [ [[package]] name = "darling_core" -version = "0.21.3" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1247195ecd7e3c85f83c8d2a366e4210d588e802133e1e355180a9870b517ea4" +checksum = "9865a50f7c335f53564bb694ef660825eb8610e0a53d3e11bf1b0d3df31e03b0" dependencies = [ - "fnv", "ident_case", "proc-macro2", "quote", @@ -826,15 +910,27 @@ dependencies = [ [[package]] name = "darling_macro" -version = "0.21.3" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d38308df82d1080de0afee5d069fa14b0326a88c14f15c5ccda35b4a6c414c81" +checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" dependencies = [ "darling_core", "quote", "syn", ] +[[package]] +name = "darrentsung_debug_parser" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf488eca7807ce3c8e64bee95c3fbf8f1935c905b3b73835e75db16fc458fdc4" +dependencies = [ + "anyhow", + "html-escape", + "nom", + "ordered-float", +] + [[package]] name = "data-encoding" version = "2.10.0" @@ -847,7 +943,7 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef552e6f588e446098f6ba40d89ac146c8c7b64aade83c051ee00bb5d2bc18d" dependencies = [ - "uuid 1.22.0", + "uuid 1.23.0", ] [[package]] @@ -856,7 +952,7 @@ version = "0.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" dependencies = [ - "const-oid", + "const-oid 0.9.6", "pem-rfc7468", "zeroize", ] @@ -871,18 +967,36 @@ dependencies = [ "serde_core", ] +[[package]] +name = "diff" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8" + [[package]] name = "digest" version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ - "block-buffer", - "const-oid", - "crypto-common", + "block-buffer 0.10.4", + "const-oid 0.9.6", + "crypto-common 0.1.7", "subtle", ] +[[package]] +name = "digest" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4850db49bf08e663084f7fb5c87d202ef91a3907271aff24a94eb97ff039153c" +dependencies = [ + "block-buffer 0.12.0", + "const-oid 0.10.2", + "crypto-common 0.2.1", + "ctutils", +] + [[package]] name = "dirs" version = "4.0.0" @@ -1001,15 +1115,15 @@ checksum = "1fda3bf123be441da5260717e0661c25a2fd9cb2b2c1d20bf2e05580047158ab" dependencies = [ "aes", "ctr", - "digest", + "digest 0.10.7", "hex", - "hmac", + "hmac 0.12.1", "pbkdf2", "rand 0.8.5", "scrypt", "serde", "serde_json", - "sha2", + "sha2 0.10.9", "sha3", "thiserror 1.0.69", "uuid 0.8.2", @@ -1339,6 +1453,7 @@ dependencies = [ "cfg-if", "libc", "r-efi 6.0.0", + "rand_core 0.10.0", "wasip2", "wasip3", ] @@ -1456,7 +1571,7 @@ version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b5f8eb2ad728638ea2c7d47a21db23b7b58a72ed6a38256b8a1849f15fbbdf7" dependencies = [ - "hmac", + "hmac 0.12.1", ] [[package]] @@ -1465,7 +1580,16 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" dependencies = [ - "digest", + "digest 0.10.7", +] + +[[package]] +name = "hmac" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6303bc9732ae41b04cb554b844a762b4115a61bfaa81e3e83050991eeb56863f" +dependencies = [ + "digest 0.11.2", ] [[package]] @@ -1477,6 +1601,15 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "html-escape" +version = "0.2.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d1ad449764d627e22bfd7cd5e8868264fc9236e07c752972b4080cd351cb476" +dependencies = [ + "utf8-width", +] + [[package]] name = "http" version = "1.4.0" @@ -1528,11 +1661,20 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" +[[package]] +name = "hybrid-array" +version = "0.4.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3944cf8cf766b40e2a1a333ee5e9b563f854d5fa49d6a8ca2764e97c6eddb214" +dependencies = [ + "typenum", +] + [[package]] name = "hyper" -version = "1.8.1" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ab2d4f250c3d7b1c9fcdff1cece94ea4e2dfbec68614f7b87cb205f24ca9d11" +checksum = "6299f016b246a94207e63da54dbe807655bf9e00044f73ded42c3ac5305fbcca" dependencies = [ "atomic-waker", "bytes", @@ -1545,7 +1687,6 @@ dependencies = [ "httpdate", "itoa", "pin-project-lite", - "pin-utils", "smallvec", "tokio", "want", @@ -1649,12 +1790,13 @@ dependencies = [ [[package]] name = "icu_collections" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c6b649701667bbe825c3b7e6388cb521c23d88644678e83c0c4d0a621a34b43" +checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" dependencies = [ "displaydoc", "potential_utf", + "utf8_iter", "yoke", "zerofrom", "zerovec", @@ -1662,9 +1804,9 @@ dependencies = [ [[package]] name = "icu_locale_core" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edba7861004dd3714265b4db54a3c390e880ab658fec5f7db895fae2046b5bb6" +checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" dependencies = [ "displaydoc", "litemap", @@ -1675,9 +1817,9 @@ dependencies = [ [[package]] name = "icu_normalizer" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f6c8828b67bf8908d82127b2054ea1b4427ff0230ee9141c54251934ab1b599" +checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" dependencies = [ "icu_collections", "icu_normalizer_data", @@ -1689,15 +1831,15 @@ dependencies = [ [[package]] name = "icu_normalizer_data" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7aedcccd01fc5fe81e6b489c15b247b8b0690feb23304303a9e560f37efc560a" +checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" [[package]] name = "icu_properties" -version = "2.1.2" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "020bfc02fe870ec3a66d93e677ccca0562506e5872c650f893269e08615d74ec" +checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" dependencies = [ "icu_collections", "icu_locale_core", @@ -1709,15 +1851,15 @@ dependencies = [ [[package]] name = "icu_properties_data" -version = "2.1.2" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "616c294cf8d725c6afcd8f55abc17c56464ef6211f9ed59cccffe534129c77af" +checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" [[package]] name = "icu_provider" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85962cf0ce02e1e0a629cc34e7ca3e373ce20dda4c4d7294bbd0bf1fdb59e614" +checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" dependencies = [ "displaydoc", "icu_locale_core", @@ -1918,9 +2060,9 @@ checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" [[package]] name = "iri-string" -version = "0.7.10" +version = "0.7.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c91338f0783edbd6195decb37bae672fd3b165faffb89bf7b9e6942f8b1a731a" +checksum = "25e659a4bb38e810ebc252e53b5814ff908a8c58c2a9ce2fae1bbec24cbf4e20" dependencies = [ "memchr", "serde", @@ -1972,9 +2114,9 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.17" +version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92ecc6618181def0457392ccd0ee51198e065e016d1d527a7ac1b6dc7c1f09d2" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" [[package]] name = "jobserver" @@ -1988,10 +2130,12 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.91" +version = "0.3.94" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b49715b7073f385ba4bc528e5747d02e66cb39c6146efb66b781f131f0fb399c" +checksum = "2e04e2ef80ce82e13552136fabeef8a5ed1f985a96805761cbb9a2c34e7664d9" dependencies = [ + "cfg-if", + "futures-util", "once_cell", "wasm-bindgen", ] @@ -2002,7 +2146,7 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cb26cec98cce3a3d96cbb7bced3c4b16e3d13f27ec56dbd62cbc8f39cfb9d653" dependencies = [ - "cpufeatures", + "cpufeatures 0.2.17", ] [[package]] @@ -2015,7 +2159,7 @@ dependencies = [ "rand 0.8.5", "rand_chacha 0.3.1", "serde", - "sha2", + "sha2 0.10.9", "sha3", ] @@ -2051,9 +2195,9 @@ checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" [[package]] name = "libc" -version = "0.2.182" +version = "0.2.184" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6800badb6cb2082ffd7b6a67e6125bb39f18782f793520caee8cb8846be06112" +checksum = "48f5d2a454e16a5ea0f4ced81bd44e4cfc7bd3a507b61887c99fd3538b28e4af" [[package]] name = "libm" @@ -2063,9 +2207,9 @@ checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" [[package]] name = "libredox" -version = "0.1.14" +version = "0.1.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1744e39d1d6a9948f4f388969627434e31128196de472883b39f148769bfe30a" +checksum = "7ddbf48fd451246b1f8c2610bd3b4ac0cc6e149d89832867093ab69a17194f08" dependencies = [ "bitflags 2.11.0", "libc", @@ -2092,9 +2236,9 @@ checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" [[package]] name = "litemap" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6373607a59f0be73a39b6fe456b8192fcc3585f602af20751600e974dd455e77" +checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" [[package]] name = "lock_api" @@ -2139,7 +2283,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" dependencies = [ "cfg-if", - "digest", + "digest 0.10.7", +] + +[[package]] +name = "md-5" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69b6441f590336821bb897fb28fc622898ccceb1d6cea3fde5ea86b090c4de98" +dependencies = [ + "cfg-if", + "digest 0.11.2", ] [[package]] @@ -2220,6 +2374,12 @@ dependencies = [ "unicase", ] +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + [[package]] name = "miniz_oxide" version = "0.8.9" @@ -2232,9 +2392,9 @@ dependencies = [ [[package]] name = "mio" -version = "1.1.1" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a69bcab0ad47271a0234d9422b131806bf3968021e5dc9328caf2d4cd58557fc" +checksum = "50b7e5b27aa02a74bac8c3f23f448f8d87ff11f92d3aac1a6ed369ee08cc56c1" dependencies = [ "libc", "wasi 0.11.1+wasi-snapshot-preview1", @@ -2275,6 +2435,16 @@ dependencies = [ "libc", ] +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + [[package]] name = "nu-ansi-term" version = "0.50.3" @@ -2312,9 +2482,9 @@ dependencies = [ [[package]] name = "num-conv" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf97ec579c3c42f953ef76dbf8d55ac91fb219dde70e49aa4a6b7d74e9919050" +checksum = "c6673768db2d862beb9b39a78fdcb1a69439615d5794a1be50caa9bc92c81967" [[package]] name = "num-format" @@ -2385,9 +2555,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.21.3" +version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "once_cell_polyfill" @@ -2403,9 +2573,9 @@ checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" [[package]] name = "openssl" -version = "0.10.75" +version = "0.10.76" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08838db121398ad17ab8531ce9de97b244589089e290a384c900cb9ff7434328" +checksum = "951c002c75e16ea2c65b8c7e4d3d51d5530d8dfa7d060b4776828c88cfb18ecf" dependencies = [ "bitflags 2.11.0", "cfg-if", @@ -2435,9 +2605,9 @@ checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" [[package]] name = "openssl-sys" -version = "0.9.111" +version = "0.9.112" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82cab2d520aa75e3c58898289429321eb788c3106963d0dc886ec7a5f4adc321" +checksum = "57d55af3b3e226502be1526dfdba67ab0e9c96fc293004e79576b2b9edb0dbdb" dependencies = [ "cc", "libc", @@ -2445,6 +2615,15 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "ordered-float" +version = "2.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68f19d67e5a2795c94e73e0bb1cc1a7edeb2e28efd39e2e1c9b7a40c1108b11c" +dependencies = [ + "num-traits", +] + [[package]] name = "parity-scale-codec" version = "3.7.5" @@ -2514,7 +2693,7 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "83a0692ec44e4cf1ef28ca317f14f8f07da2d95ec3fa01f86e4467b725e60917" dependencies = [ - "digest", + "digest 0.10.7", ] [[package]] @@ -2597,12 +2776,6 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" -[[package]] -name = "pin-utils" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" - [[package]] name = "pkcs1" version = "0.7.5" @@ -2672,27 +2845,27 @@ checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" [[package]] name = "postgres-protocol" -version = "0.6.10" +version = "0.6.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ee9dd5fe15055d2b6806f4736aa0c9637217074e224bbec46d4041b91bb9491" +checksum = "56201207dac53e2f38e848e31b4b91616a6bb6e0c7205b77718994a7f49e70fc" dependencies = [ "base64 0.22.1", "byteorder", "bytes", "fallible-iterator 0.2.0", - "hmac", - "md-5", + "hmac 0.13.0", + "md-5 0.11.0", "memchr", - "rand 0.9.2", - "sha2", + "rand 0.10.0", + "sha2 0.11.0", "stringprep", ] [[package]] name = "postgres-types" -version = "0.2.12" +version = "0.2.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54b858f82211e84682fecd373f68e1ceae642d8d751a1ebd13f33de6257b3e20" +checksum = "8dc729a129e682e8d24170cd30ae1aa01b336b096cbb56df6d534ffec133d186" dependencies = [ "bytes", "fallible-iterator 0.2.0", @@ -2701,9 +2874,9 @@ dependencies = [ [[package]] name = "potential_utf" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b73949432f5e2a09657003c25bca5e19a0e9c84f8058ca374f49e0ebe605af77" +checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" dependencies = [ "zerovec", ] @@ -2732,7 +2905,7 @@ dependencies = [ "prost 0.12.6", "prost-build 0.12.6", "prost-derive 0.12.6", - "sha2", + "sha2 0.10.9", "smallvec", "symbolic-demangle", "tempfile", @@ -2748,6 +2921,26 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "pretty_assertions" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ae130e2f271fbc2ac3a40fb1d07180839cdbbe443c7a27e1e3c13c5cac0116d" +dependencies = [ + "diff", + "yansi", +] + +[[package]] +name = "pretty_assertions_sorted" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa95d32882f2adbdfd30312733271b83c527ee8007bf78dc21afe510463ac6a0" +dependencies = [ + "darrentsung_debug_parser", + "pretty_assertions", +] + [[package]] name = "prettyplease" version = "0.2.37" @@ -2952,9 +3145,9 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.13" +version = "0.11.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1906b49b0c3bc04b5fe5d86a77925ae6524a19b816ae38ce1e426255f1d8a31" +checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098" dependencies = [ "bytes", "getrandom 0.3.4", @@ -3033,6 +3226,17 @@ dependencies = [ "rand_core 0.9.5", ] +[[package]] +name = "rand" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc266eb313df6c5c09c1c7b1fbe2510961e5bcd3add930c1e31f7ed9da0feff8" +dependencies = [ + "chacha20", + "getrandom 0.4.2", + "rand_core 0.10.0", +] + [[package]] name = "rand_chacha" version = "0.3.1" @@ -3071,6 +3275,12 @@ dependencies = [ "getrandom 0.3.4", ] +[[package]] +name = "rand_core" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c8d0fd677905edcbeedbf2edb6494d676f0e98d54d5cf9bda0b061cb8fb8aba" + [[package]] name = "rand_xoshiro" version = "0.7.0" @@ -3246,7 +3456,7 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dd2a808d456c4a54e300a23e9f5a67e122c3024119acbfd73e3bf664491cb2" dependencies = [ - "hmac", + "hmac 0.12.1", "subtle", ] @@ -3289,8 +3499,8 @@ version = "0.9.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8573f03f5883dcaebdfcf4725caa1ecb9c15b2ef50c43a07b816e06799bb12d" dependencies = [ - "const-oid", - "digest", + "const-oid 0.9.6", + "digest 0.10.7", "num-bigint-dig", "num-integer", "num-traits", @@ -3325,9 +3535,9 @@ checksum = "b50b8869d9fc858ce7266cce0194bd74df58b9d0e3f6df3a9fc8eb470d95c09d" [[package]] name = "rustc-hash" -version = "2.1.1" +version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" +checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" [[package]] name = "rustc-hex" @@ -3397,9 +3607,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.103.9" +version = "0.103.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7df23109aa6c1567d1c575b9952556388da57401e4ace1d15f79eedad0d8f53" +checksum = "df33b2b81ac578cabaf06b89b0631153a3f416b0a886e8a7a1707fb51abbd1ef" dependencies = [ "aws-lc-rs", "ring", @@ -3439,9 +3649,9 @@ dependencies = [ [[package]] name = "schannel" -version = "0.1.28" +version = "0.1.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "891d81b926048e76efe18581bf793546b4c0eaf8448d72be8de2bbee5fd166e1" +checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939" dependencies = [ "windows-sys 0.61.2", ] @@ -3482,10 +3692,10 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f9e24d2b632954ded8ab2ef9fea0a0c769ea56ea98bddbafbad22caeeadf45d" dependencies = [ - "hmac", + "hmac 0.12.1", "pbkdf2", "salsa20", - "sha2", + "sha2 0.10.9", ] [[package]] @@ -3596,9 +3806,9 @@ dependencies = [ [[package]] name = "serde_with" -version = "3.17.0" +version = "3.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "381b283ce7bc6b476d903296fb59d0d36633652b633b27f64db4fb46dcbfc3b9" +checksum = "dd5414fad8e6907dbdd5bc441a50ae8d6e26151a03b1de04d89a5576de61d01f" dependencies = [ "base64 0.22.1", "chrono", @@ -3615,9 +3825,9 @@ dependencies = [ [[package]] name = "serde_with_macros" -version = "3.17.0" +version = "3.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6d4e30573c8cb306ed6ab1dca8423eec9a463ea0e155f45399455e0368b27e0" +checksum = "d3db8978e608f1fe7357e211969fd9abdcae80bac1ba7a3369bb7eb6b404eb65" dependencies = [ "darling", "proc-macro2", @@ -3632,8 +3842,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" dependencies = [ "cfg-if", - "cpufeatures", - "digest", + "cpufeatures 0.2.17", + "digest 0.10.7", ] [[package]] @@ -3643,8 +3853,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", - "cpufeatures", - "digest", + "cpufeatures 0.2.17", + "digest 0.10.7", +] + +[[package]] +name = "sha2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "digest 0.11.2", ] [[package]] @@ -3653,7 +3874,7 @@ version = "0.10.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "75872d278a8f37ef87fa0ddbda7802605cb18344497949862c0d4dcb291eba60" dependencies = [ - "digest", + "digest 0.10.7", "keccak", ] @@ -3688,15 +3909,15 @@ version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" dependencies = [ - "digest", + "digest 0.10.7", "rand_core 0.6.4", ] [[package]] name = "simd-adler32" -version = "0.3.8" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e320a6c5ad31d271ad523dcf3ad13e2767ad8b1cb8f047f75a8aeaf8da139da2" +checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" [[package]] name = "siphasher" @@ -3804,7 +4025,7 @@ dependencies = [ "rustls", "serde", "serde_json", - "sha2", + "sha2 0.10.9", "smallvec", "thiserror 2.0.18", "tokio", @@ -3842,7 +4063,7 @@ dependencies = [ "quote", "serde", "serde_json", - "sha2", + "sha2 0.10.9", "sqlx-core", "sqlx-mysql", "sqlx-postgres", @@ -3865,7 +4086,7 @@ dependencies = [ "byteorder", "bytes", "crc", - "digest", + "digest 0.10.7", "dotenvy", "either", "futures-channel", @@ -3875,10 +4096,10 @@ dependencies = [ "generic-array", "hex", "hkdf", - "hmac", + "hmac 0.12.1", "itoa", "log", - "md-5", + "md-5 0.10.6", "memchr", "once_cell", "percent-encoding", @@ -3886,7 +4107,7 @@ dependencies = [ "rsa", "serde", "sha1", - "sha2", + "sha2 0.10.9", "smallvec", "sqlx-core", "stringprep", @@ -3914,18 +4135,18 @@ dependencies = [ "futures-util", "hex", "hkdf", - "hmac", + "hmac 0.12.1", "home", "itoa", "log", - "md-5", + "md-5 0.10.6", "memchr", "num-bigint", "once_cell", "rand 0.8.5", "serde", "serde_json", - "sha2", + "sha2 0.10.9", "smallvec", "sqlx-core", "stringprep", @@ -4052,12 +4273,12 @@ checksum = "1004a16c25dc6113c19d4f9d0c19ff97d85804829894bba22c0d0e9e7b249812" dependencies = [ "crypto-bigint", "hex", - "hmac", + "hmac 0.12.1", "num-bigint", "num-integer", "num-traits", "rfc6979", - "sha2", + "sha2 0.10.9", "starknet-curve", "starknet-types-core", "zeroize", @@ -4127,7 +4348,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "90d23b1bc014ee4cce40056ab3114bcbcdc2dbc1e845bbfb1f8bd0bab63507d4" dependencies = [ "blake2", - "digest", + "digest 0.10.7", "lambdaworks-crypto", "lambdaworks-math", "num-bigint", @@ -4175,21 +4396,21 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] name = "symbolic-common" -version = "12.17.2" +version = "12.17.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "751a2823d606b5d0a7616499e4130a516ebd01a44f39811be2b9600936509c23" +checksum = "52ca086c1eb5c7ee74b151ba83c6487d5d33f8c08ad991b86f3f58f6629e68d5" dependencies = [ "debugid", "memmap2", "stable_deref_trait", - "uuid 1.22.0", + "uuid 1.23.0", ] [[package]] name = "symbolic-demangle" -version = "12.17.2" +version = "12.17.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79b237cfbe320601dd24b4ac817a5b68bb28f5508e33f08d42be0682cadc8ac9" +checksum = "baa911a28a62823aaf2cc2e074212492a3ee69d0d926cc8f5b12b4a108ff5c0c" dependencies = [ "cpp_demangle", "rustc-demangle", @@ -4256,9 +4477,9 @@ checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" [[package]] name = "tempfile" -version = "3.26.0" +version = "3.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82a72c767771b47409d2345987fda8628641887d5466101319899796367354a0" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", "getrandom 0.4.2", @@ -4358,9 +4579,9 @@ dependencies = [ [[package]] name = "tinystr" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42d3e9c45c09de15d06dd8acf5f4e0e399e85927b7f00711024eb7ae10fa4869" +checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" dependencies = [ "displaydoc", "zerovec", @@ -4378,9 +4599,9 @@ dependencies = [ [[package]] name = "tinyvec" -version = "1.10.0" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa5fdc3bce6191a1dbc8c02d5c8bffcf557bafa17c124c5264a458f1b0613fa" +checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3" dependencies = [ "tinyvec_macros", ] @@ -4431,9 +4652,9 @@ dependencies = [ [[package]] name = "tokio-postgres" -version = "0.7.16" +version = "0.7.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcea47c8f71744367793f16c2db1f11cb859d28f436bdb4ca9193eb1f787ee42" +checksum = "4dd8df5ef180f6364759a6f00f7aadda4fbbac86cdee37480826a6ff9f3574ce" dependencies = [ "async-trait", "byteorder", @@ -4448,7 +4669,7 @@ dependencies = [ "pin-project-lite", "postgres-protocol", "postgres-types", - "rand 0.9.2", + "rand 0.10.0", "socket2 0.6.3", "tokio", "tokio-util", @@ -4504,18 +4725,18 @@ dependencies = [ [[package]] name = "toml_datetime" -version = "1.0.0+spec-1.1.0" +version = "1.1.1+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32c2555c699578a4f59f0cc68e5116c8d7cabbd45e1409b989d4be085b53f13e" +checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7" dependencies = [ "serde_core", ] [[package]] name = "toml_edit" -version = "0.25.4+spec-1.1.0" +version = "0.25.10+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7193cbd0ce53dc966037f54351dbbcf0d5a642c7f0038c382ef9e677ce8c13f2" +checksum = "a82418ca169e235e6c399a84e395ab6debeb3bc90edc959bf0f48647c6a32d1b" dependencies = [ "indexmap 2.13.0", "toml_datetime", @@ -4525,9 +4746,9 @@ dependencies = [ [[package]] name = "toml_parser" -version = "1.0.9+spec-1.1.0" +version = "1.1.2+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "702d4415e08923e7e1ef96cd5727c0dfed80b4d2fa25db9647fe5eb6f7c5a4c4" +checksum = "a2abe9b86193656635d2411dc43050282ca48aa31c2451210f4202550afb7526" dependencies = [ "winnow", ] @@ -4686,6 +4907,7 @@ dependencies = [ "torii-erc721", "torii-introspect-postgres-sink", "torii-introspect-sqlite-sink", + "torii-pathfinder", "torii-runtime-common", "torii-sqlite", "tracing", @@ -5101,6 +5323,23 @@ dependencies = [ "tracing", ] +[[package]] +name = "torii-pathfinder" +version = "0.1.0" +dependencies = [ + "anyhow", + "async-trait", + "bincode", + "rand 0.8.5", + "rusqlite", + "serde", + "starknet", + "thiserror 2.0.18", + "torii", + "torii-starknet", + "zstd", +] + [[package]] name = "torii-postgres" version = "0.1.0" @@ -5167,6 +5406,18 @@ dependencies = [ "torii-common", ] +[[package]] +name = "torii-starknet" +version = "0.1.0" +dependencies = [ + "assert_matches", + "bincode", + "pretty_assertions_sorted", + "serde", + "serde_json", + "starknet", +] + [[package]] name = "torii-test-utils" version = "0.1.0" @@ -5365,9 +5616,9 @@ dependencies = [ [[package]] name = "tracing-subscriber" -version = "0.3.22" +version = "0.3.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f30143827ddab0d256fd843b7a66d164e9f271cfa0dde49142c5ca0ca291f1e" +checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" dependencies = [ "matchers", "nu-ansi-term", @@ -5470,9 +5721,9 @@ checksum = "7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d" [[package]] name = "unicode-segmentation" -version = "1.12.0" +version = "1.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" +checksum = "9629274872b2bfaf8d66f5f15725007f635594914870f65218920345aa11aa8c" [[package]] name = "unicode-xid" @@ -5486,6 +5737,12 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" +[[package]] +name = "unty" +version = "0.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d49784317cd0d1ee7ec5c716dd598ec5b4483ea832a2dced265471cc0f690ae" + [[package]] name = "url" version = "2.5.8" @@ -5510,6 +5767,12 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" +[[package]] +name = "utf8-width" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1292c0d970b54115d14f2492fe0170adf21d68a1de108eebc51c1df4f346a091" + [[package]] name = "utf8_iter" version = "1.0.4" @@ -5534,9 +5797,9 @@ dependencies = [ [[package]] name = "uuid" -version = "1.22.0" +version = "1.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a68d3c8f01c0cfa54a75291d83601161799e4a89a39e0929f4b0354d88757a37" +checksum = "5ac8b6f42ead25368cf5b098aeb3dc8a1a2c05a3eee8a9a1a68c640edbfc79d9" dependencies = [ "js-sys", "wasm-bindgen", @@ -5560,6 +5823,12 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "virtue" +version = "0.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "051eb1abcf10076295e815102942cc58f9d5e3b4560e46e53c21e8ff6f3af7b1" + [[package]] name = "walkdir" version = "2.5.0" @@ -5629,9 +5898,9 @@ dependencies = [ [[package]] name = "wasm-bindgen" -version = "0.2.114" +version = "0.2.117" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6532f9a5c1ece3798cb1c2cfdba640b9b3ba884f5db45973a6f442510a87d38e" +checksum = "0551fc1bb415591e3372d0bc4780db7e587d84e2a7e79da121051c5c4b89d0b0" dependencies = [ "cfg-if", "once_cell", @@ -5642,23 +5911,19 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.64" +version = "0.4.67" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9c5522b3a28661442748e09d40924dfb9ca614b21c00d3fd135720e48b67db8" +checksum = "03623de6905b7206edd0a75f69f747f134b7f0a2323392d664448bf2d3c5d87e" dependencies = [ - "cfg-if", - "futures-util", "js-sys", - "once_cell", "wasm-bindgen", - "web-sys", ] [[package]] name = "wasm-bindgen-macro" -version = "0.2.114" +version = "0.2.117" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18a2d50fcf105fb33bb15f00e7a77b772945a2ee45dcf454961fd843e74c18e6" +checksum = "7fbdf9a35adf44786aecd5ff89b4563a90325f9da0923236f6104e603c7e86be" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -5666,9 +5931,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.114" +version = "0.2.117" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03ce4caeaac547cdf713d280eda22a730824dd11e6b8c3ca9e42247b25c631e3" +checksum = "dca9693ef2bab6d4e6707234500350d8dad079eb508dca05530c85dc3a529ff2" dependencies = [ "bumpalo", "proc-macro2", @@ -5679,9 +5944,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.114" +version = "0.2.117" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75a326b8c223ee17883a4251907455a2431acc2791c98c26279376490c378c16" +checksum = "39129a682a6d2d841b6c429d0c51e5cb0ed1a03829d8b3d1e69a011e62cb3d3b" dependencies = [ "unicode-ident", ] @@ -5722,9 +5987,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.91" +version = "0.3.94" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "854ba17bb104abfb26ba36da9729addc7ce7f06f5c0f90f3c391f8461cca21f9" +checksum = "cd70027e39b12f0849461e08ffc50b9cd7688d942c1c8e3c7b22273236b4dd0a" dependencies = [ "js-sys", "wasm-bindgen", @@ -6106,9 +6371,9 @@ checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" [[package]] name = "winnow" -version = "0.7.15" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945" +checksum = "09dac053f1cd375980747450bfc7250c264eaae0583872e845c0c7cd578872b5" dependencies = [ "memchr", ] @@ -6222,11 +6487,17 @@ version = "0.8.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fdd20c5420375476fbd4394763288da7eb0cc0b8c11deed431a91562af7335d3" +[[package]] +name = "yansi" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" + [[package]] name = "yoke" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72d6e5c6afb84d73944e5cedb052c4680d5657337201555f9f2a16b7406d4954" +checksum = "abe8c5fda708d9ca3df187cae8bfb9ceda00dd96231bed36e445a1a48e66f9ca" dependencies = [ "stable_deref_trait", "yoke-derive", @@ -6235,9 +6506,9 @@ dependencies = [ [[package]] name = "yoke-derive" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b659052874eb698efe5b9e8cf382204678a0086ebf46982b79d6ca3182927e5d" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" dependencies = [ "proc-macro2", "quote", @@ -6247,18 +6518,18 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.40" +version = "0.8.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a789c6e490b576db9f7e6b6d661bcc9799f7c0ac8352f56ea20193b2681532e5" +checksum = "eed437bf9d6692032087e337407a86f04cd8d6a16a37199ed57949d415bd68e9" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.40" +version = "0.8.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f65c489a7071a749c849713807783f70672b28094011623e200cb86dcb835953" +checksum = "70e3cd084b1788766f53af483dd21f93881ff30d7320490ec3ef7526d203bad4" dependencies = [ "proc-macro2", "quote", @@ -6267,18 +6538,18 @@ dependencies = [ [[package]] name = "zerofrom" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50cc42e0333e05660c3587f3bf9d0478688e15d870fab3346451ce7f8c9fbea5" +checksum = "69faa1f2a1ea75661980b013019ed6687ed0e83d069bc1114e2cc74c6c04c4df" dependencies = [ "zerofrom-derive", ] [[package]] name = "zerofrom-derive" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" +checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" dependencies = [ "proc-macro2", "quote", @@ -6294,9 +6565,9 @@ checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" [[package]] name = "zerotrie" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a59c17a5562d507e4b54960e8569ebee33bee890c70aa3fe7b97e85a9fd7851" +checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" dependencies = [ "displaydoc", "yoke", @@ -6305,9 +6576,9 @@ dependencies = [ [[package]] name = "zerovec" -version = "0.11.5" +version = "0.11.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c28719294829477f525be0186d13efa9a3c602f7ec202ca9e353d310fb9a002" +checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" dependencies = [ "yoke", "zerofrom", @@ -6316,9 +6587,9 @@ dependencies = [ [[package]] name = "zerovec-derive" -version = "0.11.2" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eadce39539ca5cb3985590102671f2567e659fca9666581ad3411d59207951f3" +checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" dependencies = [ "proc-macro2", "quote", @@ -6330,3 +6601,31 @@ name = "zmij" version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" + +[[package]] +name = "zstd" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.16+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/Cargo.toml b/Cargo.toml index 41acb5e4..7043f6e0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,6 +20,8 @@ members = [ "crates/testing", "crates/postgres", "crates/sqlite", + "crates/pathfinder", + "crates/starknet", "bins/torii-erc20", "bins/torii-erc20-synth", "bins/torii-introspect-bin", @@ -122,6 +124,7 @@ dojo-introspect = { git = "https://github.com/dojoengine/dojo-introspect", rev = # Utils resolve-path = "0.1.0" itertools = "0.14.0" +bincode = { version = "2.0.0" } # Torii core torii.path = "." @@ -146,6 +149,8 @@ torii-introspect-sqlite-sink.path = "./crates/introspect-sqlite-sink" torii-test-utils.path = "crates/testing" torii-postgres.path = "crates/postgres" torii-sqlite.path = "crates/sqlite" +torii-pathfinder.path = "crates/pathfinder" +torii-starknet.path = "crates/starknet" [lib] @@ -198,7 +203,11 @@ serde.workspace = true sqlx.workspace = true starknet.workspace = true tokio-stream.workspace = true -tokio-rustls = { version = "0.26", default-features = false, features = ["logging", "tls12", "ring"] } +tokio-rustls = { version = "0.26", default-features = false, features = [ + "logging", + "tls12", + "ring", +] } tokio-util.workspace = true tokio.workspace = true tonic-reflection.workspace = true diff --git a/bins/torii-arcade/Cargo.toml b/bins/torii-arcade/Cargo.toml index d3a56e75..0ec65787 100644 --- a/bins/torii-arcade/Cargo.toml +++ b/bins/torii-arcade/Cargo.toml @@ -23,6 +23,7 @@ torii-introspect-postgres-sink = { path = "../../crates/introspect-postgres-sink torii-introspect-sqlite-sink = { path = "../../crates/introspect-sqlite-sink" } torii-runtime-common.workspace = true torii-sqlite.workspace = true +torii-pathfinder = { workspace = true, features = ["etl"] } tokio = { version = "1", features = ["full"] } tracing = "0.1" @@ -31,7 +32,11 @@ starknet = "0.17" url = "2.5" clap = { version = "4.5", features = ["derive", "env"] } anyhow = "1.0" -sqlx = { version = "0.8", features = ["postgres", "sqlite", "runtime-tokio-rustls"] } +sqlx = { version = "0.8", features = [ + "postgres", + "sqlite", + "runtime-tokio-rustls", +] } tonic.workspace = true tonic-reflection.workspace = true tonic-web.workspace = true diff --git a/bins/torii-arcade/src/config.rs b/bins/torii-arcade/src/config.rs index 55052d51..661d5b1b 100644 --- a/bins/torii-arcade/src/config.rs +++ b/bins/torii-arcade/src/config.rs @@ -204,6 +204,9 @@ pub struct Config { #[arg(long, value_delimiter = ',')] pub historical: Vec, + + #[arg(long)] + pub pathfinder_path: Option, } impl Config { @@ -358,6 +361,10 @@ impl Config { _ => bail!("--tls-cert and --tls-key must be provided together"), } } + + pub fn pathfinder_path(&self) -> Option { + self.pathfinder_path.as_ref().map(PathBuf::from) + } } fn is_postgres_url(url: &str) -> bool { diff --git a/bins/torii-arcade/src/main.rs b/bins/torii-arcade/src/main.rs index 6ab09c01..f2896b91 100644 --- a/bins/torii-arcade/src/main.rs +++ b/bins/torii-arcade/src/main.rs @@ -1,6 +1,6 @@ mod config; -use anyhow::Result; +use anyhow::{Error, Result}; use clap::Parser; use config::{Config, MetadataMode}; use sqlx::postgres::PgPoolOptions; @@ -52,6 +52,7 @@ use torii_erc721::{ }; use torii_introspect_postgres_sink::processor::IntrospectPgDb; use torii_introspect_sqlite_sink::processor::IntrospectSqliteDb; +use torii_pathfinder::extractor::PathfinderCombinedExtractor; use torii_runtime_common::database::{ validate_uniform_backends, DatabaseBackend, DEFAULT_SQLITE_MAX_CONNECTIONS, }; @@ -392,7 +393,7 @@ async fn run_indexer(config: Config) -> Result<()> { &erc721_addresses, &erc1155_addresses, &config, - ); + )?; let (dojo_decoder, introspect_sink): ( Arc, @@ -738,7 +739,7 @@ fn build_extractor( erc721_addresses: &[Felt], erc1155_addresses: &[Felt], config: &Config, -) -> Box { +) -> Result> { let to_block = config.to_block.unwrap_or(u64::MAX); let mut contracts = Vec::new(); let mut seen = HashSet::new(); @@ -771,18 +772,39 @@ fn build_extractor( config.from_block, to_block, ); - - Box::new(EventExtractor::new( + let event_extractor = EventExtractor::new( provider, EventExtractorConfig { - contracts, + contracts: contracts.clone(), chunk_size: config.event_chunk_size, block_batch_size: config.event_block_batch_size, retry_policy: RetryPolicy::default(), ignore_saved_state: config.ignore_saved_state, rpc_parallelism: config.rpc_parallelism, }, - )) + ); + #[allow(clippy::single_match_else)] + match config.pathfinder_path() { + Some(path) => { + tracing::info!( + "Using Pathfinder for event extraction with endpoint: {}", + path.display() + ); + PathfinderCombinedExtractor::new( + path, + config.event_block_batch_size, + config.from_block, + config.to_block.unwrap_or(u64::MAX), + event_extractor, + ) + .map(|e| Box::new(e) as Box) + .map_err(Error::new) + } + None => { + tracing::info!("Using direct RPC for event extraction"); + Ok(Box::new(event_extractor)) + } + } } fn append_unique_contract_configs( diff --git a/bins/torii-introspect-synth/src/main.rs b/bins/torii-introspect-synth/src/main.rs index 9078bd37..9f71e3e6 100644 --- a/bins/torii-introspect-synth/src/main.rs +++ b/bins/torii-introspect-synth/src/main.rs @@ -354,6 +354,10 @@ impl SyntheticIntrospectExtractor { #[async_trait] impl Extractor for SyntheticIntrospectExtractor { + fn set_start_block(&mut self, start_block: u64) { + self.current_block = start_block.max(self.current_block); + } + async fn extract( &mut self, cursor: Option, diff --git a/crates/pathfinder/Cargo.toml b/crates/pathfinder/Cargo.toml new file mode 100644 index 00000000..92bad707 --- /dev/null +++ b/crates/pathfinder/Cargo.toml @@ -0,0 +1,27 @@ +[package] +name = "torii-pathfinder" +version = "0.1.0" +edition = "2021" +description = "Pathfinding utilities for Torii" + +[dependencies] +starknet.workspace = true +rusqlite = { version = "0.32.1", features = ["bundled"] } +zstd = { version = "0.13.2", features = ["experimental"] } +serde.workspace = true +bincode = { workspace = true, features = ["serde"] } +thiserror.workspace = true +rand = "0.8.5" +torii-starknet.workspace = true + +# Etl dependencies +torii = { workspace = true, optional = true } +async-trait = { workspace = true, optional = true } +anyhow = { workspace = true, optional = true } + +[lints] +workspace = true + +[features] +default = ["etl"] +etl = ["torii", "async-trait", "anyhow", "torii-starknet/starknet"] diff --git a/crates/pathfinder/src/assets/events.zdict b/crates/pathfinder/src/assets/events.zdict new file mode 100644 index 00000000..7f4fa284 Binary files /dev/null and b/crates/pathfinder/src/assets/events.zdict differ diff --git a/crates/pathfinder/src/assets/txs.zdict b/crates/pathfinder/src/assets/txs.zdict new file mode 100644 index 00000000..724448c6 Binary files /dev/null and b/crates/pathfinder/src/assets/txs.zdict differ diff --git a/crates/pathfinder/src/decoding.rs b/crates/pathfinder/src/decoding.rs new file mode 100644 index 00000000..eb7cd0d1 --- /dev/null +++ b/crates/pathfinder/src/decoding.rs @@ -0,0 +1,165 @@ +use crate::sqlite::BlockEventsRow; +use crate::{PFError, PFResult}; +use serde::{Deserialize, Serialize}; +use std::cell::RefCell; +use std::fmt::{Formatter, Result as FmtResult}; +use std::io::Result as IoResult; +use std::sync::LazyLock; +use torii_starknet::event::Event; +use torii_starknet::Felt; +use zstd::bulk::Decompressor; + +// Taken from pathfinder-common but with some optimizations + +const MAX_EVENTS_UNCOMPRESSED_SIZE: usize = 128usize * 1024 * 1024; + +static ZSTD_EVENTS_DECODER_DICTIONARY: LazyLock> = + LazyLock::new(|| zstd::dict::DecoderDictionary::new(include_bytes!("assets/events.zdict"))); + +thread_local! { + static EVENT_DECOMPRESSOR: RefCell> = RefCell::new( + Decompressor::with_prepared_dictionary( + &ZSTD_EVENTS_DECODER_DICTIONARY + ).expect("failed to create decompressor") + ); +} + +/// Minimally encoded Felt value. +#[repr(transparent)] +#[derive(Clone, Debug, PartialEq, Eq, Default)] +pub struct MinimalFelt([u8; 32]); + +impl serde::Serialize for MinimalFelt { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + let bytes = self.0; + let zeros = bytes.iter().take_while(|&&x| x == 0).count(); + bytes[zeros..].serialize(serializer) + } +} + +impl<'de> serde::Deserialize<'de> for MinimalFelt { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + struct Visitor; + + impl<'de> serde::de::Visitor<'de> for Visitor { + type Value = MinimalFelt; + + fn expecting(&self, formatter: &mut Formatter<'_>) -> FmtResult { + formatter.write_str("a sequence") + } + + fn visit_seq(self, mut seq: B) -> Result + where + B: serde::de::SeqAccess<'de>, + { + let len = seq.size_hint().unwrap(); + let mut bytes = [0; 32]; + let num_zeros = bytes.len() - len; + let mut i = num_zeros; + while let Some(value) = seq.next_element()? { + bytes[i] = value; + i += 1; + } + Ok(MinimalFelt(bytes)) + } + } + + deserializer.deserialize_seq(Visitor) + } +} + +impl MinimalFelt { + #[allow(unsafe_code)] + fn into_felt_vec(v: Vec) -> Vec { + // Safe: both repr(transparent) over [u8; 32] + unsafe { + let mut v = std::mem::ManuallyDrop::new(v); + Vec::from_raw_parts(v.as_mut_ptr().cast::(), v.len(), v.capacity()) + } + } +} + +impl From for Felt { + fn from(value: MinimalFelt) -> Self { + value.0.into() + } +} + +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub enum EventsForBlock { + V0 { events: Vec> }, +} + +impl EventsForBlock { + pub fn events(self) -> Vec> { + match self { + EventsForBlock::V0 { events } => events + .into_iter() + .map(|evs| evs.into_iter().map(Event::from).collect()) + .collect(), + } + } +} + +pub(crate) fn decompress_events(input: &[u8]) -> IoResult> { + EVENT_DECOMPRESSOR.with(|d| { + d.borrow_mut() + .decompress(input, MAX_EVENTS_UNCOMPRESSED_SIZE) + }) +} + +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct EncodedEvent { + pub data: Vec, + pub from_address: MinimalFelt, + pub keys: Vec, +} + +impl From for Event { + fn from(value: EncodedEvent) -> Self { + Self { + data: MinimalFelt::into_felt_vec(value.data), + from_address: value.from_address.into(), + keys: MinimalFelt::into_felt_vec(value.keys), + } + } +} + +impl BlockEventsRow { + pub fn decompress_events(&self) -> PFResult>> { + match self.events { + None => Ok(Vec::new()), + Some(ref events) => { + let events = decompress_events(events)?; + let events: EventsForBlock = + bincode::serde::decode_from_slice(&events, bincode::config::standard())?.0; + Ok(events.events()) + } + } + } +} + +impl TryFrom for BlockEvents { + type Error = PFError; + + fn try_from(value: BlockEventsRow) -> Result { + Ok(Self { + block_number: value.block_number, + transactions: value.decompress_events()?, + }) + } +} + +#[derive(Debug)] +pub struct BlockEvents { + pub block_number: u64, + pub transactions: Vec>, +} diff --git a/crates/pathfinder/src/error.rs b/crates/pathfinder/src/error.rs new file mode 100644 index 00000000..45dbab12 --- /dev/null +++ b/crates/pathfinder/src/error.rs @@ -0,0 +1,43 @@ +use std::sync::PoisonError; + +use bincode::error::DecodeError; + +#[derive(Debug, thiserror::Error)] +pub enum PFError { + #[error(transparent)] + Decode(#[from] DecodeError), + #[error(transparent)] + Io(#[from] std::io::Error), + #[error(transparent)] + Sqlite(#[from] rusqlite::Error), + #[error("data mismatch: {0}")] + DataMismatch(String), + #[error("Block Hash missing for block {0}")] + BlockHashMissing(u64), + #[error("Block context missing for block {0}")] + BlockContextMissing(u64), + #[error("Lock poisoned: {0}")] + LockPoison(String), +} + +impl From> for PFError { + fn from(err: PoisonError) -> Self { + Self::LockPoison(err.to_string()) + } +} + +pub type PFResult = std::result::Result; + +impl PFError { + pub fn tx_hash_mismatch(block_number: u64) -> Self { + Self::DataMismatch(format!( + "Transactions hash missing for block {block_number}" + )) + } + pub fn block_hash_missing(block_number: u64) -> Self { + Self::BlockHashMissing(block_number) + } + pub fn block_context_missing(block_number: u64) -> Self { + Self::BlockContextMissing(block_number) + } +} diff --git a/crates/pathfinder/src/extractor.rs b/crates/pathfinder/src/extractor.rs new file mode 100644 index 00000000..46b86456 --- /dev/null +++ b/crates/pathfinder/src/extractor.rs @@ -0,0 +1,133 @@ +use crate::{connect, EventFetcher, PFResult}; +use anyhow::Result as AnyResult; +use async_trait::async_trait; +use rusqlite::Connection; +use starknet::core::types::EmittedEvent; +use std::collections::HashMap; +use std::path::Path; +use std::sync::{Arc, Mutex}; +use torii::etl::{BlockContext, EngineDb, ExtractionBatch, Extractor}; + +#[derive(Debug)] +pub struct PathfinderExtractor { + pub conn: Mutex, + pub batch: u64, + pub current: u64, + pub end: u64, + pub finished: bool, +} + +#[derive(Debug)] +pub struct PathfinderCombinedExtractor { + pub pathfinder: PathfinderExtractor, + pub head: T, + pub on_head: bool, +} + +impl PathfinderExtractor { + pub fn new>(path: P, batch: u64, start: u64, end: u64) -> PFResult { + Ok(Self { + conn: connect(path)?.into(), + batch, + current: start, + end, + finished: false, + }) + } + + pub fn next_batch(&mut self) -> PFResult<(Vec, Vec)> { + let mut last = self.current + self.batch; + if last >= self.end { + last = self.end; + self.finished = true; + } + let (blocks, events) = self + .conn + .lock()? + .get_emitted_events_with_context(self.current, last)?; + let last_block = blocks.last().map_or(self.current, |b| b.number); + if last_block < last { + self.finished = true; + } + self.current = last_block + 1; + Ok((blocks, events.into_iter().map(Into::into).collect())) + } +} + +impl PathfinderCombinedExtractor { + pub fn new>( + path: P, + batch: u64, + start: u64, + end: u64, + head: T, + ) -> PFResult { + Ok(Self { + pathfinder: PathfinderExtractor::new(path, batch, start, end)?, + head, + on_head: false, + }) + } +} + +#[async_trait] +impl Extractor for PathfinderExtractor { + fn set_start_block(&mut self, start_block: u64) { + self.current = start_block.max(self.current); + } + async fn extract( + &mut self, + _cursor: Option, + _engine_db: &EngineDb, + ) -> AnyResult { + let (blocks, events) = self.next_batch()?; + let blocks = blocks + .into_iter() + .map(|b| (b.number, Arc::new(b))) + .collect(); + Ok(ExtractionBatch { + events, + blocks, + transactions: HashMap::new(), + declared_classes: Vec::new(), + deployed_contracts: Vec::new(), + cursor: None, + chain_head: None, + }) + } + fn is_finished(&self) -> bool { + self.current >= self.end + } + fn as_any(&self) -> &dyn std::any::Any { + self + } +} + +#[async_trait] +impl Extractor for PathfinderCombinedExtractor { + fn set_start_block(&mut self, start_block: u64) { + self.pathfinder.set_start_block(start_block); + self.head.set_start_block(start_block); + } + async fn extract( + &mut self, + cursor: Option, + engine_db: &EngineDb, + ) -> AnyResult { + if self.on_head { + self.head.extract(cursor, engine_db).await + } else if self.pathfinder.is_finished() { + self.on_head = true; + self.head.set_start_block(self.pathfinder.current); + self.head.extract(cursor, engine_db).await + } else { + self.pathfinder.extract(cursor, engine_db).await + } + } + fn is_finished(&self) -> bool { + self.pathfinder.is_finished() && self.head.is_finished() + } + fn as_any(&self) -> &dyn std::any::Any { + self + } +} diff --git a/crates/pathfinder/src/fetcher.rs b/crates/pathfinder/src/fetcher.rs new file mode 100644 index 00000000..8f64f919 --- /dev/null +++ b/crates/pathfinder/src/fetcher.rs @@ -0,0 +1,174 @@ +use std::collections::HashMap; + +use crate::sqlite::{BlockContextRow, SqliteExt}; +use crate::PFError; +use crate::{decoding::BlockEvents, PFResult}; +use rusqlite::Connection; + +#[cfg(not(feature = "etl"))] +/// Block context information Copied from core to avoid importing the entire crate +#[derive(Debug, Clone, Default)] +pub struct BlockContext { + pub number: u64, + pub hash: Felt, + pub parent_hash: Felt, + pub timestamp: u64, +} + +#[cfg(feature = "etl")] +pub use torii::etl::extractor::BlockContext; +use torii_starknet::event::EmittedEvent; +use torii_starknet::Felt; + +impl From for BlockContext { + fn from(value: BlockContextRow) -> Self { + BlockContext { + number: value.number, + hash: value.hash.into(), + parent_hash: value.parent_hash.into(), + timestamp: value.timestamp, + } + } +} + +pub trait EmittedEventExt { + fn with_block_hash(self, block_hashes: &HashMap) -> PFResult; +} + +impl EmittedEventExt for EmittedEvent { + fn with_block_hash(mut self, block_hashes: &HashMap) -> PFResult { + let block_number = self.block_number.unwrap(); + match block_hashes.get(&block_number) { + Some(block_hash) => self.block_hash = Some(*block_hash), + None => return Err(PFError::block_hash_missing(block_number)), + } + Ok(self) + } +} + +pub trait EventFetcher { + fn get_emitted_events_wo_block_hashes( + &self, + from_block: u64, + to_block: u64, + ) -> PFResult>; + fn get_emitted_events(&self, from_block: u64, to_block: u64) -> PFResult>; + fn get_emitted_events_with_context( + &self, + from_block: u64, + to_block: u64, + ) -> PFResult<(Vec, Vec)>; +} +impl EventFetcher for Connection { + fn get_emitted_events_wo_block_hashes( + &self, + from_block: u64, + to_block: u64, + ) -> PFResult> { + let total_events = self.get_number_of_events_for_blocks(from_block, to_block)?; + let mut tx_hashes = self + .get_block_tx_hash_rows(from_block, to_block)? + .into_iter(); + let mut emitted_events = Vec::with_capacity(total_events as usize); + + for row in self.get_block_events_rows(from_block, to_block)? { + let block: BlockEvents = row.try_into()?; + for transaction in block.transactions { + let transaction_hash = tx_hashes + .next() + .map(|r| r.1) + .ok_or_else(|| PFError::tx_hash_mismatch(block.block_number))?; + for event in transaction { + emitted_events.push(EmittedEvent { + block_hash: None, + block_number: Some(block.block_number), + data: event.data, + from_address: event.from_address, + keys: event.keys, + transaction_hash, + }); + } + } + } + Ok(emitted_events) + } + + fn get_emitted_events(&self, from_block: u64, to_block: u64) -> PFResult> { + let contexts = self.get_block_hash_rows_with_count(from_block, to_block)?; + let total_events: u64 = contexts.iter().map(|r| r.event_count).sum(); + let mut tx_hashes = self + .get_block_tx_hash_rows(from_block, to_block)? + .into_iter(); + let mut hash_rows = contexts.into_iter(); + let mut emitted_events = Vec::with_capacity(total_events as usize); + + for row in self.get_block_events_rows(from_block, to_block)? { + let block: BlockEvents = row.try_into()?; + let block_hash = hash_rows + .next() + .map(|r| r.hash) + .ok_or_else(|| PFError::block_hash_missing(block.block_number))?; + for transaction in block.transactions { + let transaction_hash = tx_hashes + .next() + .map(|r| r.1) + .ok_or_else(|| PFError::tx_hash_mismatch(block.block_number))?; + for event in transaction { + emitted_events.push(EmittedEvent { + block_hash: Some(block_hash), + block_number: Some(block.block_number), + data: event.data, + from_address: event.from_address, + keys: event.keys, + transaction_hash, + }); + } + } + } + Ok(emitted_events) + } + + fn get_emitted_events_with_context( + &self, + from_block: u64, + to_block: u64, + ) -> PFResult<(Vec, Vec)> { + let context_rows = self.get_block_context_rows(from_block, to_block)?; + let total_events: u64 = context_rows.iter().map(|r| r.event_count).sum(); + let mut tx_hashes = self + .get_block_tx_hash_rows(from_block, to_block)? + .into_iter(); + let mut emitted_events = Vec::with_capacity(total_events as usize); + let mut contexts = Vec::with_capacity(context_rows.len()); + let mut ctx_iter = context_rows.into_iter(); + + for row in self.get_block_events_rows(from_block, to_block)? { + let block = BlockEvents::try_from(row)?; + let ctx = ctx_iter + .next() + .ok_or_else(|| PFError::block_context_missing(block.block_number))?; + if ctx.number != block.block_number { + return Err(PFError::block_context_missing(block.block_number)); + } + for transaction in block.transactions { + let transaction_hash = tx_hashes + .next() + .map(|r| r.1) + .ok_or_else(|| PFError::tx_hash_mismatch(block.block_number))?; + for event in transaction { + emitted_events.push(EmittedEvent { + block_hash: Some(ctx.hash), + block_number: Some(block.block_number), + data: event.data, + from_address: event.from_address, + keys: event.keys, + transaction_hash, + }); + } + } + contexts.push(ctx.into()); + } + + Ok((contexts, emitted_events)) + } +} diff --git a/crates/pathfinder/src/lib.rs b/crates/pathfinder/src/lib.rs new file mode 100644 index 00000000..2286741f --- /dev/null +++ b/crates/pathfinder/src/lib.rs @@ -0,0 +1,15 @@ +pub mod decoding; +pub mod error; +pub mod fetcher; +pub mod sqlite; +pub mod utils; + +#[cfg(feature = "etl")] +pub mod extractor; + +#[cfg(test)] +mod test; + +pub use error::{PFError, PFResult}; +pub use fetcher::{BlockContext, EventFetcher}; +pub use utils::connect; diff --git a/crates/pathfinder/src/sqlite.rs b/crates/pathfinder/src/sqlite.rs new file mode 100644 index 00000000..fb71976c --- /dev/null +++ b/crates/pathfinder/src/sqlite.rs @@ -0,0 +1,223 @@ +use rusqlite::params; +use rusqlite::types::ValueRef; +use rusqlite::Connection; +use rusqlite::Error as SqliteError; +use rusqlite::Row; +use torii_starknet::Felt; + +pub type SqliteResult = Result; + +#[derive(Debug)] +pub struct BlockEventsRow { + pub block_number: u64, + pub events: Option>, +} + +#[derive(Debug)] +pub struct NumberAndHashRow(pub u64, pub Felt); + +#[derive(Debug)] +pub struct BlockContextRow { + pub number: u64, + pub hash: Felt, + pub parent_hash: Felt, + pub timestamp: u64, + pub event_count: u64, +} + +#[derive(Debug)] +pub struct BlockHashAndEventCountRow { + pub number: u64, + pub hash: Felt, + pub event_count: u64, +} + +impl From for (u64, Felt) { + fn from(value: NumberAndHashRow) -> Self { + (value.0, value.1) + } +} + +impl TryFrom<&Row<'_>> for BlockEventsRow { + type Error = SqliteError; + + fn try_from(value: &Row<'_>) -> Result { + Ok(Self { + block_number: value.get::<_, i64>(0)? as u64, + events: value.get(1)?, + }) + } +} + +impl TryFrom<&Row<'_>> for NumberAndHashRow { + type Error = SqliteError; + + fn try_from(value: &Row<'_>) -> Result { + Ok(Self(value.get::<_, i64>(0)? as u64, value.get_felt(1)?)) + } +} + +impl TryFrom<&Row<'_>> for BlockContextRow { + type Error = SqliteError; + + fn try_from(value: &Row<'_>) -> Result { + Ok(Self { + number: value.get::<_, i64>(0)? as u64, + hash: value.get_felt(1)?, + parent_hash: value.get_felt(2)?, + timestamp: value.get::<_, i64>(3)? as u64, + event_count: value.get::<_, i64>(4)? as u64, + }) + } +} + +impl TryFrom<&Row<'_>> for BlockHashAndEventCountRow { + type Error = SqliteError; + + fn try_from(value: &Row<'_>) -> Result { + Ok(Self { + number: value.get::<_, i64>(0)? as u64, + hash: value.get_felt(1)?, + event_count: value.get::<_, i64>(2)? as u64, + }) + } +} + +pub trait RowExt { + fn get_felt(&self, idx: usize) -> SqliteResult; + fn get_felt_opt(&self, idx: usize) -> SqliteResult>; +} + +impl RowExt for Row<'_> { + fn get_felt(&self, idx: usize) -> SqliteResult { + match self.get_ref(idx)? { + ValueRef::Blob(b) => Ok(Felt::from_bytes_be_slice(b)), + _ => Err(SqliteError::InvalidColumnType( + idx, + "felt".into(), + rusqlite::types::Type::Blob, + )), + } + } + + fn get_felt_opt(&self, idx: usize) -> SqliteResult> { + match self.get_ref(idx)? { + ValueRef::Blob(b) => Ok(Some(Felt::from_bytes_be_slice(b))), + ValueRef::Null => Ok(None), + _ => Err(SqliteError::InvalidColumnType( + idx, + "felt".into(), + rusqlite::types::Type::Blob, + )), + } + } +} + +pub trait SqliteExt { + fn query_range( + &self, + sql: &str, + from_block: u64, + to_block: u64, + count: usize, + ) -> SqliteResult> + where + T: for<'a> TryFrom<&'a Row<'a>, Error = SqliteError>; + + fn get_block_events_rows( + &self, + from_block: u64, + to_block: u64, + ) -> SqliteResult> { + self.query_range( + "SELECT block_number, events FROM transactions WHERE block_number BETWEEN ? AND ? ORDER BY block_number", + from_block, + to_block, + (to_block.saturating_sub(from_block) + 1) as usize, + ) + } + fn get_block_tx_hash_rows( + &self, + from_block: u64, + to_block: u64, + ) -> SqliteResult>; + fn get_block_hash_rows( + &self, + from_block: u64, + to_block: u64, + ) -> SqliteResult> { + self.query_range( + "SELECT number, hash FROM block_headers WHERE number BETWEEN ? AND ? ORDER BY number", + from_block, + to_block, + (to_block.saturating_sub(from_block) + 1) as usize, + ) + } + fn get_block_hash_rows_with_count( + &self, + from_block: u64, + to_block: u64, + ) -> SqliteResult> { + self.query_range( + "SELECT number, hash, event_count FROM block_headers WHERE number BETWEEN ? AND ? ORDER BY number", + from_block, + to_block, + (to_block.saturating_sub(from_block) + 1) as usize, + ) + } + fn get_block_context_rows( + &self, + from_block: u64, + to_block: u64, + ) -> SqliteResult> { + self.query_range("SELECT number, hash, parent_hash, timestamp, event_count FROM block_headers WHERE number BETWEEN ? AND ? ORDER BY number", from_block, to_block, (to_block.saturating_sub(from_block) + 1) as usize) + } + fn get_number_of_events_for_blocks(&self, from_block: u64, to_block: u64) -> SqliteResult; +} + +impl SqliteExt for Connection { + fn query_range( + &self, + sql: &str, + from_block: u64, + to_block: u64, + count: usize, + ) -> SqliteResult> + where + T: for<'a> TryFrom<&'a Row<'a>, Error = SqliteError>, + { + let mut stmt = self.prepare_cached(sql)?; + let mut rows = stmt.query(params![from_block, to_block])?; + let mut result = Vec::with_capacity(count); + while let Some(row) = rows.next()? { + result.push(T::try_from(row)?); + } + Ok(result) + } + fn get_block_tx_hash_rows( + &self, + from_block: u64, + to_block: u64, + ) -> SqliteResult> { + let count = self + .prepare_cached( + "SELECT COUNT(*) FROM transaction_hashes WHERE block_number BETWEEN ? AND ?", + )? + .query_row(params![from_block, to_block], |r| r.get::<_, i64>(0)) + .map(|v| v as usize)?; + self.query_range( + "SELECT block_number, hash FROM transaction_hashes WHERE block_number BETWEEN ? AND ? ORDER BY block_number", + from_block, + to_block, + count, + ) + } + + fn get_number_of_events_for_blocks(&self, from_block: u64, to_block: u64) -> SqliteResult { + self.prepare_cached( + "SELECT SUM(event_count) FROM block_headers WHERE number BETWEEN ? AND ?", + )? + .query_row(params![from_block, to_block], |r| r.get::<_, i64>(0)) + .map(|v| v as u64) + } +} diff --git a/crates/pathfinder/src/test.rs b/crates/pathfinder/src/test.rs new file mode 100644 index 00000000..33324343 --- /dev/null +++ b/crates/pathfinder/src/test.rs @@ -0,0 +1,44 @@ +use crate::{connect, extractor::PathfinderExtractor, fetcher::EventFetcher}; + +const DB_PATH: &str = "/mnt/store/mainnet.sqlite"; + +#[test] +#[ignore = "requires /mnt/store/mainnet.sqlite snapshot"] +fn test_emitted_events() { + let mut extractor = + PathfinderExtractor::new(DB_PATH, 2000, 0, 4000000).expect("failed to create extractor"); + for n in 0..10000 { + let (blocks, events) = extractor.next_batch().expect("failed to fetch batch"); + println!( + "Fetched {} blocks and {} events", + blocks.len(), + events.len() + ); + if blocks.is_empty() || n >= 20 { + break; + } + } +} + +#[test] +#[ignore = "requires /mnt/store/mainnet.sqlite snapshot"] +fn test_get_emitted_events_with_context() { + let conn = connect(DB_PATH).unwrap(); + let (blocks, events) = conn + .get_emitted_events_with_context(3000000, 3000010) + .expect("failed to fetch events with context"); + println!( + "Fetched {} blocks and {} events", + blocks.len(), + events.len() + ); + for block in blocks { + println!( + "Block {}: hash {} timestamp {}", + block.number, block.hash, block.timestamp + ); + } + for event in events { + println!("{event:?}"); + } +} diff --git a/crates/pathfinder/src/utils.rs b/crates/pathfinder/src/utils.rs new file mode 100644 index 00000000..c0a75fb5 --- /dev/null +++ b/crates/pathfinder/src/utils.rs @@ -0,0 +1,10 @@ +use std::path::Path; + +use rusqlite::{Connection, OpenFlags, Result as RusqliteResult}; + +pub fn connect>(path: P) -> RusqliteResult { + Connection::open_with_flags( + path, + OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_NO_MUTEX, + ) +} diff --git a/crates/starknet/Cargo.toml b/crates/starknet/Cargo.toml new file mode 100644 index 00000000..9517b699 --- /dev/null +++ b/crates/starknet/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "torii-starknet" +version = "0.1.0" +edition = "2021" +description = "StarkNet utilities for Torii" + +[dependencies] +starknet = { workspace = true, optional = true } +serde = { workspace = true, optional = true } + +[dev-dependencies] +pretty_assertions_sorted = "1.2.3" +assert_matches = "1.5.0" +serde_json.workspace = true +bincode = { workspace = true, features = ["serde"] } + +[features] +default = ["serde"] +starknet = ["dep:starknet"] +serde = ["dep:serde"] + +[lints] +workspace = true diff --git a/crates/starknet/src/event.rs b/crates/starknet/src/event.rs new file mode 100644 index 00000000..d04e86d6 --- /dev/null +++ b/crates/starknet/src/event.rs @@ -0,0 +1,23 @@ +use crate::Felt; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +pub struct EmittedEvent { + pub from_address: Felt, + pub keys: Vec, + pub data: Vec, + #[cfg_attr(feature = "serde", serde(skip_serializing_if = "Option::is_none"))] + pub block_hash: Option, + #[cfg_attr(feature = "serde", serde(skip_serializing_if = "Option::is_none"))] + pub block_number: Option, + pub transaction_hash: Felt, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +pub struct Event { + pub from_address: Felt, + pub keys: Vec, + pub data: Vec, +} diff --git a/crates/starknet/src/felt.rs b/crates/starknet/src/felt.rs new file mode 100644 index 00000000..90dbb411 --- /dev/null +++ b/crates/starknet/src/felt.rs @@ -0,0 +1,714 @@ +use std::borrow::Cow; +use std::error::Error; + +/// Starknet Field Element. +/// +/// A field element is a number 0..p-1 with p=2^{251}+17*2^{192}+1, and it forms +/// the basic building block of most Starknet interactions. +#[repr(transparent)] +#[derive(Clone, Copy, PartialEq, Hash, Eq, PartialOrd, Ord)] +pub struct Felt(pub(crate) [u8; 32]); + +const MODULUS_U64: [u64; 4] = [576460752303423505u64, 0, 0, 1]; + +impl std::fmt::Debug for Felt { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "0x{self:x}") + } +} + +impl std::fmt::Display for Felt { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{self:x}") + } +} + +impl std::fmt::LowerHex for Felt { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let bytes = &self.0; + let first = bytes.iter().position(|&b| b != 0).unwrap_or(31); + write!(f, "{:x}", bytes[first])?; + bytes[first + 1..] + .iter() + .try_for_each(|&b| write!(f, "{b:02x}")) + } +} + +impl std::fmt::UpperHex for Felt { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let bytes = &self.0; + let first = bytes.iter().position(|&b| b != 0).unwrap_or(31); + write!(f, "{:X}", bytes[first])?; + bytes[first + 1..] + .iter() + .try_for_each(|&b| write!(f, "{b:02X}")) + } +} +impl Default for Felt { + fn default() -> Self { + Felt::ZERO + } +} + +/// Error returned by [Felt::from_be_bytes] indicating the maximum field value +/// was exceeded. +#[derive(Debug, PartialEq, Eq, Clone, Copy)] +pub struct OverflowError; + +impl Error for OverflowError {} + +const OVERFLOW_MSG: &str = "The maximum field value was exceeded."; + +impl std::fmt::Display for OverflowError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(OVERFLOW_MSG) + } +} + +impl Felt { + pub const ZERO: Felt = Felt([0u8; 32]); + pub const ONE: Felt = Self::from_u64(1); + + /// Return true if the element is zero. + pub fn is_zero(&self) -> bool { + self == &Felt::ZERO + } + + /// Returns the big-endian representation of this [Felt]. + pub const fn to_be_bytes(self) -> [u8; 32] { + self.0 + } + + /// Returns the little-endian representation of this [Felt]. + pub fn to_le_bytes(self) -> [u8; 32] { + let mut tmp = self.0; + tmp.reverse(); + tmp + } + + /// Big-endian representation of this [Felt]. + pub const fn as_be_bytes(&self) -> &[u8; 32] { + &self.0 + } + + /// Big-endian mutable representation of this [Felt]. + pub fn as_mut_be_bytes(&mut self) -> &mut [u8; 32] { + &mut self.0 + } + + /// Convenience function which extends [Felt::from_be_bytes] to work with + /// slices. + pub const fn from_be_slice(bytes: &[u8]) -> Result { + if bytes.len() > 32 { + return Err(OverflowError); + } + + let mut buf = [0u8; 32]; + let mut index = 0; + loop { + if index == bytes.len() { + break; + } + buf[32 - bytes.len() + index] = bytes[index]; + index += 1; + } + + Felt::from_be_bytes(buf) + } + + /// Creates a [Felt] from big-endian bytes. + /// + /// Returns [OverflowError] if not less than the field modulus. + pub const fn from_be_bytes(bytes: [u8; 32]) -> Result { + #[rustfmt::skip] + let limbs = [ + u64::from_be_bytes([ + bytes[0], bytes[1], bytes[2], bytes[3], + bytes[4], bytes[5], bytes[6], bytes[7], + ]), + u64::from_be_bytes([ + bytes[8], bytes[9], bytes[10], bytes[11], + bytes[12], bytes[13], bytes[14], bytes[15], + ]), + u64::from_be_bytes([ + bytes[16], bytes[17], bytes[18], bytes[19], + bytes[20], bytes[21], bytes[22], bytes[23], + ]), + u64::from_be_bytes([ + bytes[24], bytes[25], bytes[26], bytes[27], + bytes[28], bytes[29], bytes[30], bytes[31], + ]), + ]; + + // Loop over each word, if all previous are equal and current is less, we are + // good. + let mut maybe_overflow = true; + let mut i = 0; + while i < 4 && maybe_overflow { + if limbs[i] < MODULUS_U64[i] { + maybe_overflow = false; + } else if limbs[i] > MODULUS_U64[i] { + return Err(OverflowError); + } + i += 1; + } + if maybe_overflow { + Err(OverflowError) + } else { + Ok(Felt(bytes)) + } + } + + /// Creates a Felt from a big-endian byte slice of up to 32 bytes. + /// Panics if slice is longer than 32 bytes. + pub fn from_bytes_be_slice(bytes: &[u8]) -> Self { + assert!(bytes.len() <= 32, "slice too long"); + let mut buf = [0u8; 32]; + buf[32 - bytes.len()..].copy_from_slice(bytes); + Self(buf) + } + + /// Creates a Felt from a little-endian byte slice of up to 32 bytes. + pub fn from_bytes_le_slice(bytes: &[u8]) -> Self { + assert!(bytes.len() <= 32, "slice too long"); + let mut buf = [0u8; 32]; + for (i, &b) in bytes.iter().enumerate() { + buf[31 - i] = b; + } + Self(buf) + } + + /// Returns `true` if the value of [`Felt`] is larger than `2^251 - 1`. + /// + /// Every [`Felt`] that is used to traverse a Merkle-Patricia Tree + /// must not exceed 251 bits, since 251 is the height of the tree. + pub const fn has_more_than_251_bits(&self) -> bool { + self.0[0] & 0b1111_1000 > 0 + } + + pub const fn from_u64(u: u64) -> Self { + const_expect!( + Self::from_be_slice(&u.to_be_bytes()), + "64 bits is less than 251 bits" + ) + } + + pub const fn from_u128(u: u128) -> Self { + const_expect!( + Self::from_be_slice(&u.to_be_bytes()), + "128 bits is less than 251 bits" + ) + } +} + +macro_rules! const_expect { + ($e:expr, $why:expr) => {{ + match $e { + Ok(x) => x, + Err(_) => panic!(concat!("Expectation failed: ", $why)), + } + }}; +} + +use const_expect; + +impl From for Felt { + fn from(value: u64) -> Self { + Self::from_u64(value) + } +} + +impl From for Felt { + fn from(value: usize) -> Self { + Self::from_u64(value.try_into().expect("ptr size is 64 bits")) + } +} + +impl From for Felt { + fn from(value: u128) -> Self { + Self::from_u128(value) + } +} + +impl From<[u8; 32]> for Felt { + fn from(bytes: [u8; 32]) -> Self { + Self(bytes) + } +} + +impl From<[u8; 31]> for Felt { + fn from(bytes: [u8; 31]) -> Self { + let mut buf = [0u8; 32]; + buf[1..].copy_from_slice(&bytes); + Self(buf) + } +} + +impl TryInto for Felt { + type Error = OverflowError; + + fn try_into(self) -> Result { + let initial_zeroes = self.0.iter().take_while(|b| **b == 0).count(); + const EXPECTED_ZEROES: usize = (32 - u128::BITS / u8::BITS) as usize; + + if initial_zeroes < EXPECTED_ZEROES { + return Err(OverflowError); + } + + let bytes = self.0[EXPECTED_ZEROES..] + .try_into() + .expect("Should match u128 size"); + Ok(u128::from_be_bytes(bytes)) + } +} + +impl TryInto for Felt { + type Error = OverflowError; + + fn try_into(self) -> Result { + let initial_zeroes = self.0.iter().take_while(|b| **b == 0).count(); + const EXPECTED_ZEROES: usize = (32 - u64::BITS / u8::BITS) as usize; + + if initial_zeroes < EXPECTED_ZEROES { + return Err(OverflowError); + } + let bytes = self.0[EXPECTED_ZEROES..] + .try_into() + .expect("Should match u64 size"); + Ok(u64::from_be_bytes(bytes)) + } +} + +impl Felt { + /// A convenience function which parses a hex string into a [Felt]. + /// + /// Supports both upper and lower case hex strings, as well as an optional + /// "0x" prefix. + pub const fn from_hex_str(hex_str: &str) -> Result { + const fn parse_hex_digit(digit: u8) -> Result { + match digit { + b'0'..=b'9' => Ok(digit - b'0'), + b'A'..=b'F' => Ok(digit - b'A' + 10), + b'a'..=b'f' => Ok(digit - b'a' + 10), + other => Err(HexParseError::InvalidNibble(other)), + } + } + + let bytes = hex_str.as_bytes(); + let start = if bytes.len() >= 2 && bytes[0] == b'0' && bytes[1] == b'x' { + 2 + } else { + 0 + }; + let len = bytes.len() - start; + + if len > 64 { + return Err(HexParseError::InvalidLength { + max: 64, + actual: bytes.len(), + }); + } + + let mut buf = [0u8; 32]; + + // We want the result in big-endian so reverse iterate over each pair of + // nibbles. let chunks = hex_str.as_bytes().rchunks_exact(2); + + // Handle a possible odd nibble remaining nibble. + if len % 2 == 1 { + let idx = len / 2; + buf[31 - idx] = match parse_hex_digit(bytes[start]) { + Ok(b) => b, + Err(e) => return Err(e), + }; + } + + let chunks = len / 2; + let mut chunk = 0; + + while chunk < chunks { + let lower = match parse_hex_digit(bytes[bytes.len() - chunk * 2 - 1]) { + Ok(b) => b, + Err(e) => return Err(e), + }; + let upper = match parse_hex_digit(bytes[bytes.len() - chunk * 2 - 2]) { + Ok(b) => b, + Err(e) => return Err(e), + }; + buf[31 - chunk] = (upper << 4) | lower; + chunk += 1; + } + + let felt = match Felt::from_be_bytes(buf) { + Ok(felt) => felt, + Err(OverflowError) => return Err(HexParseError::Overflow), + }; + Ok(felt) + } + + /// The first stage of conversion - skip leading zeros + fn skip_zeros(&self) -> (impl Iterator, usize, usize) { + // Skip all leading zero bytes + let it = self.0.iter().skip_while(|&&b| b == 0); + let num_bytes = it.clone().count(); + let skipped = self.0.len() - num_bytes; + // The first high nibble can be 0 + let start = if self.0[skipped] < 0x10 { 1 } else { 2 }; + // Number of characters to display + let len = start + num_bytes * 2; + (it, start, len) + } + + /// The second stage of conversion - map bytes to hex str + fn it_to_hex_str<'a>( + it: impl Iterator, + start: usize, + len: usize, + buf: &'a mut [u8], + ) -> &'a [u8] { + const LUT: [u8; 16] = *b"0123456789abcdef"; + buf[0] = b'0'; + // Same small lookup table is ~25% faster than hex::encode_from_slice 🤷 + it.enumerate().for_each(|(i, &b)| { + let idx = b as usize; + let pos = start + i * 2; + let x = [LUT[(idx & 0xf0) >> 4], LUT[idx & 0x0f]]; + buf[pos..pos + 2].copy_from_slice(&x); + }); + buf[1] = b'x'; + &buf[..len] + } + + /// A convenience function which produces a "0x" prefixed hex str slice in a + /// given buffer `buf` from a [Felt]. + /// Panics if `self.0.len() * 2 + 2 > buf.len()` + pub fn as_hex_str<'a>(&'a self, buf: &'a mut [u8]) -> &'a str { + let expected_buf_len = self.0.len() * 2 + 2; + assert!( + buf.len() >= expected_buf_len, + "buffer size is {}, expected at least {}", + buf.len(), + expected_buf_len + ); + + if !self.0.iter().any(|b| *b != 0) { + return "0x0"; + } + + let (it, start, len) = self.skip_zeros(); + let res = Self::it_to_hex_str(it, start, len, buf); + // Unwrap is safe because `buf` holds valid UTF8 characters. + std::str::from_utf8(res).unwrap() + } + + /// A convenience function which produces a "0x" prefixed hex string from a + /// [Felt]. + pub fn to_hex_str(&self) -> Cow<'static, str> { + if !self.0.iter().any(|b| *b != 0) { + return Cow::from("0x0"); + } + let (it, start, len) = self.skip_zeros(); + let mut buf = vec![0u8; len]; + Self::it_to_hex_str(it, start, len, &mut buf); + // Unwrap is safe as the buffer contains valid utf8 + String::from_utf8(buf).unwrap().into() + } +} + +/// Error returned by [Felt::from_hex_str] indicating an invalid hex string. +#[derive(Debug, PartialEq, Eq)] +pub enum HexParseError { + InvalidNibble(u8), + InvalidLength { max: usize, actual: usize }, + Overflow, +} + +impl Error for HexParseError {} + +impl From for HexParseError { + fn from(_: OverflowError) -> Self { + Self::Overflow + } +} + +impl std::fmt::Display for HexParseError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::InvalidNibble(n) => f.write_fmt(format_args!("Invalid nibble found: 0x{:x}", *n)), + Self::InvalidLength { max, actual } => { + f.write_fmt(format_args!("More than {} digits found: {}", *max, *actual)) + } + Self::Overflow => f.write_str(OVERFLOW_MSG), + } + } +} + +#[cfg(test)] +mod tests { + use pretty_assertions_sorted::assert_eq; + + use super::*; + + const MODULUS_U8: [u8; 32] = [ + 8, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 1, + ]; + + #[test] + fn bytes_round_trip() { + let original = [ + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, + 0x0E, 0x0F, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1A, 0x1B, + 0x1C, 0x1D, 0x1E, 0x1F, + ]; + let hash = Felt::from_be_bytes(original).unwrap(); + let bytes = hash.to_be_bytes(); + assert_eq!(bytes, original); + } + + #[test] + fn from_bytes_overflow() { + // Field modulus + assert_eq!(Felt::from_be_bytes(MODULUS_U8), Err(OverflowError)); + // Field modulus - 1 + let mut max_val = MODULUS_U8; + max_val[31] -= 1; + Felt::from_be_bytes(max_val).unwrap(); + } + + mod from_be_slice { + use pretty_assertions_sorted::assert_eq; + + use super::*; + + #[test] + fn round_trip() { + let original = Felt::from_hex_str("abcdef0123456789").unwrap(); + let bytes = original.to_be_bytes(); + let result = Felt::from_be_slice(&bytes[..]).unwrap(); + + assert_eq!(result, original); + } + + #[test] + fn too_long() { + let original = Felt::from_hex_str("abcdef0123456789").unwrap(); + let mut bytes = original.to_be_bytes().to_vec(); + bytes.push(0); + Felt::from_be_slice(&bytes[..]).unwrap_err(); + } + + #[test] + fn short_slice() { + let original = Felt::from_hex_str("abcdef0123456789").unwrap(); + let bytes = original.to_be_bytes(); + let result = Felt::from_be_slice(&bytes[24..]); + + assert_eq!(result, Ok(original)); + } + + #[test] + fn max() { + let mut max_val = MODULUS_U8; + max_val[31] -= 1; + Felt::from_be_slice(&max_val[..]).unwrap(); + } + + #[test] + fn overflow() { + assert_eq!(Felt::from_be_slice(&MODULUS_U8[..]), Err(OverflowError)); + } + } + + mod fmt { + use pretty_assertions_sorted::assert_eq; + + use super::Felt; + + #[test] + fn debug() { + let hex_str = "1234567890abcdef000edcba0987654321"; + let felt = Felt::from_hex_str(hex_str).unwrap(); + let result = format!("{felt:?}"); + + let expected = format!("0x{felt}"); + + assert_eq!(result, expected); + } + + #[test] + fn fmt() { + let hex_str = "1234567890abcdef000edcba0987654321"; + let starkhash = Felt::from_hex_str(hex_str).unwrap(); + let result = format!("{starkhash:x}"); + + // We don't really care which casing is used by fmt. + assert_eq!(result.to_lowercase(), hex_str.to_lowercase()); + } + + #[test] + fn lower_hex() { + let hex_str = "1234567890abcdef000edcba0987654321"; + let starkhash = Felt::from_hex_str(hex_str).unwrap(); + let result = format!("{starkhash:x}"); + + assert_eq!(result, hex_str.to_lowercase()); + } + + #[test] + fn upper_hex() { + let hex_str = "1234567890abcdef000edcba0987654321"; + let starkhash = Felt::from_hex_str(hex_str).unwrap(); + let result = format!("{starkhash:X}"); + + assert_eq!(result, hex_str.to_uppercase()); + } + } + + mod from_hex_str { + use assert_matches::assert_matches; + use pretty_assertions_sorted::assert_eq; + + use super::*; + + /// Test hex string with its expected [Felt]. + fn test_data() -> (&'static str, Felt) { + let mut expected = [0; 32]; + expected[31] = 0xEF; + expected[30] = 0xCD; + expected[29] = 0xAB; + expected[28] = 0xef; + expected[27] = 0xcd; + expected[26] = 0xab; + expected[25] = 0x89; + expected[24] = 0x67; + expected[23] = 0x45; + expected[22] = 0x23; + expected[21] = 0x01; + let expected = Felt::from_be_bytes(expected).unwrap(); + + ("0123456789abcdefABCDEF", expected) + } + + #[test] + fn simple() { + let (test_str, expected) = test_data(); + let uut = Felt::from_hex_str(test_str).unwrap(); + assert_eq!(uut, expected); + } + + #[test] + fn prefix() { + let (test_str, expected) = test_data(); + let uut = Felt::from_hex_str(&format!("0x{test_str}")).unwrap(); + assert_eq!(uut, expected); + } + + #[test] + fn leading_zeros() { + let (test_str, expected) = test_data(); + let uut = Felt::from_hex_str(&format!("000000000{test_str}")).unwrap(); + assert_eq!(uut, expected); + } + + #[test] + fn prefix_and_leading_zeros() { + let (test_str, expected) = test_data(); + let uut = Felt::from_hex_str(&format!("0x000000000{test_str}")).unwrap(); + assert_eq!(uut, expected); + } + + #[test] + fn invalid_nibble() { + assert_matches!(Felt::from_hex_str("0x123z").unwrap_err(), HexParseError::InvalidNibble(n) => assert_eq!(n, b'z')); + } + + #[test] + fn invalid_len() { + assert_matches!(Felt::from_hex_str(&"1".repeat(65)).unwrap_err(), HexParseError::InvalidLength{max: 64, actual: n} => assert_eq!(n, 65)); + } + + #[test] + fn overflow() { + // Field modulus + let mut modulus = + "0x800000000000011000000000000000000000000000000000000000000000001".to_string(); + assert_eq!( + Felt::from_hex_str(&modulus).unwrap_err(), + HexParseError::Overflow + ); + // Field modulus - 1 + modulus.pop(); + modulus.push('0'); + Felt::from_hex_str(&modulus).unwrap(); + } + } + + mod to_hex_str { + use pretty_assertions_sorted::assert_eq; + + use super::*; + + const ODD: &str = "0x1234567890abcde"; + const EVEN: &str = "0x1234567890abcdef"; + const MAX: &str = "0x800000000000011000000000000000000000000000000000000000000000000"; + + #[test] + fn zero() { + assert_eq!(Felt::ZERO.to_hex_str(), "0x0"); + let mut buf = [0u8; 66]; + assert_eq!(Felt::ZERO.as_hex_str(&mut buf), "0x0"); + } + + #[test] + fn odd() { + let hash = Felt::from_hex_str(ODD).unwrap(); + assert_eq!(hash.to_hex_str(), ODD); + let mut buf = [0u8; 66]; + assert_eq!(hash.as_hex_str(&mut buf), ODD); + } + + #[test] + fn even() { + let hash = Felt::from_hex_str(EVEN).unwrap(); + assert_eq!(hash.to_hex_str(), EVEN); + let mut buf = [0u8; 66]; + assert_eq!(hash.as_hex_str(&mut buf), EVEN); + } + + #[test] + fn max() { + let hash = Felt::from_hex_str(MAX).unwrap(); + assert_eq!(hash.to_hex_str(), MAX); + let mut buf = [0u8; 66]; + assert_eq!(hash.as_hex_str(&mut buf), MAX); + } + + #[test] + #[should_panic(expected = "buffer size is 65, expected at least 66")] + fn buffer_too_small() { + let mut buf = [0u8; 65]; + Felt::ZERO.as_hex_str(&mut buf); + } + } + + mod has_more_than_251_bits { + use super::*; + + #[test] + fn has_251_bits() { + let mut bytes = [0xFFu8; 32]; + bytes[0] = 0x07; + let h = Felt::from_be_bytes(bytes).unwrap(); + assert!(!h.has_more_than_251_bits()); + } + + #[test] + fn has_252_bits() { + let mut bytes = [0u8; 32]; + bytes[0] = 0x08; + let h = Felt::from_be_bytes(bytes).unwrap(); + assert!(h.has_more_than_251_bits()); + } + } +} diff --git a/crates/starknet/src/lib.rs b/crates/starknet/src/lib.rs new file mode 100644 index 00000000..54ae3603 --- /dev/null +++ b/crates/starknet/src/lib.rs @@ -0,0 +1,10 @@ +pub mod event; +pub mod felt; + +#[cfg(feature = "serde")] +pub mod serde; + +#[cfg(feature = "starknet")] +pub mod starknet; + +pub use felt::Felt; diff --git a/crates/starknet/src/serde.rs b/crates/starknet/src/serde.rs new file mode 100644 index 00000000..1aece7f2 --- /dev/null +++ b/crates/starknet/src/serde.rs @@ -0,0 +1,134 @@ +use serde::{Deserialize, Serialize}; + +use crate::Felt; + +impl Serialize for Felt { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + if serializer.is_human_readable() { + serializer.serialize_str(&self.to_hex_str()) + } else { + let bytes = self.as_be_bytes(); + let first = bytes.iter().position(|&b| b != 0).unwrap_or(31); + serializer.serialize_bytes(&bytes[first..]) + } + } +} + +impl<'de> Deserialize<'de> for Felt { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + if deserializer.is_human_readable() { + deserializer.deserialize_str(FeltVisitor) + } else { + deserializer.deserialize_bytes(FeltVisitor) + } + } +} + +struct FeltVisitor; + +impl serde::de::Visitor<'_> for FeltVisitor { + type Value = Felt; + + fn expecting(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + write!(f, "a hex string prefixed with 0x or up to 32 bytes") + } + + fn visit_str(self, value: &str) -> Result { + Felt::from_hex_str(value).map_err(|e| E::custom(e.to_string())) + } + + fn visit_bytes(self, value: &[u8]) -> Result { + if value.len() > 32 { + return Err(E::invalid_length(value.len(), &self)); + } + let mut buf = [0u8; 32]; + buf[32 - value.len()..].copy_from_slice(value); + Ok(Felt(buf)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json; + + #[test] + fn serialize_zero_human_readable() { + let felt = Felt::ZERO; + assert_eq!(serde_json::to_string(&felt).unwrap(), "\"0x0\""); + } + + #[test] + fn serialize_one_human_readable() { + let felt = Felt::ONE; + assert_eq!(serde_json::to_string(&felt).unwrap(), "\"0x1\""); + } + + #[test] + fn serialize_human_readable_round_trip() { + let original = Felt::from_hex_str("0xdeadbeef").unwrap(); + let serialized = serde_json::to_string(&original).unwrap(); + let deserialized: Felt = serde_json::from_str(&serialized).unwrap(); + assert_eq!(original, deserialized); + } + + #[test] + fn serialize_max_human_readable() { + let max = + Felt::from_hex_str("0x800000000000011000000000000000000000000000000000000000000000000") + .unwrap(); + let serialized = serde_json::to_string(&max).unwrap(); + let deserialized: Felt = serde_json::from_str(&serialized).unwrap(); + assert_eq!(max, deserialized); + } + + #[test] + fn deserialize_overflow_fails() { + // Field modulus - should fail + assert!(serde_json::from_str::( + "\"0x800000000000011000000000000000000000000000000000000000000000001\"" + ) + .is_err()); + } + + #[test] + fn serialize_binary_round_trip() { + let original = Felt::from_hex_str("0xdeadbeef").unwrap(); + let serialized = + bincode::serde::encode_to_vec(original, bincode::config::standard()).unwrap(); + let deserialized: Felt = + bincode::serde::decode_from_slice(&serialized, bincode::config::standard()) + .unwrap() + .0; + assert_eq!(original, deserialized); + } + + #[test] + fn serialize_binary_zero() { + let felt = Felt::ZERO; + let serialized = bincode::serde::encode_to_vec(felt, bincode::config::standard()).unwrap(); + let deserialized: Felt = + bincode::serde::decode_from_slice(&serialized, bincode::config::standard()) + .unwrap() + .0; + assert_eq!(felt, deserialized); + } + + #[test] + fn serialize_binary_compact() { + let felt = Felt::from_hex_str("0xbabe").unwrap(); + let serialized = bincode::serde::encode_to_vec(felt, bincode::config::standard()).unwrap(); + assert!(serialized.len() < 32); + let deserialized: Felt = + bincode::serde::decode_from_slice(&serialized, bincode::config::standard()) + .unwrap() + .0; + assert_eq!(felt, deserialized); + } +} diff --git a/crates/starknet/src/starknet.rs b/crates/starknet/src/starknet.rs new file mode 100644 index 00000000..d3ac96c7 --- /dev/null +++ b/crates/starknet/src/starknet.rs @@ -0,0 +1,73 @@ +use crate::event::{EmittedEvent, Event}; +use crate::Felt; +use starknet::core::types::Felt as SnFelt; +use starknet::core::types::{EmittedEvent as SnEmittedEvent, Event as SnEvent}; + +impl From for Felt { + fn from(value: SnFelt) -> Self { + Self(value.to_bytes_be()) + } +} + +impl From<&SnFelt> for Felt { + fn from(value: &SnFelt) -> Self { + Self(value.to_bytes_be()) + } +} + +impl From for SnFelt { + fn from(value: Felt) -> Self { + Self::from_bytes_be(&value.0) + } +} +impl From<&Felt> for SnFelt { + fn from(value: &Felt) -> Self { + Self::from_bytes_be(&value.0) + } +} + +impl From for Event { + fn from(value: SnEvent) -> Self { + Self { + from_address: value.from_address.into(), + keys: value.keys.into_iter().map(Into::into).collect(), + data: value.data.into_iter().map(Into::into).collect(), + } + } +} + +impl From for EmittedEvent { + fn from(value: SnEmittedEvent) -> Self { + Self { + from_address: value.from_address.into(), + keys: value.keys.into_iter().map(Into::into).collect(), + data: value.data.into_iter().map(Into::into).collect(), + block_hash: value.block_hash.map(Into::into), + block_number: value.block_number, + transaction_hash: value.transaction_hash.into(), + } + } +} + +impl From for SnEvent { + fn from(value: Event) -> Self { + Self { + from_address: value.from_address.into(), + keys: value.keys.into_iter().map(Into::into).collect(), + data: value.data.into_iter().map(Into::into).collect(), + } + } +} + +impl From for SnEmittedEvent { + fn from(value: EmittedEvent) -> Self { + Self { + from_address: value.from_address.into(), + keys: value.keys.into_iter().map(Into::into).collect(), + data: value.data.into_iter().map(Into::into).collect(), + block_hash: value.block_hash.map(Into::into), + block_number: value.block_number, + transaction_hash: value.transaction_hash.into(), + } + } +} diff --git a/src/etl/extractor/block_range.rs b/src/etl/extractor/block_range.rs index a31f5945..060c279d 100644 --- a/src/etl/extractor/block_range.rs +++ b/src/etl/extractor/block_range.rs @@ -400,7 +400,9 @@ impl Extractor for BlockRangeExtractor { fn is_finished(&self) -> bool { self.reached_end } - + fn set_start_block(&mut self, start_block: u64) { + self.current_block = start_block.max(self.current_block); + } async fn commit_cursor(&mut self, cursor: &str, engine_db: &EngineDb) -> Result<()> { if let Some(block_str) = cursor.strip_prefix("block:") { let block_num: u64 = block_str.parse().context("Invalid cursor format")?; diff --git a/src/etl/extractor/composite.rs b/src/etl/extractor/composite.rs index e0cc18cc..9ad93d84 100644 --- a/src/etl/extractor/composite.rs +++ b/src/etl/extractor/composite.rs @@ -96,6 +96,11 @@ impl CompositeExtractor { #[async_trait] impl Extractor for CompositeExtractor { + fn set_start_block(&mut self, start_block: u64) { + for extractor in &mut self.extractors { + extractor.set_start_block(start_block); + } + } async fn extract( &mut self, _cursor: Option, diff --git a/src/etl/extractor/event.rs b/src/etl/extractor/event.rs index 0b6b4856..5f8d8511 100644 --- a/src/etl/extractor/event.rs +++ b/src/etl/extractor/event.rs @@ -269,6 +269,8 @@ pub struct EventExtractor { /// Per-contract extraction state. contract_states: HashMap, + start_block: u64, + /// Whether the extractor has been initialized. initialized: bool, @@ -294,6 +296,7 @@ impl EventExtractor { initialized: false, chain_head: None, has_following_contracts, + start_block: 0, } } @@ -448,7 +451,7 @@ impl EventExtractor { tracing::trace!( target: "torii::etl::event", contract = %state.state_key(), - from_block = state.current_block, + from_block = state.current_block.max(self.start_block), to_block = range_end, following_head = state.is_following_head(), "Queueing event range request" @@ -457,7 +460,7 @@ impl EventExtractor { tracing::debug!( target: "torii::etl::event", contract = %state.state_key(), - from_block = state.current_block, + from_block = state.current_block.max(self.start_block), to_block = range_end, continuation_token = %state.continuation_token.as_deref().unwrap_or_default(), "Queueing paginated event request" @@ -466,7 +469,7 @@ impl EventExtractor { let request = ProviderRequestData::GetEvents(GetEventsRequest { filter: EventFilterWithPage { event_filter: EventFilter { - from_block: Some(BlockId::Number(state.current_block)), + from_block: Some(BlockId::Number(state.current_block.max(self.start_block))), to_block: Some(BlockId::Number(range_end)), address: Some(state.address), keys: None, @@ -533,6 +536,9 @@ impl EventExtractor { #[async_trait] impl Extractor for EventExtractor { + fn set_start_block(&mut self, start_block: u64) { + self.start_block = start_block; + } async fn extract( &mut self, _cursor: Option, diff --git a/src/etl/extractor/global_event.rs b/src/etl/extractor/global_event.rs index f1b3e89a..d9b115bd 100644 --- a/src/etl/extractor/global_event.rs +++ b/src/etl/extractor/global_event.rs @@ -261,6 +261,10 @@ impl GlobalEventExtractor { #[async_trait] impl Extractor for GlobalEventExtractor { + fn set_start_block(&mut self, start_block: u64) { + self.state.current_block = start_block.max(self.state.current_block); + self.config.from_block = start_block.max(self.config.from_block); + } async fn extract( &mut self, _cursor: Option, diff --git a/src/etl/extractor/mod.rs b/src/etl/extractor/mod.rs index a408461f..541fab4a 100644 --- a/src/etl/extractor/mod.rs +++ b/src/etl/extractor/mod.rs @@ -324,6 +324,10 @@ impl ExtractionBatch { /// Extractor trait for fetching enriched event batches #[async_trait] pub trait Extractor: Send + Sync { + /// Set the starting block for extraction + /// Must be called before the first call to `extract()`. Extractors that support block-based extraction + fn set_start_block(&mut self, start_block: u64); + /// Extract events with enriched context (blocks, transactions) /// /// The cursor parameter is an opaque string that allows resuming from a previous extraction. diff --git a/src/etl/extractor/sample.rs b/src/etl/extractor/sample.rs index b6e42909..54797cb5 100644 --- a/src/etl/extractor/sample.rs +++ b/src/etl/extractor/sample.rs @@ -75,6 +75,9 @@ impl SampleExtractor { #[async_trait] impl Extractor for SampleExtractor { + fn set_start_block(&mut self, start_block: u64) { + self.current_block = start_block.max(self.current_block); + } fn is_finished(&self) -> bool { false // Sample extractor cycles infinitely, never finishes } diff --git a/src/etl/extractor/synthetic_adapter.rs b/src/etl/extractor/synthetic_adapter.rs index c545520b..5d8dca21 100644 --- a/src/etl/extractor/synthetic_adapter.rs +++ b/src/etl/extractor/synthetic_adapter.rs @@ -40,6 +40,7 @@ impl Extractor for SyntheticExtractorAdapter where T: SyntheticExtractor + Send + Sync + 'static, { + fn set_start_block(&mut self, _start_block: u64) {} async fn extract( &mut self, cursor: Option, diff --git a/src/etl/extractor/synthetic_erc20.rs b/src/etl/extractor/synthetic_erc20.rs index 2a655c22..b418ff7d 100644 --- a/src/etl/extractor/synthetic_erc20.rs +++ b/src/etl/extractor/synthetic_erc20.rs @@ -257,6 +257,9 @@ impl SyntheticErc20Extractor { #[async_trait] impl Extractor for SyntheticErc20Extractor { + fn set_start_block(&mut self, start_block: u64) { + self.current_block = start_block.max(self.current_block); + } async fn extract( &mut self, cursor: Option,