diff --git a/.gitignore b/.gitignore index 0b8dbc5d..6afa48dc 100644 --- a/.gitignore +++ b/.gitignore @@ -25,3 +25,4 @@ pg-data/ /torii-data/image-cache/ HANDOFF.md .cargo/ +certs/*.pem diff --git a/Cargo.lock b/Cargo.lock index 600f4eea..8778fe29 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1510,6 +1510,12 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "http-range-header" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9171a2ea8a68358193d15dd5d70c1c10a2afc3e7e4c5bc92bc9f025cebd7359c" + [[package]] name = "httparse" version = "1.10.1" @@ -2204,6 +2210,16 @@ version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" +[[package]] +name = "mime_guess" +version = "2.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7c44f8e672c00fe5308fa235f821cb4198414e1c77935c1ab6948d3fd78550e" +dependencies = [ + "mime", + "unicase", +] + [[package]] name = "miniz_oxide" version = "0.8.9" @@ -3339,6 +3355,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "758025cb5fccfd3bc2fd74708fd4682be41d99e5dff73c377c0646c6012c73a4" dependencies = [ "aws-lc-rs", + "log", "once_cell", "ring", "rustls-pki-types", @@ -3359,6 +3376,15 @@ dependencies = [ "security-framework", ] +[[package]] +name = "rustls-pemfile" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "rustls-pki-types" version = "1.14.0" @@ -4605,12 +4631,14 @@ dependencies = [ "prost 0.13.5", "prost-types 0.13.5", "resolve-path", + "rustls-pemfile", "serde", "serde_json", "sqlx", "starknet", "tempfile", "tokio", + "tokio-rustls", "tokio-stream", "tokio-util", "tonic", @@ -4684,6 +4712,7 @@ dependencies = [ "torii-common", "torii-erc721", "torii-introspect", + "torii-runtime-common", "tracing", ] @@ -4727,6 +4756,7 @@ dependencies = [ "starknet", "tokio", "torii", + "torii-runtime-common", "tracing", ] @@ -4784,6 +4814,7 @@ dependencies = [ "torii", "torii-dojo", "torii-introspect", + "torii-runtime-common", "tracing", ] @@ -4802,6 +4833,7 @@ dependencies = [ "tokio", "torii", "torii-introspect", + "torii-runtime-common", "tracing", ] @@ -5240,12 +5272,21 @@ checksum = "1e9cd434a998747dd2c4276bc96ee2e0c7a2eadf3cae88e52be55a05fa9053f5" dependencies = [ "bitflags 2.11.0", "bytes", + "futures-util", "http", "http-body", "http-body-util", + "http-range-header", + "httpdate", + "mime", + "mime_guess", + "percent-encoding", "pin-project-lite", + "tokio", + "tokio-util", "tower-layer", "tower-service", + "tracing", ] [[package]] @@ -5394,6 +5435,12 @@ dependencies = [ "static_assertions", ] +[[package]] +name = "unicase" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbc4bc3a9f746d862c45cb89d705aa10f187bb96c76001afab07a0d35ce60142" + [[package]] name = "unicode-bidi" version = "0.3.18" diff --git a/Cargo.toml b/Cargo.toml index bbe4ceb2..41acb5e4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -58,7 +58,7 @@ prost-types = "0.13" # HTTP server and routing axum = { version = "0.7", features = ["ws"] } tower = "0.5" -tower-http = { version = "0.5", features = ["cors"] } +tower-http = { version = "0.5", features = ["cors", "fs"] } hyper = "1.0" # Serialization @@ -187,20 +187,25 @@ axum.workspace = true chrono.workspace = true futures-util.workspace = true futures.workspace = true +hyper.workspace = true +hyper-util = { version = "0.1", features = ["tokio", "server-auto", "service"] } metrics.workspace = true metrics-exporter-prometheus.workspace = true prost-types.workspace = true +rustls-pemfile = "2.0" serde_json.workspace = true serde.workspace = true sqlx.workspace = true starknet.workspace = true tokio-stream.workspace = true +tokio-rustls = { version = "0.26", default-features = false, features = ["logging", "tls12", "ring"] } tokio-util.workspace = true tokio.workspace = true tonic-reflection.workspace = true tonic-web.workspace = true tonic.workspace = true torii-common.workspace = true +tower.workspace = true tower-http.workspace = true tracing.workspace = true prost.workspace = true diff --git a/bins/torii-arcade/README.md b/bins/torii-arcade/README.md index a80c5992..e2474858 100644 --- a/bins/torii-arcade/README.md +++ b/bins/torii-arcade/README.md @@ -69,6 +69,39 @@ cargo run --bin torii-arcade -- \ --from-block 0 ``` +## Local TLS + ALPN + +For browser-compatible local HTTPS, use `mkcert` instead of a raw self-signed certificate. +Modern browsers require a trusted local CA and `subjectAltName` entries for `localhost`. + +Install and trust the local development CA: + +```bash +brew install mkcert nss +mkcert -install +mkdir -p certs +mkcert -cert-file certs/dev-cert.pem -key-file certs/dev-key.pem localhost 127.0.0.1 ::1 +``` + +Start `torii-arcade` with TLS enabled: + +```bash +cargo run --bin torii-arcade -- \ + --from-block 0 \ + --tls-cert ./certs/dev-cert.pem \ + --tls-key ./certs/dev-key.pem +``` + +The local listener advertises ALPN for `h2` and `http/1.1`. Native gRPC clients will negotiate +HTTP/2 automatically; HTTPS health and metrics endpoints remain available on the same port. + +Verify the local HTTPS listener: + +```bash +curl https://localhost:3000/health +grpcurl -insecure localhost:3000 list +``` + ## Operational validation The mixed backend can be checked end-to-end from the repository root: diff --git a/bins/torii-arcade/src/config.rs b/bins/torii-arcade/src/config.rs index 34fa83b4..55052d51 100644 --- a/bins/torii-arcade/src/config.rs +++ b/bins/torii-arcade/src/config.rs @@ -1,7 +1,7 @@ use anyhow::{bail, Result}; use clap::{Parser, ValueEnum}; use starknet::core::types::Felt; -use std::path::Path; +use std::path::{Path, PathBuf}; const DEFAULT_RPC_URL: &str = "https://api.cartridge.gg/x/starknet/mainnet"; const DEFAULT_WORLD_ADDRESS: &str = @@ -139,6 +139,14 @@ pub struct Config { #[arg(long, default_value = "3000")] pub port: u16, + /// PEM-encoded TLS certificate for the local HTTP/gRPC listener. + #[arg(long, env = "TORII_TLS_CERT")] + pub tls_cert: Option, + + /// PEM-encoded TLS private key for the local HTTP/gRPC listener. + #[arg(long, env = "TORII_TLS_KEY")] + pub tls_key: Option, + #[arg(long)] pub observability: bool, @@ -151,6 +159,10 @@ pub struct Config { #[arg(long, default_value = "2")] pub max_prefetch_batches: usize, + /// Delay between ETL idle/retry cycles in seconds. + #[arg(long, default_value = "3")] + pub cycle_interval: u64, + #[arg(long, default_value = "0")] pub rpc_parallelism: usize, @@ -336,6 +348,16 @@ impl Config { .to_string(), )) } + + pub fn tls_config(&self) -> Result> { + match (&self.tls_cert, &self.tls_key) { + (Some(cert), Some(key)) => { + Ok(Some(torii::ToriiTlsConfig::new(cert.clone(), key.clone()))) + } + (None, None) => Ok(None), + _ => bail!("--tls-cert and --tls-key must be provided together"), + } + } } fn is_postgres_url(url: &str) -> bool { @@ -416,6 +438,32 @@ mod tests { ))); } + #[test] + fn tls_flags_parse_when_both_paths_are_present() { + let cfg = Config::parse_from([ + "torii-arcade", + "--tls-cert", + "./certs/dev-cert.pem", + "--tls-key", + "./certs/dev-key.pem", + ]); + + let tls = cfg.tls_config().unwrap().unwrap(); + assert_eq!(tls.cert_path, PathBuf::from("./certs/dev-cert.pem")); + assert_eq!(tls.key_path, PathBuf::from("./certs/dev-key.pem")); + assert_eq!( + tls.alpn_protocols, + vec![b"h2".to_vec(), b"http/1.1".to_vec()] + ); + } + + #[test] + fn tls_flags_require_both_cert_and_key() { + let cfg = Config::parse_from(["torii-arcade", "--tls-cert", "./certs/dev-cert.pem"]); + + assert!(cfg.tls_config().is_err()); + } + #[test] fn postgres_storage_defaults_to_shared_postgres_for_tokens() { let cfg = Config::parse_from([ diff --git a/bins/torii-arcade/src/main.rs b/bins/torii-arcade/src/main.rs index 3075f3cc..6ab09c01 100644 --- a/bins/torii-arcade/src/main.rs +++ b/bins/torii-arcade/src/main.rs @@ -52,7 +52,10 @@ use torii_erc721::{ }; use torii_introspect_postgres_sink::processor::IntrospectPgDb; use torii_introspect_sqlite_sink::processor::IntrospectSqliteDb; -use torii_runtime_common::database::{validate_uniform_backends, DatabaseBackend}; +use torii_runtime_common::database::{ + validate_uniform_backends, DatabaseBackend, DEFAULT_SQLITE_MAX_CONNECTIONS, +}; +use torii_runtime_common::token_support::{resolve_installed_token_support, InstalledTokenSupport}; use torii_sqlite::{is_sqlite_memory_path, sqlite_connect_options}; type StarknetProvider = @@ -207,6 +210,20 @@ impl Sink for ArcadeProjectionPipeline { } } +fn advertised_token_services(installed_token_support: InstalledTokenSupport) -> Vec<&'static str> { + let mut services = Vec::new(); + if installed_token_support.erc20 { + services.push("torii.sinks.erc20.Erc20"); + } + if installed_token_support.erc721 { + services.push("torii.sinks.erc721.Erc721"); + } + if installed_token_support.erc1155 { + services.push("torii.sinks.erc1155.Erc1155"); + } + services +} + #[tokio::main] async fn main() -> Result<()> { let env_filter = tracing_subscriber::EnvFilter::try_from_default_env() @@ -293,8 +310,9 @@ async fn run_indexer(config: Config) -> Result<()> { tracing::info!("Database backend: {:?}", backend); tracing::info!("Metadata mode: {:?}", config.metadata_mode); tracing::info!( - "ETL concurrency: prefetch_batches={} rpc_parallelism={}", + "ETL concurrency: prefetch_batches={} cycle_interval={}s rpc_parallelism={}", config.max_prefetch_batches, + config.cycle_interval, config.rpc_parallelism, ); tracing::info!( @@ -349,9 +367,17 @@ async fn run_indexer(config: Config) -> Result<()> { ) .await?; - let install_erc20 = config.index_external_contracts || !erc20_addresses.is_empty(); - let install_erc721 = config.index_external_contracts || !erc721_addresses.is_empty(); - let install_erc1155 = config.index_external_contracts || !erc1155_addresses.is_empty(); + let installed_token_support = resolve_installed_token_support( + config.index_external_contracts, + InstalledTokenSupport { + erc20: !erc20_addresses.is_empty(), + erc721: !erc721_addresses.is_empty(), + erc1155: !erc1155_addresses.is_empty(), + }, + ); + let install_erc20 = installed_token_support.erc20; + let install_erc721 = installed_token_support.erc721; + let install_erc1155 = installed_token_support.erc1155; let installed_external_decoders = installed_external_decoder_ids( config.index_external_contracts, install_erc20, @@ -396,7 +422,7 @@ async fn run_indexer(config: Config) -> Result<()> { let max_db_connections = match config.max_db_connections { Some(limit) => limit.max(1), None if is_sqlite_memory_path(&storage_database_url) => 1, - None => 1, + None => DEFAULT_SQLITE_MAX_CONNECTIONS, }; let pool = Arc::new( SqlitePoolOptions::new() @@ -471,6 +497,7 @@ async fn run_indexer(config: Config) -> Result<()> { .port(config.port) .database_root(&config.db_dir) .command_bus_queue_size(TOKEN_COMMAND_QUEUE_SIZE) + .cycle_interval(config.cycle_interval) .etl_concurrency(EtlConcurrencyConfig { max_prefetch_batches: config.max_prefetch_batches, }) @@ -480,6 +507,10 @@ async fn run_indexer(config: Config) -> Result<()> { .add_sink_boxed(Box::new(ecs_sink)) .add_sink_boxed(Box::new(arcade_projection_pipeline)); + if let Some(tls) = config.tls_config()? { + torii_config = torii_config.with_tls(tls); + } + if config.index_external_contracts { torii_config = torii_config .with_registry_cache(decoder_registry.clone()) @@ -687,14 +718,8 @@ async fn run_indexer(config: Config) -> Result<()> { tracing::info!(" - torii.Torii"); tracing::info!(" - world.World"); tracing::info!(" - arcade.v1.Arcade"); - if !erc20_addresses.is_empty() { - tracing::info!(" - torii.sinks.erc20.Erc20"); - } - if !erc721_addresses.is_empty() { - tracing::info!(" - torii.sinks.erc721.Erc721"); - } - if !erc1155_addresses.is_empty() { - tracing::info!(" - torii.sinks.erc1155.Erc1155"); + for service in advertised_token_services(installed_token_support) { + tracing::info!(" - {}", service); } torii::run(torii_config) @@ -777,3 +802,38 @@ fn append_unique_contract_configs( } } } + +#[cfg(test)] +mod tests { + use super::advertised_token_services; + use torii_runtime_common::token_support::InstalledTokenSupport; + + #[test] + fn advertised_token_services_include_installed_services_without_explicit_targets() { + let services = advertised_token_services(InstalledTokenSupport { + erc20: true, + erc721: true, + erc1155: true, + }); + + assert_eq!( + services, + vec![ + "torii.sinks.erc20.Erc20", + "torii.sinks.erc721.Erc721", + "torii.sinks.erc1155.Erc1155", + ] + ); + } + + #[test] + fn advertised_token_services_skip_uninstalled_services() { + let services = advertised_token_services(InstalledTokenSupport { + erc20: false, + erc721: true, + erc1155: false, + }); + + assert_eq!(services, vec!["torii.sinks.erc721.Erc721"]); + } +} diff --git a/bins/torii-erc20/src/config.rs b/bins/torii-erc20/src/config.rs index ba4e859d..de20fc71 100644 --- a/bins/torii-erc20/src/config.rs +++ b/bins/torii-erc20/src/config.rs @@ -54,6 +54,10 @@ pub struct Config { /// Port for the HTTP API #[arg(long, default_value = "3000")] pub port: u16, + + /// Delay between ETL idle/retry cycles in seconds. + #[arg(long, default_value = "3")] + pub cycle_interval: u64, } impl Config { diff --git a/bins/torii-erc20/src/main.rs b/bins/torii-erc20/src/main.rs index 33b1272d..82cc1bb3 100644 --- a/bins/torii-erc20/src/main.rs +++ b/bins/torii-erc20/src/main.rs @@ -74,6 +74,7 @@ async fn main() -> Result<()> { tracing::info!("RPC URL: {}", config.rpc_url); tracing::info!("From block: {}", config.from_block); tracing::info!("Database: {}", config.db_path); + tracing::info!("ETL cycle interval: {}s", config.cycle_interval); if let Some(url) = &config.database_url { tracing::info!("Engine database URL: {}", url); } @@ -146,6 +147,7 @@ async fn main() -> Result<()> { .port(config.port) .database_root(&db_setup.database_root) .command_bus_queue_size(ERC20_METADATA_COMMAND_QUEUE_SIZE) + .cycle_interval(config.cycle_interval) .engine_database_url(db_setup.engine_url.clone()) .with_extractor(extractor) .add_decoder(decoder) diff --git a/bins/torii-introspect-bin/README.md b/bins/torii-introspect-bin/README.md index 590e4b9a..ef6cdc14 100644 --- a/bins/torii-introspect-bin/README.md +++ b/bins/torii-introspect-bin/README.md @@ -45,6 +45,39 @@ Notes: - `--batch-size`: block range queried per iteration. - `--max-prefetch-batches`: extracted batches buffered ahead of decode/store. +## Local TLS + ALPN + +For browser-compatible local HTTPS, use `mkcert` instead of a raw self-signed certificate. +Modern browsers require a trusted local CA and `subjectAltName` entries for `localhost`. + +Install and trust the local development CA: + +```bash +brew install mkcert nss +mkcert -install +mkdir -p certs +mkcert -cert-file certs/dev-cert.pem -key-file certs/dev-key.pem localhost 127.0.0.1 ::1 +``` + +Start `torii-server` with TLS enabled: + +```bash +cargo run --bin torii-server -- \ + --contract 0x123...,0x456... \ + --tls-cert ./certs/dev-cert.pem \ + --tls-key ./certs/dev-key.pem +``` + +The local listener advertises ALPN for `h2` and `http/1.1`. Native gRPC clients will negotiate +HTTP/2 automatically; HTTPS health and metrics endpoints remain available on the same port. + +Verify the local HTTPS listener: + +```bash +curl https://localhost:3000/health +grpcurl -insecure localhost:3000 list +``` + ## Notes - `--storage-database-url` must be PostgreSQL when provided. diff --git a/bins/torii-introspect-bin/src/config.rs b/bins/torii-introspect-bin/src/config.rs index 851c360e..2af7c2bf 100644 --- a/bins/torii-introspect-bin/src/config.rs +++ b/bins/torii-introspect-bin/src/config.rs @@ -1,7 +1,7 @@ use anyhow::{bail, Result}; use clap::{ArgGroup, Parser}; use starknet::core::types::Felt; -use std::path::Path; +use std::path::{Path, PathBuf}; #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum StorageBackend { @@ -85,6 +85,14 @@ pub struct Config { #[arg(long, default_value = "3000")] pub port: u16, + /// PEM-encoded TLS certificate for the local HTTP/gRPC listener. + #[arg(long, env = "TORII_TLS_CERT")] + pub tls_cert: Option, + + /// PEM-encoded TLS private key for the local HTTP/gRPC listener. + #[arg(long, env = "TORII_TLS_KEY")] + pub tls_key: Option, + /// Cartridge-compatible GraphQL API used to fetch controller usernames. #[arg(long, default_value = "https://api.cartridge.gg/query")] pub controllers_api_url: String, @@ -109,6 +117,10 @@ pub struct Config { #[arg(long, default_value = "2")] pub max_prefetch_batches: usize, + /// Delay between ETL idle/retry cycles in seconds. + #[arg(long, default_value = "3")] + pub cycle_interval: u64, + /// Maximum chunked RPC requests to run concurrently (`0` = auto). #[arg(long, default_value = "0")] pub rpc_parallelism: usize, @@ -201,6 +213,16 @@ impl Config { None => Ok(db_dir.join("introspect.db").to_string_lossy().to_string()), } } + + pub fn tls_config(&self) -> Result> { + match (&self.tls_cert, &self.tls_key) { + (Some(cert), Some(key)) => { + Ok(Some(torii::ToriiTlsConfig::new(cert.clone(), key.clone()))) + } + (None, None) => Ok(None), + _ => bail!("--tls-cert and --tls-key must be provided together"), + } + } } #[cfg(test)] @@ -289,6 +311,40 @@ mod tests { assert_eq!(cfg.rpc_parallelism, 6); } + #[test] + fn tls_flags_parse_when_both_paths_are_present() { + let cfg = Config::parse_from([ + "torii-server", + "--contract", + "0x1", + "--tls-cert", + "./certs/dev-cert.pem", + "--tls-key", + "./certs/dev-key.pem", + ]); + + let tls = cfg.tls_config().unwrap().unwrap(); + assert_eq!(tls.cert_path, PathBuf::from("./certs/dev-cert.pem")); + assert_eq!(tls.key_path, PathBuf::from("./certs/dev-key.pem")); + assert_eq!( + tls.alpn_protocols, + vec![b"h2".to_vec(), b"http/1.1".to_vec()] + ); + } + + #[test] + fn tls_flags_require_both_cert_and_key() { + let cfg = Config::parse_from([ + "torii-server", + "--contract", + "0x1", + "--tls-cert", + "./certs/dev-cert.pem", + ]); + + assert!(cfg.tls_config().is_err()); + } + #[test] fn token_flags_parse() { let cfg = Config::parse_from([ diff --git a/bins/torii-introspect-bin/src/main.rs b/bins/torii-introspect-bin/src/main.rs index 0c2bc8ce..96f523eb 100644 --- a/bins/torii-introspect-bin/src/main.rs +++ b/bins/torii-introspect-bin/src/main.rs @@ -49,7 +49,10 @@ use torii_erc721::{ }; use torii_introspect_postgres_sink::processor::IntrospectPgDb; use torii_introspect_sqlite_sink::processor::IntrospectSqliteDb; -use torii_runtime_common::database::{resolve_token_db_setup, TokenDbSetup}; +use torii_runtime_common::database::{ + resolve_token_db_setup, TokenDbSetup, DEFAULT_SQLITE_MAX_CONNECTIONS, +}; +use torii_runtime_common::token_support::{resolve_installed_token_support, InstalledTokenSupport}; use torii_sqlite::{is_sqlite_memory_path, sqlite_connect_options}; type StarknetProvider = @@ -156,10 +159,6 @@ impl TokenTargets { }) } - fn has_any(&self) -> bool { - !self.erc20.is_empty() || !self.erc721.is_empty() || !self.erc1155.is_empty() - } - fn total_len(&self) -> usize { self.erc20.len() + self.erc721.len() + self.erc1155.len() } @@ -290,10 +289,45 @@ fn apply_contract_mappings( torii_config } +fn ecs_token_storage_urls( + token_db_setup: Option<&TokenDbSetup>, + installed_token_support: InstalledTokenSupport, +) -> (Option<&str>, Option<&str>, Option<&str>) { + if !installed_token_support.any() { + return (None, None, None); + } + + let db_setup = + token_db_setup.expect("token DB setup must exist when installed token support is enabled"); + + ( + installed_token_support + .erc20 + .then_some(db_setup.erc20_url.as_str()), + installed_token_support + .erc721 + .then_some(db_setup.erc721_url.as_str()), + installed_token_support + .erc1155 + .then_some(db_setup.erc1155_url.as_str()), + ) +} + +fn log_installed_token_services(installed_token_support: InstalledTokenSupport) { + if installed_token_support.erc20 { + tracing::info!(" - torii.sinks.erc20.Erc20"); + } + if installed_token_support.erc721 { + tracing::info!(" - torii.sinks.erc721.Erc721"); + } + if installed_token_support.erc1155 { + tracing::info!(" - torii.sinks.erc1155.Erc1155"); + } +} + async fn configure_token_support( - token_targets: &TokenTargets, + installed_token_support: InstalledTokenSupport, token_db_setup: Option<&TokenDbSetup>, - enable_external_contracts: bool, provider: Arc, mut torii_config: ToriiConfigBuilder, mut reflection_builder: ReflectionBuilder, @@ -305,11 +339,8 @@ async fn configure_token_support( )> { let mut services = TokenGrpcServices::default(); let mut token_uri_services = Vec::new(); - let install_erc20 = enable_external_contracts || !token_targets.erc20.is_empty(); - let install_erc721 = enable_external_contracts || !token_targets.erc721.is_empty(); - let install_erc1155 = enable_external_contracts || !token_targets.erc1155.is_empty(); - if !install_erc20 && !install_erc721 && !install_erc1155 { + if !installed_token_support.any() { return Ok(( torii_config, reflection_builder, @@ -321,7 +352,7 @@ async fn configure_token_support( let db_setup = token_db_setup.expect("token DB setup must exist when token support is configured"); - if install_erc20 { + if installed_token_support.erc20 { let storage = Arc::new(Erc20Storage::new(&db_setup.erc20_url).await?); tracing::info!("ERC20 database initialized: {}", db_setup.erc20_url); @@ -351,7 +382,7 @@ async fn configure_token_support( reflection_builder.register_encoded_file_descriptor_set(ERC20_DESCRIPTOR_SET); } - if install_erc721 { + if installed_token_support.erc721 { let storage = Arc::new(Erc721Storage::new(&db_setup.erc721_url).await?); tracing::info!("ERC721 database initialized: {}", db_setup.erc721_url); @@ -386,7 +417,7 @@ async fn configure_token_support( reflection_builder.register_encoded_file_descriptor_set(ERC721_DESCRIPTOR_SET); } - if install_erc1155 { + if installed_token_support.erc1155 { let storage = Arc::new(Erc1155Storage::new(&db_setup.erc1155_url).await?); tracing::info!("ERC1155 database initialized: {}", db_setup.erc1155_url); @@ -453,8 +484,16 @@ async fn run_indexer(config: Config) -> Result<()> { let engine_database_url = config.engine_database_url(db_dir); let contracts = config.contract_addresses()?; let token_targets = TokenTargets::from_config(&config)?; + let installed_token_support = resolve_installed_token_support( + config.index_external_contracts, + InstalledTokenSupport { + erc20: !token_targets.erc20.is_empty(), + erc721: !token_targets.erc721.is_empty(), + erc1155: !token_targets.erc1155.is_empty(), + }, + ); let historical_models = config.historical_models(); - let token_db_setup = if token_targets.has_any() || config.index_external_contracts { + let token_db_setup = if installed_token_support.any() { Some(resolve_token_db_setup( db_dir, config.database_url.as_deref(), @@ -501,8 +540,9 @@ async fn run_indexer(config: Config) -> Result<()> { } tracing::info!("Database backend: {:?}", backend); tracing::info!( - "ETL concurrency: prefetch_batches={} rpc_parallelism={}", + "ETL concurrency: prefetch_batches={} cycle_interval={}s rpc_parallelism={}", config.max_prefetch_batches, + config.cycle_interval, config.rpc_parallelism, ); tracing::info!( @@ -610,6 +650,7 @@ async fn run_indexer(config: Config) -> Result<()> { engine_database_url, contracts, token_targets, + installed_token_support, token_db_setup, registry_engine_db.clone(), decoder_registry.clone(), @@ -628,6 +669,7 @@ async fn run_indexer(config: Config) -> Result<()> { engine_database_url, contracts, token_targets, + installed_token_support, token_db_setup, registry_engine_db.clone(), decoder_registry.clone(), @@ -651,6 +693,7 @@ async fn run_with_postgres( engine_database_url: String, contracts: Vec, token_targets: TokenTargets, + installed_token_support: InstalledTokenSupport, token_db_setup: Option, registry_engine_db: Arc, decoder_registry: SharedDecoderRegistry, @@ -676,18 +719,8 @@ async fn run_with_postgres( let decoder: Arc = Arc::new(decoder); - let erc20_url = token_db_setup - .as_ref() - .filter(|_| !token_targets.erc20.is_empty()) - .map(|s| s.erc20_url.as_str()); - let erc721_url = token_db_setup - .as_ref() - .filter(|_| !token_targets.erc721.is_empty()) - .map(|s| s.erc721_url.as_str()); - let erc1155_url = token_db_setup - .as_ref() - .filter(|_| !token_targets.erc1155.is_empty()) - .map(|s| s.erc1155_url.as_str()); + let (erc20_url, erc721_url, erc1155_url) = + ecs_token_storage_urls(token_db_setup.as_ref(), installed_token_support); let ecs_sink = EcsSink::new( storage_database_url, config.max_db_connections, @@ -710,6 +743,7 @@ async fn run_with_postgres( .port(config.port) .command_bus_queue_size(TOKEN_COMMAND_QUEUE_SIZE) .with_custom_reflection(true) + .cycle_interval(config.cycle_interval) .etl_concurrency(EtlConcurrencyConfig { max_prefetch_batches: config.max_prefetch_batches, }) @@ -730,6 +764,9 @@ async fn run_with_postgres( .await?, )), )); + if let Some(tls) = config.tls_config()? { + torii_config = torii_config.with_tls(tls); + } if config.index_external_contracts { torii_config = torii_config .with_registry_cache(decoder_registry.clone()) @@ -754,9 +791,8 @@ async fn run_with_postgres( let (torii_config, reflection_builder, token_services, _token_uri_services) = configure_token_support( - &token_targets, + installed_token_support, token_db_setup.as_ref(), - config.index_external_contracts, token_provider, torii_config, reflection_builder, @@ -823,15 +859,7 @@ async fn run_with_postgres( tracing::info!("gRPC service available on port {}", config.port); tracing::info!(" - torii.Torii (core subscriptions and metrics endpoint)"); tracing::info!(" - world.World (legacy ECS gRPC service)"); - if !token_targets.erc20.is_empty() { - tracing::info!(" - torii.sinks.erc20.Erc20"); - } - if !token_targets.erc721.is_empty() { - tracing::info!(" - torii.sinks.erc721.Erc721"); - } - if !token_targets.erc1155.is_empty() { - tracing::info!(" - torii.sinks.erc1155.Erc1155"); - } + log_installed_token_services(installed_token_support); torii::run(torii_config.build()) .await @@ -845,6 +873,7 @@ async fn run_with_sqlite( engine_database_url: String, contracts: Vec, token_targets: TokenTargets, + installed_token_support: InstalledTokenSupport, token_db_setup: Option, registry_engine_db: Arc, decoder_registry: SharedDecoderRegistry, @@ -859,7 +888,7 @@ async fn run_with_sqlite( let max_db_connections = match config.max_db_connections { Some(limit) => limit.max(1), None if is_sqlite_memory_path(storage_database_url) => 1, - None => 1, + None => DEFAULT_SQLITE_MAX_CONNECTIONS, }; let pool = Arc::new( SqlitePoolOptions::new() @@ -884,18 +913,8 @@ async fn run_with_sqlite( let decoder: Arc = Arc::new(decoder); - let erc20_url = token_db_setup - .as_ref() - .filter(|_| !token_targets.erc20.is_empty()) - .map(|s| s.erc20_url.as_str()); - let erc721_url = token_db_setup - .as_ref() - .filter(|_| !token_targets.erc721.is_empty()) - .map(|s| s.erc721_url.as_str()); - let erc1155_url = token_db_setup - .as_ref() - .filter(|_| !token_targets.erc1155.is_empty()) - .map(|s| s.erc1155_url.as_str()); + let (erc20_url, erc721_url, erc1155_url) = + ecs_token_storage_urls(token_db_setup.as_ref(), installed_token_support); let ecs_sink = EcsSink::new( storage_database_url, config.max_db_connections, @@ -918,6 +937,7 @@ async fn run_with_sqlite( .port(config.port) .command_bus_queue_size(TOKEN_COMMAND_QUEUE_SIZE) .with_custom_reflection(true) + .cycle_interval(config.cycle_interval) .etl_concurrency(EtlConcurrencyConfig { max_prefetch_batches: config.max_prefetch_batches, }) @@ -938,6 +958,9 @@ async fn run_with_sqlite( .await?, )), )); + if let Some(tls) = config.tls_config()? { + torii_config = torii_config.with_tls(tls); + } if config.index_external_contracts { torii_config = torii_config .with_registry_cache(decoder_registry.clone()) @@ -962,9 +985,8 @@ async fn run_with_sqlite( let (torii_config, reflection_builder, token_services, _token_uri_services) = configure_token_support( - &token_targets, + installed_token_support, token_db_setup.as_ref(), - config.index_external_contracts, token_provider, torii_config, reflection_builder, @@ -1031,18 +1053,53 @@ async fn run_with_sqlite( tracing::info!("gRPC service available on port {}", config.port); tracing::info!(" - torii.Torii (core subscriptions and metrics endpoint)"); tracing::info!(" - world.World (legacy ECS gRPC service)"); - if !token_targets.erc20.is_empty() { - tracing::info!(" - torii.sinks.erc20.Erc20"); - } - if !token_targets.erc721.is_empty() { - tracing::info!(" - torii.sinks.erc721.Erc721"); - } - if !token_targets.erc1155.is_empty() { - tracing::info!(" - torii.sinks.erc1155.Erc1155"); - } + log_installed_token_services(installed_token_support); torii::run(torii_config.build()) .await .map_err(|e| anyhow::anyhow!("Torii error: {e}"))?; Ok(()) } + +#[cfg(test)] +mod tests { + use super::{ecs_token_storage_urls, InstalledTokenSupport}; + use torii_runtime_common::database::{DatabaseBackend, TokenDbSetup}; + + fn token_db_setup() -> TokenDbSetup { + TokenDbSetup { + engine_url: "./torii-data/engine.db".to_string(), + erc20_url: "./torii-data/erc20.db".to_string(), + erc721_url: "./torii-data/erc721.db".to_string(), + erc1155_url: "./torii-data/erc1155.db".to_string(), + engine_backend: DatabaseBackend::Sqlite, + erc20_backend: DatabaseBackend::Sqlite, + erc721_backend: DatabaseBackend::Sqlite, + erc1155_backend: DatabaseBackend::Sqlite, + } + } + + #[test] + fn ecs_token_storage_urls_follow_installed_support() { + let db_setup = token_db_setup(); + let (erc20_url, erc721_url, erc1155_url) = ecs_token_storage_urls( + Some(&db_setup), + InstalledTokenSupport { + erc20: true, + erc721: false, + erc1155: true, + }, + ); + + assert_eq!(erc20_url, Some("./torii-data/erc20.db")); + assert_eq!(erc721_url, None); + assert_eq!(erc1155_url, Some("./torii-data/erc1155.db")); + } + + #[test] + fn ecs_token_storage_urls_are_none_when_no_token_support_is_installed() { + let db_setup = token_db_setup(); + let urls = ecs_token_storage_urls(Some(&db_setup), InstalledTokenSupport::default()); + assert_eq!(urls, (None, None, None)); + } +} diff --git a/bins/torii-tokens/src/config.rs b/bins/torii-tokens/src/config.rs index f00175e1..cbd52b3d 100644 --- a/bins/torii-tokens/src/config.rs +++ b/bins/torii-tokens/src/config.rs @@ -170,6 +170,10 @@ pub struct Config { #[arg(long, default_value = "2")] pub max_prefetch_batches: usize, + /// Delay between ETL idle/retry cycles in seconds. + #[arg(long, default_value = "3")] + pub cycle_interval: u64, + /// Maximum chunked RPC requests to run concurrently (`0` = auto). #[arg(long, default_value = "0")] pub rpc_parallelism: usize, diff --git a/bins/torii-tokens/src/main.rs b/bins/torii-tokens/src/main.rs index f81d81ac..3d49c1ff 100644 --- a/bins/torii-tokens/src/main.rs +++ b/bins/torii-tokens/src/main.rs @@ -514,6 +514,7 @@ async fn run_indexer(config: Config) -> Result<()> { let mut torii_config = torii::ToriiConfig::builder() .port(config.port) .database_root(&config.db_dir) + .cycle_interval(config.cycle_interval) .etl_concurrency(EtlConcurrencyConfig { max_prefetch_batches: config.max_prefetch_batches, }) @@ -745,8 +746,9 @@ async fn run_indexer(config: Config) -> Result<()> { tracing::info!("Torii configured, starting ETL pipeline..."); tracing::info!("Enabled token types: {}", enabled_types.join(", ")); tracing::info!( - "ETL concurrency: prefetch_batches={} rpc_parallelism={} metadata_parallelism={} metadata_queue_capacity={} metadata_max_retries={}", + "ETL concurrency: prefetch_batches={} cycle_interval={}s rpc_parallelism={} metadata_parallelism={} metadata_queue_capacity={} metadata_max_retries={}", config.max_prefetch_batches, + config.cycle_interval, config.rpc_parallelism, config.metadata_parallelism, config.metadata_queue_capacity, diff --git a/certs/.gitkeep b/certs/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/crates/arcade-sink/Cargo.toml b/crates/arcade-sink/Cargo.toml index b7dd3e35..1c182a02 100644 --- a/crates/arcade-sink/Cargo.toml +++ b/crates/arcade-sink/Cargo.toml @@ -8,6 +8,7 @@ torii = { path = "../.." } torii-common = { path = "../torii-common" } torii-introspect = { path = "../introspect" } torii-erc721 = { path = "../torii-erc721" } +torii-runtime-common.workspace = true anyhow.workspace = true async-trait.workspace = true diff --git a/crates/arcade-sink/src/grpc_service.rs b/crates/arcade-sink/src/grpc_service.rs index ae65a60f..205cf48c 100644 --- a/crates/arcade-sink/src/grpc_service.rs +++ b/crates/arcade-sink/src/grpc_service.rs @@ -11,6 +11,7 @@ use starknet::core::types::{Felt, U256}; use tonic::{Request, Response, Status}; use torii_common::{blob_to_u256, u256_to_blob}; use torii_erc721::{storage::OwnershipCursor, Erc721Storage}; +use torii_runtime_common::database::DEFAULT_SQLITE_MAX_CONNECTIONS; use crate::proto::arcade::{ arcade_server::Arcade, Collection, Edition, Game, GetPlayerInventoryRequest, @@ -111,7 +112,7 @@ impl ArcadeService { let pool = AnyPoolOptions::new() .max_connections(max_connections.unwrap_or(if backend == DbBackend::Sqlite { - 1 + DEFAULT_SQLITE_MAX_CONNECTIONS } else { 5 })) diff --git a/crates/introspect-sqlite-sink/src/processor.rs b/crates/introspect-sqlite-sink/src/processor.rs index 07e14456..d3a00a93 100644 --- a/crates/introspect-sqlite-sink/src/processor.rs +++ b/crates/introspect-sqlite-sink/src/processor.rs @@ -595,50 +595,6 @@ impl IntrospectSqliteDb { .map(|id| &table.columns[id].type_def) .collect(); - let placeholders = std::iter::once("?".to_string()) - .chain(column_type_defs.iter().map(|td| { - if is_json_type(td) { - "jsonb(?)".to_string() - } else { - "?".to_string() - } - })) - .collect::>() - .join(", "); - - let update_columns = column_names - .iter() - .skip(1) - .zip(column_type_defs.iter()) - .map(|(name, td)| { - if is_json_type(td) { - format!( - r#""{name}" = COALESCE(jsonb(excluded."{name}"), "{table_name}"."{name}")"#, - table_name = table.storage_name - ) - } else { - format!( - r#""{name}" = COALESCE(excluded."{name}", "{table_name}"."{name}")"#, - table_name = table.storage_name - ) - } - }) - .collect::>() - .join(", "); - - let sql = format!( - r#"INSERT INTO "{}" ({}) VALUES ({}) ON CONFLICT("{}") DO UPDATE SET {}"#, - table.storage_name, - column_names - .iter() - .map(|name| format!(r#""{name}""#)) - .collect::>() - .join(", "), - placeholders, - table.primary.name, - update_columns - ); - let mut bytes = Vec::new(); let mut serializer = JsonSerializer::new(&mut bytes); record_schema.parse_records_with_metadata( @@ -653,7 +609,7 @@ impl IntrospectSqliteDb { for value in rows { let object = value.as_object().ok_or(SqliteDbError::InvalidRecordFrame)?; - let mut query = sqlx::query(&sql); + let mut query = sqlx::query(&table.upsert_sql); let primary_value = object .get(table.primary.name.as_str()) diff --git a/crates/introspect-sqlite-sink/src/table.rs b/crates/introspect-sqlite-sink/src/table.rs index 14b7a559..8bbb653e 100644 --- a/crates/introspect-sqlite-sink/src/table.rs +++ b/crates/introspect-sqlite-sink/src/table.rs @@ -21,6 +21,7 @@ pub struct SqliteTable { pub primary: PrimaryDef, pub columns: HashMap, pub order: Vec, + pub upsert_sql: String, pub alive: bool, } @@ -37,8 +38,10 @@ impl SqliteTable { primary, order: columns.ids(), columns: columns.into_iter().map_into().collect(), + upsert_sql: String::new(), alive: true, } + .with_upsert_sql() } pub fn new_from_table(namespace: &str, table: impl Into) -> (Felt, Self) { @@ -67,4 +70,82 @@ impl SqliteTable { .collect::, _>>()?; Ok(RecordSchema::new(&self.primary, columns)) } + + fn with_upsert_sql(mut self) -> Self { + self.upsert_sql = build_upsert_sql(&self); + self + } +} + +fn sqlite_column_type(type_def: &introspect_types::TypeDef) -> &'static str { + if matches!( + type_def, + introspect_types::TypeDef::Struct(_) + | introspect_types::TypeDef::Enum(_) + | introspect_types::TypeDef::Tuple(_) + | introspect_types::TypeDef::Array(_) + | introspect_types::TypeDef::FixedArray(_) + | introspect_types::TypeDef::Option(_) + | introspect_types::TypeDef::Nullable(_) + | introspect_types::TypeDef::Result(_) + ) { + "JSONB" + } else { + "" + } +} + +fn build_upsert_sql(table: &SqliteTable) -> String { + let column_names = std::iter::once(table.primary.name.as_str()) + .chain(table.order.iter().map(|id| table.columns[id].name.as_str())) + .collect::>(); + let column_type_defs = table + .order + .iter() + .map(|id| &table.columns[id].type_def) + .collect::>(); + + let placeholders = std::iter::once("?".to_string()) + .chain(column_type_defs.iter().map(|td| { + if sqlite_column_type(td) == "JSONB" { + "jsonb(?)".to_string() + } else { + "?".to_string() + } + })) + .collect::>() + .join(", "); + + let update_columns = column_names + .iter() + .skip(1) + .zip(column_type_defs.iter()) + .map(|(name, td)| { + if sqlite_column_type(td) == "JSONB" { + format!( + r#""{name}" = COALESCE(jsonb(excluded."{name}"), "{table_name}"."{name}")"#, + table_name = table.storage_name + ) + } else { + format!( + r#""{name}" = COALESCE(excluded."{name}", "{table_name}"."{name}")"#, + table_name = table.storage_name + ) + } + }) + .collect::>() + .join(", "); + + format!( + r#"INSERT INTO "{}" ({}) VALUES ({}) ON CONFLICT("{}") DO UPDATE SET {}"#, + table.storage_name, + column_names + .iter() + .map(|name| format!(r#""{name}""#)) + .collect::>() + .join(", "), + placeholders, + table.primary.name, + update_columns + ) } diff --git a/crates/torii-controllers-sink/Cargo.toml b/crates/torii-controllers-sink/Cargo.toml index 0879849c..4f5c07a6 100644 --- a/crates/torii-controllers-sink/Cargo.toml +++ b/crates/torii-controllers-sink/Cargo.toml @@ -5,6 +5,7 @@ edition = "2021" [dependencies] torii = { path = "../.." } +torii-runtime-common.workspace = true anyhow.workspace = true async-trait.workspace = true diff --git a/crates/torii-controllers-sink/src/lib.rs b/crates/torii-controllers-sink/src/lib.rs index 1e56d498..3005d2a5 100644 --- a/crates/torii-controllers-sink/src/lib.rs +++ b/crates/torii-controllers-sink/src/lib.rs @@ -14,6 +14,7 @@ use torii::axum::Router; use torii::etl::extractor::ExtractionBatch; use torii::etl::sink::{EventBus, Sink, SinkContext, TopicInfo}; use torii::etl::TypeId; +use torii_runtime_common::database::DEFAULT_SQLITE_MAX_CONNECTIONS; pub const DEFAULT_API_QUERY_URL: &str = "https://api.cartridge.gg/query"; pub const CONTROLLERS_TABLE: &str = "controllers"; @@ -133,7 +134,7 @@ impl ControllersStore { let pool = AnyPoolOptions::new() .max_connections(max_connections.unwrap_or(if backend == DbBackend::Sqlite { - 1 + DEFAULT_SQLITE_MAX_CONNECTIONS } else { 5 })) diff --git a/crates/torii-ecs-sink/Cargo.toml b/crates/torii-ecs-sink/Cargo.toml index 3d1458a1..3598c8c9 100644 --- a/crates/torii-ecs-sink/Cargo.toml +++ b/crates/torii-ecs-sink/Cargo.toml @@ -7,6 +7,7 @@ edition = "2021" torii = { path = "../.." } torii-dojo = { path = "../dojo" } torii-introspect = { path = "../introspect" } +torii-runtime-common.workspace = true dojo-introspect.workspace = true introspect-types.workspace = true diff --git a/crates/torii-ecs-sink/src/grpc_service.rs b/crates/torii-ecs-sink/src/grpc_service.rs index 25c0c159..5013a67c 100644 --- a/crates/torii-ecs-sink/src/grpc_service.rs +++ b/crates/torii-ecs-sink/src/grpc_service.rs @@ -1,6 +1,8 @@ -use std::collections::{HashMap, HashSet}; +use std::collections::{HashMap, HashSet, VecDeque}; +use std::pin::Pin; use std::str::FromStr; use std::sync::Arc; +use std::task::{Context, Poll}; use std::time::Instant; use anyhow::{anyhow, Result}; @@ -13,9 +15,11 @@ use introspect_types::{ }; use num_bigint::{BigInt, BigUint, Sign}; use primitive_types::{U256, U512}; +use prost::Message; use serde::ser::SerializeMap; use serde::Serializer; use serde_json::{Map, Serializer as JsonSerializer, Value}; +use sqlx::AnyConnection; use sqlx::{ any::AnyPoolOptions, pool::PoolConnection, postgres::PgPoolOptions, sqlite::SqliteConnectOptions, sqlite::SqlitePoolOptions, Any, Column, ConnectOptions, Pool, @@ -23,9 +27,9 @@ use sqlx::{ }; use starknet::core::types::Felt; use tokio::sync::mpsc::error::TrySendError; -use tokio::sync::{mpsc, Mutex}; +use tokio::sync::{mpsc, Mutex, RwLock}; use tokio::time::{sleep, Duration}; -use tokio_stream::wrappers::ReceiverStream; +use tokio_stream::{wrappers::ReceiverStream, Stream}; use tonic::{Request, Response, Status}; use torii_dojo::store::postgres::PgStore; use torii_dojo::store::sqlite::SqliteStore; @@ -33,6 +37,7 @@ use torii_dojo::store::DojoStoreTrait; use torii_dojo::DojoTable; use torii_introspect::events::{CreateTable, Record, UpdateTable}; use torii_introspect::schema::TableSchema; +use torii_runtime_common::database::DEFAULT_SQLITE_MAX_CONNECTIONS; use crate::proto::types::{ self, clause::ClauseType, member_value::ValueType, ComparisonOperator, ContractType, @@ -55,6 +60,8 @@ use crate::proto::world::{ WorldsResponse, }; +const SUBSCRIPTION_SEEN_CACHE_CAPACITY: usize = 4096; + #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum TableKind { Entity, @@ -92,6 +99,13 @@ impl DbBackend { Self::Sqlite } } + + fn as_str(self) -> &'static str { + match self { + Self::Sqlite => "sqlite", + Self::Postgres => "postgres", + } + } } #[derive(Clone)] @@ -99,6 +113,11 @@ pub struct EcsService { state: Arc, } +struct CachedViewsSql { + controllers: String, + token_balances: String, +} + struct EcsState { pool: Pool, backend: DbBackend, @@ -110,8 +129,9 @@ struct EcsState { erc721_url: Option, erc1155_url: Option, managed_tables: Mutex>>>, - entity_subscriptions: Mutex>, - event_message_subscriptions: Mutex>, + cached_views_sql: RwLock>, + entity_subscriptions: RwLock, + event_message_subscriptions: RwLock, event_subscriptions: Mutex>, contract_subscriptions: Mutex>, token_subscriptions: Mutex>, @@ -126,6 +146,111 @@ struct EntitySubscription { sender: mpsc::Sender>, } +#[derive(Default)] +struct EntitySubscriptionRegistry { + subscriptions: HashMap, + groups: HashMap, +} + +#[derive(Clone)] +struct EntitySubscriptionEntry { + key: EntitySubscriptionKey, + sender: mpsc::Sender>, +} + +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +struct EntitySubscriptionKey { + clause_bytes: Vec, + world_addresses: Vec>, +} + +struct SharedEntitySubscription { + clause: Option, + world_addresses: HashSet>, + senders: HashMap>>, +} + +#[derive(Clone)] +struct EntitySubscriptionGroupSnapshot { + clause: Option, + world_addresses: HashSet>, + senders: Vec<(u64, mpsc::Sender>)>, +} + +pub struct ManagedSubscriptionStream { + stream: ReceiverStream>, + cleanup: Option, +} + +struct SubscriptionCleanup { + state: Arc, + kind: SubscriptionKind, + subscription_id: u64, +} + +#[derive(Clone, Copy)] +enum SubscriptionKind { + Entity(TableKind), + Event, + Contract, + Token, + TokenBalance, + TokenTransfer, + Transaction, +} + +#[derive(Clone, Default)] +struct SeenCache { + items: HashSet, + order: VecDeque, + capacity: usize, +} + +impl SeenCache { + fn with_capacity(capacity: usize) -> Self { + Self { + items: HashSet::new(), + order: VecDeque::new(), + capacity, + } + } + + fn from_items(items: I, capacity: usize) -> Self + where + I: IntoIterator, + { + let mut cache = Self::with_capacity(capacity); + cache.extend(items); + cache + } + + fn contains(&self, item: &str) -> bool { + self.items.contains(item) + } + + fn extend(&mut self, items: I) + where + I: IntoIterator, + { + for item in items { + if self.items.insert(item.clone()) { + self.order.push_back(item); + } + while self.items.len() > self.capacity { + if let Some(evicted) = self.order.pop_front() { + self.items.remove(&evicted); + } else { + break; + } + } + } + } + + fn snapshot(&self) -> Self { + self.clone() + } +} + struct EventSubscription { keys: Vec, sender: mpsc::Sender>, @@ -139,7 +264,7 @@ struct ContractSubscription { struct TokenSubscription { contract_addresses: Vec>, token_ids: Vec>, - seen: HashSet, + seen: SeenCache, sender: mpsc::Sender>, } @@ -147,7 +272,7 @@ struct TokenBalanceSubscription { contract_addresses: Vec>, account_addresses: Vec>, token_ids: Vec>, - seen: HashSet, + latest: HashMap>, sender: mpsc::Sender>, } @@ -155,16 +280,332 @@ struct TokenTransferSubscription { contract_addresses: Vec>, account_addresses: Vec>, token_ids: Vec>, - seen: HashSet, + seen: SeenCache, sender: mpsc::Sender>, } struct TransactionSubscription { filter: types::TransactionFilter, - seen: HashSet, + seen: SeenCache, sender: mpsc::Sender>, } +impl EntitySubscriptionKey { + fn new(clause: Option<&types::Clause>, world_addresses: &HashSet>) -> Self { + let mut world_addresses = world_addresses.iter().cloned().collect::>(); + world_addresses.sort(); + + Self { + clause_bytes: clause.map_or_else(Vec::new, Message::encode_to_vec), + world_addresses, + } + } +} + +impl EntitySubscriptionRegistry { + fn insert(&mut self, subscription_id: u64, subscription: EntitySubscription) { + let key = + EntitySubscriptionKey::new(subscription.clause.as_ref(), &subscription.world_addresses); + let sender = subscription.sender.clone(); + + self.groups + .entry(key.clone()) + .or_insert_with(|| SharedEntitySubscription { + clause: subscription.clause, + world_addresses: subscription.world_addresses, + senders: HashMap::new(), + }) + .senders + .insert(subscription_id, sender.clone()); + + self.subscriptions + .insert(subscription_id, EntitySubscriptionEntry { key, sender }); + } + + fn update( + &mut self, + subscription_id: u64, + clause: Option, + world_addresses: HashSet>, + ) -> bool { + let Some(entry) = self.subscriptions.get_mut(&subscription_id) else { + return false; + }; + + let next_key = EntitySubscriptionKey::new(clause.as_ref(), &world_addresses); + if entry.key == next_key { + return false; + } + + let sender = entry.sender.clone(); + let previous_key = entry.key.clone(); + let remove_previous_group = if let Some(group) = self.groups.get_mut(&previous_key) { + group.senders.remove(&subscription_id); + group.senders.is_empty() + } else { + false + }; + if remove_previous_group { + self.groups.remove(&previous_key); + } + + self.groups + .entry(next_key.clone()) + .or_insert_with(|| SharedEntitySubscription { + clause, + world_addresses, + senders: HashMap::new(), + }) + .senders + .insert(subscription_id, sender); + + entry.key = next_key; + true + } + + fn remove(&mut self, subscription_id: u64) -> bool { + let Some(entry) = self.subscriptions.remove(&subscription_id) else { + return false; + }; + + let remove_group = if let Some(group) = self.groups.get_mut(&entry.key) { + group.senders.remove(&subscription_id); + group.senders.is_empty() + } else { + false + }; + if remove_group { + self.groups.remove(&entry.key); + } + + true + } + + fn group_count(&self) -> usize { + self.groups.len() + } + + fn subscription_count(&self) -> usize { + self.subscriptions.len() + } + + fn is_empty(&self) -> bool { + self.subscriptions.is_empty() + } + + fn snapshot_groups(&self) -> Vec { + self.groups + .values() + .map(|group| EntitySubscriptionGroupSnapshot { + clause: group.clause.clone(), + world_addresses: group.world_addresses.clone(), + senders: group + .senders + .iter() + .map(|(&subscription_id, sender)| (subscription_id, sender.clone())) + .collect(), + }) + .collect() + } +} + +impl EcsState { + fn entity_subscription_registry(&self, kind: TableKind) -> &RwLock { + match kind { + TableKind::Entity => &self.entity_subscriptions, + TableKind::EventMessage => &self.event_message_subscriptions, + } + } + + async fn insert_entity_subscription( + &self, + kind: TableKind, + subscription_id: u64, + subscription: EntitySubscription, + ) { + let (subscription_count, group_count) = { + let mut subscriptions = self.entity_subscription_registry(kind).write().await; + subscriptions.insert(subscription_id, subscription); + ( + subscriptions.subscription_count(), + subscriptions.group_count(), + ) + }; + record_active_entity_subscriptions(kind, subscription_count); + record_active_entity_subscription_groups(kind, group_count); + } + + async fn update_entity_subscription( + &self, + kind: TableKind, + subscription_id: u64, + clause: Option, + world_addresses: HashSet>, + ) { + let group_count = { + let mut subscriptions = self.entity_subscription_registry(kind).write().await; + if !subscriptions.update(subscription_id, clause, world_addresses) { + return; + } + subscriptions.group_count() + }; + record_active_entity_subscription_groups(kind, group_count); + } + + async fn remove_entity_subscription(&self, kind: TableKind, subscription_id: u64) -> bool { + let (subscription_count, group_count) = { + let mut subscriptions = self.entity_subscription_registry(kind).write().await; + if !subscriptions.remove(subscription_id) { + return false; + } + ( + subscriptions.subscription_count(), + subscriptions.group_count(), + ) + }; + record_active_entity_subscriptions(kind, subscription_count); + record_active_entity_subscription_groups(kind, group_count); + true + } + + async fn remove_subscription(&self, kind: SubscriptionKind, subscription_id: u64) -> bool { + match kind { + SubscriptionKind::Entity(kind) => { + self.remove_entity_subscription(kind, subscription_id).await + } + SubscriptionKind::Event => self + .event_subscriptions + .lock() + .await + .remove(&subscription_id) + .is_some(), + SubscriptionKind::Contract => self + .contract_subscriptions + .lock() + .await + .remove(&subscription_id) + .is_some(), + SubscriptionKind::Token => self + .token_subscriptions + .lock() + .await + .remove(&subscription_id) + .is_some(), + SubscriptionKind::TokenBalance => self + .token_balance_subscriptions + .lock() + .await + .remove(&subscription_id) + .is_some(), + SubscriptionKind::TokenTransfer => self + .token_transfer_subscriptions + .lock() + .await + .remove(&subscription_id) + .is_some(), + SubscriptionKind::Transaction => self + .transaction_subscriptions + .lock() + .await + .remove(&subscription_id) + .is_some(), + } + } +} + +impl ManagedSubscriptionStream { + fn new( + state: Arc, + kind: SubscriptionKind, + subscription_id: u64, + receiver: mpsc::Receiver>, + ) -> Self { + Self { + stream: ReceiverStream::new(receiver), + cleanup: Some(SubscriptionCleanup { + state, + kind, + subscription_id, + }), + } + } +} + +impl Stream for ManagedSubscriptionStream { + type Item = Result; + + fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + let this = self.get_mut(); + Pin::new(&mut this.stream).poll_next(cx) + } +} + +impl Drop for ManagedSubscriptionStream { + fn drop(&mut self) { + if let Some(cleanup) = self.cleanup.take() { + cleanup.schedule(); + } + } +} + +impl SubscriptionCleanup { + fn schedule(self) { + let state = self.state; + let kind = self.kind; + let subscription_id = self.subscription_id; + + if let Ok(handle) = tokio::runtime::Handle::try_current() { + handle.spawn(async move { + let _ = state.remove_subscription(kind, subscription_id).await; + }); + return; + } + + match kind { + SubscriptionKind::Entity(kind) => { + let mut subscriptions = match state.entity_subscription_registry(kind).try_write() { + Ok(subscriptions) => subscriptions, + Err(_) => return, + }; + if subscriptions.remove(subscription_id) { + record_active_entity_subscriptions(kind, subscriptions.subscription_count()); + record_active_entity_subscription_groups(kind, subscriptions.group_count()); + } + } + SubscriptionKind::Event => { + if let Ok(mut subscriptions) = state.event_subscriptions.try_lock() { + subscriptions.remove(&subscription_id); + } + } + SubscriptionKind::Contract => { + if let Ok(mut subscriptions) = state.contract_subscriptions.try_lock() { + subscriptions.remove(&subscription_id); + } + } + SubscriptionKind::Token => { + if let Ok(mut subscriptions) = state.token_subscriptions.try_lock() { + subscriptions.remove(&subscription_id); + } + } + SubscriptionKind::TokenBalance => { + if let Ok(mut subscriptions) = state.token_balance_subscriptions.try_lock() { + subscriptions.remove(&subscription_id); + } + } + SubscriptionKind::TokenTransfer => { + if let Ok(mut subscriptions) = state.token_transfer_subscriptions.try_lock() { + subscriptions.remove(&subscription_id); + } + } + SubscriptionKind::Transaction => { + if let Ok(mut subscriptions) = state.transaction_subscriptions.try_lock() { + subscriptions.remove(&subscription_id); + } + } + } + } +} + #[derive(Clone)] struct ManagedTable { world_address: Felt, @@ -211,18 +652,47 @@ impl EcsService { DbBackend::Sqlite => sqlite_url(database_url)?, }; - let pool = AnyPoolOptions::new() - .max_connections(max_connections.unwrap_or(if backend == DbBackend::Sqlite { - 1 - } else { - 5 - })) - .connect(&database_url) - .await?; - let has_erc20 = erc20_url.is_some(); let has_erc721 = erc721_url.is_some(); let has_erc1155 = erc1155_url.is_some(); + let erc20_url = erc20_url.map(std::string::ToString::to_string); + let erc721_url = erc721_url.map(std::string::ToString::to_string); + let erc1155_url = erc1155_url.map(std::string::ToString::to_string); + + let pool_options = AnyPoolOptions::new().max_connections(max_connections.unwrap_or( + if backend == DbBackend::Sqlite { + DEFAULT_SQLITE_MAX_CONNECTIONS + } else { + 5 + }, + )); + let pool = match backend { + DbBackend::Sqlite => { + pool_options + .after_connect({ + let erc20_url = erc20_url.clone(); + let erc721_url = erc721_url.clone(); + let erc1155_url = erc1155_url.clone(); + move |conn, _meta| { + let erc20_url = erc20_url.clone(); + let erc721_url = erc721_url.clone(); + let erc1155_url = erc1155_url.clone(); + Box::pin(async move { + attach_sqlite_databases( + conn, + erc20_url.as_deref(), + erc721_url.as_deref(), + erc1155_url.as_deref(), + ) + .await + }) + } + }) + .connect(&database_url) + .await? + } + DbBackend::Postgres => pool_options.connect(&database_url).await?, + }; let service = Self { state: Arc::new(EcsState { @@ -232,12 +702,13 @@ impl EcsService { has_erc20, has_erc721, has_erc1155, - erc20_url: erc20_url.map(std::string::ToString::to_string), - erc721_url: erc721_url.map(std::string::ToString::to_string), - erc1155_url: erc1155_url.map(std::string::ToString::to_string), + erc20_url, + erc721_url, + erc1155_url, managed_tables: Mutex::new(None), - entity_subscriptions: Mutex::new(HashMap::new()), - event_message_subscriptions: Mutex::new(HashMap::new()), + cached_views_sql: RwLock::new(None), + entity_subscriptions: RwLock::new(EntitySubscriptionRegistry::default()), + event_message_subscriptions: RwLock::new(EntitySubscriptionRegistry::default()), event_subscriptions: Mutex::new(HashMap::new()), contract_subscriptions: Mutex::new(HashMap::new()), token_subscriptions: Mutex::new(HashMap::new()), @@ -379,20 +850,54 @@ impl EcsService { .execute(&self.state.pool) .await?; + sqlx::query("DROP INDEX IF EXISTS torii_ecs_entity_models_key_lookup_idx") + .execute(&self.state.pool) + .await + .ok(); + + sqlx::query( + "CREATE INDEX IF NOT EXISTS torii_ecs_events_cursor_idx + ON torii_ecs_events(event_id)", + ) + .execute(&self.state.pool) + .await?; + sqlx::query( - "CREATE INDEX IF NOT EXISTS torii_ecs_entity_models_key_lookup_idx - ON torii_ecs_entity_models(kind, world_address, entity_id)", + "CREATE INDEX IF NOT EXISTS torii_ecs_entity_meta_page_idx + ON torii_ecs_entity_meta(kind, deleted, entity_id, world_address, table_id, created_at, updated_at, executed_at)", ) .execute(&self.state.pool) .await?; sqlx::query( - "CREATE INDEX IF NOT EXISTS torii_ecs_events_cursor_idx - ON torii_ecs_events(event_id)", + "CREATE INDEX IF NOT EXISTS torii_ecs_entity_meta_kind_table_page_idx + ON torii_ecs_entity_meta(kind, deleted, table_id, entity_id, world_address)", ) .execute(&self.state.pool) .await?; + let emh_sql = self.event_messages_historical_view_sql().await?; + match self.state.backend { + DbBackend::Sqlite => { + // sqlite-dynamic-ok: persistent view DDL for event_messages_historical. + sqlx::query(&format!( + "CREATE VIEW IF NOT EXISTS event_messages_historical AS {emh_sql}" + )) + .execute(&self.state.pool) + .await + .ok(); + } + DbBackend::Postgres => { + // sqlite-dynamic-ok: persistent view DDL for event_messages_historical. + sqlx::query(&format!( + "CREATE OR REPLACE VIEW event_messages_historical AS {emh_sql}" + )) + .execute(&self.state.pool) + .await + .ok(); + } + } + Ok(()) } @@ -416,11 +921,9 @@ impl EcsService { "Attaching ERC database" ); attach_sqlite_database(&mut conn, schema, url).await?; - match sqlx::query(&format!( - "SELECT name FROM {schema}.sqlite_master WHERE type='table' LIMIT 5" - )) - .fetch_all(&mut *conn) - .await + match sqlx::query(sqlite_master_preview_sql(schema)) + .fetch_all(&mut *conn) + .await { Ok(rows) => { let tables: Vec = @@ -441,16 +944,21 @@ impl EcsService { } async fn acquire_initialized_connection(&self) -> Result> { - let mut conn = self.state.pool.acquire().await?; - if self.state.backend == DbBackend::Sqlite { - attach_sqlite_databases( - &mut conn, - self.state.erc20_url.as_deref(), - self.state.erc721_url.as_deref(), - self.state.erc1155_url.as_deref(), - ) - .await?; - } + Ok(self.state.pool.acquire().await?) + } + + async fn acquire_query_connection( + &self, + operation: &'static str, + ) -> Result> { + let start = Instant::now(); + let conn = self.state.pool.acquire().await?; + ::metrics::histogram!( + "torii_ecs_db_pool_acquire_seconds", + "operation" => operation, + "backend" => self.state.backend.as_str() + ) + .record(start.elapsed().as_secs_f64()); Ok(conn) } @@ -777,13 +1285,14 @@ impl EcsService { entity_id: Felt, ) -> Result<()> { let start = Instant::now(); - let subscriptions = match kind { - TableKind::Entity => &self.state.entity_subscriptions, - TableKind::EventMessage => &self.state.event_message_subscriptions, + let subscriptions = self.state.entity_subscription_registry(kind); + let group_snapshots = { + let subscriptions = subscriptions.read().await; + if subscriptions.is_empty() { + return Ok(()); + } + subscriptions.snapshot_groups() }; - if subscriptions.lock().await.is_empty() { - return Ok(()); - } let entity = self .load_entity_by_id(kind, world_address, entity_id) @@ -793,53 +1302,62 @@ impl EcsService { return Ok(()); }; - let (checked, targets) = { - let subscriptions = subscriptions.lock().await; - let checked = subscriptions.len(); - let targets = subscriptions - .iter() - .filter_map(|(&subscription_id, subscription)| { - if !subscription.world_addresses.is_empty() - && !subscription.world_addresses.contains(&entity.world_address) - { + let checked_groups = group_snapshots.len(); + let checked_subscribers = group_snapshots + .iter() + .map(|group| group.senders.len()) + .sum::(); + let targets = group_snapshots + .into_iter() + .filter_map(|group| { + if !group.world_addresses.is_empty() + && !group.world_addresses.contains(&entity.world_address) + { + return None; + } + if let Some(clause) = &group.clause { + if !entity_matches_clause(&entity, clause) { return None; } - if let Some(clause) = &subscription.clause { - if !entity_matches_clause(&entity, clause) { - return None; - } + } + Some(group.senders) + }) + .collect::>(); + let matched_groups = targets.len(); + let matched_subscribers = targets.iter().map(Vec::len).sum::(); + + let mut closed = Vec::new(); + for group_targets in targets { + for (subscription_id, sender) in group_targets { + if let Err(err) = sender.try_send(Ok(SubscribeEntityResponse { + entity: Some(entity.clone()), + subscription_id, + })) { + if matches!(err, TrySendError::Closed(_)) { + closed.push(subscription_id); } - Some((subscription_id, subscription.sender.clone())) - }) - .collect::>(); - (checked, targets) - }; - let matched = targets.len(); - - let mut closed = Vec::new(); - for (subscription_id, sender) in targets { - if let Err(err) = sender.try_send(Ok(SubscribeEntityResponse { - entity: Some(entity.clone()), - subscription_id, - })) { - if matches!(err, TrySendError::Closed(_)) { - closed.push(subscription_id); } } } if !closed.is_empty() { - let mut subscriptions = subscriptions.lock().await; for subscription_id in closed { - subscriptions.remove(&subscription_id); + let _ = self + .state + .remove_entity_subscription(kind, subscription_id) + .await; } } ::metrics::histogram!("torii_ecs_publish_entity_update_seconds") .record(start.elapsed().as_secs_f64()); + ::metrics::counter!("torii_ecs_publish_entity_subscription_groups_checked_total") + .increment(checked_groups as u64); + ::metrics::counter!("torii_ecs_publish_entity_subscription_groups_matched_total") + .increment(matched_groups as u64); ::metrics::counter!("torii_ecs_publish_entity_subscribers_checked_total") - .increment(checked as u64); + .increment(checked_subscribers as u64); ::metrics::counter!("torii_ecs_publish_entity_subscribers_matched_total") - .increment(matched as u64); + .increment(matched_subscribers as u64); Ok(()) } @@ -1008,7 +1526,7 @@ impl EcsService { direction: PaginationDirection::Forward as i32, order_by: Vec::new(), }); - let limit = Self::pagination_limit(Some(&pagination), 100); + let limit = Self::pagination_limit_with_max(Some(&pagination), 100, None); let target_limit = limit.saturating_add(1); let direction_is_backward = pagination.direction == PaginationDirection::Backward as i32; let order_sql = if direction_is_backward { "DESC" } else { "ASC" }; @@ -1177,14 +1695,17 @@ impl EcsService { ) -> Result<()> { match self.state.backend { DbBackend::Sqlite => { + // sqlite-dynamic-ok: view DDL requires the target view identifier in SQL text. sqlx::query(&format!("DROP VIEW IF EXISTS {view_name}")) .execute(&mut **conn) .await?; + // sqlite-dynamic-ok: view DDL requires the target view identifier and SELECT body in SQL text. sqlx::query(&format!("CREATE TEMP VIEW {view_name} AS {select_sql}")) .execute(&mut **conn) .await?; } DbBackend::Postgres => { + // sqlite-dynamic-ok: compatibility view DDL is an intentional dynamic exception. sqlx::query(&format!( "CREATE OR REPLACE VIEW {view_name} AS {select_sql}" )) @@ -1195,28 +1716,81 @@ impl EcsService { Ok(()) } + async fn compat_view_exists_on_connection( + &self, + conn: &mut PoolConnection, + name: &str, + ) -> Result { + if self + .sql_object_exists_on_connection(conn, None, name) + .await? + { + return Ok(true); + } + if self.state.backend == DbBackend::Sqlite { + return self + .sql_object_exists_on_connection(conn, Some("temp"), name) + .await; + } + Ok(false) + } + async fn ensure_sql_compat_views_on_connection( &self, conn: &mut PoolConnection, ) -> Result<()> { + { + let cached = self.state.cached_views_sql.read().await; + if let Some(views) = cached.as_ref() { + if !self + .compat_view_exists_on_connection(conn, "controllers") + .await? + { + self.replace_sql_compat_view_on_connection( + conn, + "controllers", + &views.controllers, + ) + .await?; + } + if !self + .compat_view_exists_on_connection(conn, "token_balances") + .await? + { + self.replace_sql_compat_view_on_connection( + conn, + "token_balances", + &views.token_balances, + ) + .await?; + } + return Ok(()); + } + } + + let controllers_sql = self.controllers_view_sql(conn).await?; + let token_balances_sql = self.token_balances_view_sql(conn).await?; + if !self - .sql_object_exists_on_connection(conn, None, "controllers") + .compat_view_exists_on_connection(conn, "controllers") .await? { - let controllers_sql = self.controllers_view_sql(conn).await?; self.replace_sql_compat_view_on_connection(conn, "controllers", &controllers_sql) .await?; } - let event_messages_historical_sql = self.event_messages_historical_view_sql().await?; - self.replace_sql_compat_view_on_connection( - conn, - "event_messages_historical", - &event_messages_historical_sql, - ) - .await?; - let token_balances_sql = self.token_balances_view_sql(conn).await?; - self.replace_sql_compat_view_on_connection(conn, "token_balances", &token_balances_sql) - .await?; + if !self + .compat_view_exists_on_connection(conn, "token_balances") + .await? + { + self.replace_sql_compat_view_on_connection(conn, "token_balances", &token_balances_sql) + .await?; + } + + let mut cached = self.state.cached_views_sql.write().await; + *cached = Some(CachedViewsSql { + controllers: controllers_sql, + token_balances: token_balances_sql, + }); Ok(()) } @@ -1372,10 +1946,19 @@ impl EcsService { } fn pagination_limit(pagination: Option<&types::Pagination>, default: usize) -> usize { + Self::pagination_limit_with_max(pagination, default, Some(1000)) + } + + fn pagination_limit_with_max( + pagination: Option<&types::Pagination>, + default: usize, + max: Option, + ) -> usize { pagination .map(|pagination| pagination.limit) .filter(|limit| *limit > 0) - .map_or(default, |limit| limit.min(1000) as usize) + .map_or(default, |limit| limit as usize) + .min(max.unwrap_or(usize::MAX)) } async fn load_token_contracts( @@ -1664,32 +2247,64 @@ impl EcsService { query: &types::TokenBalanceQuery, ) -> Result<(Vec, String)> { let limit = Self::pagination_limit(query.pagination.as_ref(), 100); - let mut items = Vec::new(); + let mut items = Vec::with_capacity(limit.saturating_mul(3)); + let acquire_start = Instant::now(); let mut conn = self.acquire_initialized_connection().await?; + ::metrics::histogram!( + "torii_ecs_retrieve_token_balances_stage_seconds", + "stage" => "acquire", + "backend" => self.state.backend.as_str() + ) + .record(acquire_start.elapsed().as_secs_f64()); + tracing::debug!( + target: "torii::ecs_sink", + limit, + account_filters = query.account_addresses.len(), + contract_filters = query.contract_addresses.len(), + token_id_filters = query.token_ids.len(), + acquire_ms = acquire_start.elapsed().as_millis(), + "RetrieveTokenBalances acquired initialized connection" + ); + + let normalized_token_ids = normalize_token_filter_values(&query.token_ids); if self.state.has_erc20 { let table = self.scoped_table("erc20", "balances"); let mut builder = QueryBuilder::::new(format!("SELECT wallet, token, balance FROM {table}")); let mut has_where = false; - has_where |= push_blob_in_filter(&mut builder, "wallet", &query.account_addresses); - if !query.contract_addresses.is_empty() { - if has_where { - builder.push(" AND "); - } else { - builder.push(" WHERE "); - } - builder.push("hex(token) IN ("); - { - let mut separated = builder.separated(", "); - for contract in &query.contract_addresses { - separated.push_bind(hex::encode_upper(contract)); - } - } - builder.push(")"); - } - match builder.build().fetch_all(&mut *conn).await { + has_where = push_blob_in_filter_with_mode( + &mut builder, + "wallet", + &query.account_addresses, + has_where, + ); + push_blob_in_filter_with_mode( + &mut builder, + "token", + &query.contract_addresses, + has_where, + ); + builder.push(" ORDER BY token ASC, wallet ASC"); + builder.push(" LIMIT "); + builder.push_bind(limit as i64); + + let query_start = Instant::now(); + let result = builder.build().fetch_all(&mut *conn).await; + ::metrics::histogram!( + "torii_ecs_retrieve_token_balances_stage_seconds", + "stage" => "erc20_query", + "backend" => self.state.backend.as_str() + ) + .record(query_start.elapsed().as_secs_f64()); + match result { Ok(rows) => { + tracing::debug!( + target: "torii::ecs_sink", + rows = rows.len(), + elapsed_ms = query_start.elapsed().as_millis(), + "RetrieveTokenBalances queried ERC20 balances" + ); for row in rows { let balance = canonical_u256_bytes_from_db(&row.try_get::, _>("balance")?)?; @@ -1707,6 +2322,7 @@ impl EcsService { } Err(e) => { tracing::warn!( + target: "torii::ecs_sink", error = %e, "Failed to query erc20.balances — returning empty ERC20 balances" ); @@ -1719,40 +2335,46 @@ impl EcsService { let mut builder = QueryBuilder::::new(format!("SELECT owner, token, token_id FROM {table}")); let mut has_where = false; - has_where |= push_blob_in_filter(&mut builder, "owner", &query.account_addresses); - if !query.contract_addresses.is_empty() { - if has_where { - builder.push(" AND "); - } else { - builder.push(" WHERE "); - has_where = true; - } - builder.push("hex(token) IN ("); - { - let mut separated = builder.separated(", "); - for contract in &query.contract_addresses { - separated.push_bind(hex::encode_upper(contract)); - } - } - builder.push(")"); - } - if !query.token_ids.is_empty() { - if has_where { - builder.push(" AND "); - } else { - builder.push(" WHERE "); - } - builder.push("hex(token_id) IN ("); - { - let mut separated = builder.separated(", "); - for token_id in &query.token_ids { - separated.push_bind(hex::encode_upper(token_id)); - } - } - builder.push(")"); - } - match builder.build().fetch_all(&mut *conn).await { + has_where = push_blob_in_filter_with_mode( + &mut builder, + "owner", + &query.account_addresses, + has_where, + ); + has_where = push_blob_in_filter_with_mode( + &mut builder, + "token", + &query.contract_addresses, + has_where, + ); + push_blob_in_filter_with_mode( + &mut builder, + "token_id", + &normalized_token_ids, + has_where, + ); + builder.push(" ORDER BY token ASC, owner ASC, "); + builder.push(sql_blob_numeric_order_expr(self.state.backend, "token_id")); + builder.push(" ASC, token_id ASC"); + builder.push(" LIMIT "); + builder.push_bind(limit as i64); + + let query_start = Instant::now(); + let result = builder.build().fetch_all(&mut *conn).await; + ::metrics::histogram!( + "torii_ecs_retrieve_token_balances_stage_seconds", + "stage" => "erc721_query", + "backend" => self.state.backend.as_str() + ) + .record(query_start.elapsed().as_secs_f64()); + match result { Ok(rows) => { + tracing::debug!( + target: "torii::ecs_sink", + rows = rows.len(), + elapsed_ms = query_start.elapsed().as_millis(), + "RetrieveTokenBalances queried ERC721 ownership" + ); for row in rows { let account_address = canonical_felt_bytes_from_db(&row.try_get::, _>("owner")?)?; @@ -1770,6 +2392,7 @@ impl EcsService { } Err(e) => { tracing::warn!( + target: "torii::ecs_sink", error = %e, "Failed to query erc721.nft_ownership — returning empty ERC721 balances" ); @@ -1783,40 +2406,46 @@ impl EcsService { "SELECT wallet, contract, token_id, balance FROM {table}" )); let mut has_where = false; - has_where |= push_blob_in_filter(&mut builder, "wallet", &query.account_addresses); - if !query.contract_addresses.is_empty() { - if has_where { - builder.push(" AND "); - } else { - builder.push(" WHERE "); - has_where = true; - } - builder.push("hex(contract) IN ("); - { - let mut separated = builder.separated(", "); - for contract in &query.contract_addresses { - separated.push_bind(hex::encode_upper(contract)); - } - } - builder.push(")"); - } - if !query.token_ids.is_empty() { - if has_where { - builder.push(" AND "); - } else { - builder.push(" WHERE "); - } - builder.push("hex(token_id) IN ("); - { - let mut separated = builder.separated(", "); - for token_id in &query.token_ids { - separated.push_bind(hex::encode_upper(token_id)); - } - } - builder.push(")"); - } - match builder.build().fetch_all(&mut *conn).await { + has_where = push_blob_in_filter_with_mode( + &mut builder, + "wallet", + &query.account_addresses, + has_where, + ); + has_where = push_blob_in_filter_with_mode( + &mut builder, + "contract", + &query.contract_addresses, + has_where, + ); + push_blob_in_filter_with_mode( + &mut builder, + "token_id", + &normalized_token_ids, + has_where, + ); + builder.push(" ORDER BY contract ASC, wallet ASC, "); + builder.push(sql_blob_numeric_order_expr(self.state.backend, "token_id")); + builder.push(" ASC, token_id ASC"); + builder.push(" LIMIT "); + builder.push_bind(limit as i64); + + let query_start = Instant::now(); + let result = builder.build().fetch_all(&mut *conn).await; + ::metrics::histogram!( + "torii_ecs_retrieve_token_balances_stage_seconds", + "stage" => "erc1155_query", + "backend" => self.state.backend.as_str() + ) + .record(query_start.elapsed().as_secs_f64()); + match result { Ok(rows) => { + tracing::debug!( + target: "torii::ecs_sink", + rows = rows.len(), + elapsed_ms = query_start.elapsed().as_millis(), + "RetrieveTokenBalances queried ERC1155 balances" + ); for row in rows { let balance = canonical_u256_bytes_from_db(&row.try_get::, _>("balance")?)?; @@ -1836,6 +2465,7 @@ impl EcsService { } Err(e) => { tracing::warn!( + target: "torii::ecs_sink", error = %e, "Failed to query erc1155.erc1155_balances — returning empty ERC1155 balances" ); @@ -1850,6 +2480,11 @@ impl EcsService { .then_with(|| a.token_id.cmp(&b.token_id)) }); items.truncate(limit); + tracing::debug!( + target: "torii::ecs_sink", + balances = items.len(), + "RetrieveTokenBalances merged and truncated results" + ); Ok((items, String::new())) } @@ -2222,7 +2857,7 @@ impl EcsService { &self, contract_addresses: &[Vec], token_ids: &[Vec], - ) -> Result> { + ) -> Result { let (tokens, _) = self .load_tokens(&types::TokenQuery { contract_addresses: contract_addresses.to_vec(), @@ -2236,18 +2871,20 @@ impl EcsService { }), }) .await?; - Ok(tokens - .into_iter() - .map(|token| token_subscription_key(&token)) - .collect()) + Ok(SeenCache::from_items( + tokens + .into_iter() + .map(|token| token_subscription_key(&token)), + SUBSCRIPTION_SEEN_CACHE_CAPACITY, + )) } - async fn snapshot_token_balance_seen( + async fn snapshot_token_balance_state( &self, account_addresses: &[Vec], contract_addresses: &[Vec], token_ids: &[Vec], - ) -> Result> { + ) -> Result>> { let (balances, _) = self .load_token_balances(&types::TokenBalanceQuery { account_addresses: account_addresses.to_vec(), @@ -2261,10 +2898,7 @@ impl EcsService { }), }) .await?; - Ok(balances - .into_iter() - .map(|balance| token_balance_subscription_key(&balance)) - .collect()) + Ok(token_balance_subscription_state(&balances)) } async fn snapshot_token_transfer_seen( @@ -2272,7 +2906,7 @@ impl EcsService { account_addresses: &[Vec], contract_addresses: &[Vec], token_ids: &[Vec], - ) -> Result> { + ) -> Result { let (transfers, _) = self .load_token_transfers(&types::TokenTransferQuery { account_addresses: account_addresses.to_vec(), @@ -2286,13 +2920,16 @@ impl EcsService { }), }) .await?; - Ok(transfers.into_iter().map(|transfer| transfer.id).collect()) + Ok(SeenCache::from_items( + transfers.into_iter().map(|transfer| transfer.id), + SUBSCRIPTION_SEEN_CACHE_CAPACITY, + )) } async fn snapshot_transaction_seen( &self, filter: &types::TransactionFilter, - ) -> Result> { + ) -> Result { let (transactions, _) = self .load_transactions(&types::TransactionQuery { filter: Some(filter.clone()), @@ -2304,10 +2941,12 @@ impl EcsService { }), }) .await?; - Ok(transactions - .into_iter() - .map(|transaction| hex::encode(transaction.transaction_hash)) - .collect()) + Ok(SeenCache::from_items( + transactions + .into_iter() + .map(|transaction| hex::encode(transaction.transaction_hash)), + SUBSCRIPTION_SEEN_CACHE_CAPACITY, + )) } fn spawn_token_poll(&self, subscription_id: u64) { @@ -2323,7 +2962,7 @@ impl EcsService { ( subscription.contract_addresses.clone(), subscription.token_ids.clone(), - subscription.seen.clone(), + subscription.seen.snapshot(), subscription.sender.clone(), ) }; @@ -2388,7 +3027,7 @@ impl EcsService { tokio::spawn(async move { loop { sleep(Duration::from_secs(1)).await; - let (account_addresses, contract_addresses, token_ids, seen, sender) = { + let (account_addresses, contract_addresses, token_ids, latest, sender) = { let subscriptions = service.state.token_balance_subscriptions.lock().await; let Some(subscription) = subscriptions.get(&subscription_id) else { break; @@ -2397,7 +3036,7 @@ impl EcsService { subscription.account_addresses.clone(), subscription.contract_addresses.clone(), subscription.token_ids.clone(), - subscription.seen.clone(), + subscription.latest.clone(), subscription.sender.clone(), ) }; @@ -2418,10 +3057,11 @@ impl EcsService { Ok((balances, _)) => balances, Err(_) => continue, }; - let mut sent = Vec::new(); + let current = token_balance_subscription_state(&balances); for balance in balances { - let key = token_balance_subscription_key(&balance); - if seen.contains(&key) { + let identity = token_balance_identity_key(&balance); + let fingerprint = token_balance_state_fingerprint(&balance); + if latest.get(&identity) == Some(&fingerprint) { continue; } if sender @@ -2440,18 +3080,15 @@ impl EcsService { .remove(&subscription_id); return; } - sent.push(key); } - if !sent.is_empty() { - if let Some(subscription) = service - .state - .token_balance_subscriptions - .lock() - .await - .get_mut(&subscription_id) - { - subscription.seen.extend(sent); - } + if let Some(subscription) = service + .state + .token_balance_subscriptions + .lock() + .await + .get_mut(&subscription_id) + { + subscription.latest = current; } } }); @@ -2471,7 +3108,7 @@ impl EcsService { subscription.account_addresses.clone(), subscription.contract_addresses.clone(), subscription.token_ids.clone(), - subscription.seen.clone(), + subscription.seen.snapshot(), subscription.sender.clone(), ) }; @@ -2542,7 +3179,7 @@ impl EcsService { }; ( subscription.filter.clone(), - subscription.seen.clone(), + subscription.seen.snapshot(), subscription.sender.clone(), ) }; @@ -2772,41 +3409,33 @@ impl EcsService { .iter() .map(String::as_str) .collect::>(); - let models = self - .load_managed_tables(Some(kind)) - .await? - .into_iter() - .filter(|table| { - (world_filter_set.is_empty() || world_filter_set.contains(&table.world_address)) - && (model_filter_set.is_empty() - || model_filter_set.contains(table.table.name.as_str())) - }) - .collect::>(); - if models.is_empty() { - return Ok((Vec::new(), String::new())); + let mut model_map = HashMap::new(); + let mut all_model_map = HashMap::new(); + for table in self.load_managed_tables(Some(kind)).await? { + if !world_filter_set.is_empty() && !world_filter_set.contains(&table.world_address) { + continue; + } + + let table_id = felt_hex(table.table.id); + if model_filter_set.is_empty() || model_filter_set.contains(table.table.name.as_str()) { + model_map.insert(table_id.clone(), table.clone()); + } + all_model_map.insert(table_id, table); } - let model_map = models - .into_iter() - .map(|table| (felt_hex(table.table.id), table)) - .collect::>(); + if model_map.is_empty() { + return Ok((Vec::new(), String::new())); + } let table_ids = model_map.keys().cloned().collect::>(); let table_id_filter = if query.models.is_empty() { None } else { Some(table_ids.as_slice()) }; - let all_model_map = if table_id_filter.is_some() { - self.load_managed_tables(Some(kind)) - .await? - .into_iter() - .filter(|t| { - world_filter_set.is_empty() || world_filter_set.contains(&t.world_address) - }) - .map(|t| (felt_hex(t.table.id), t)) - .collect::>() + let models_for_entity = if table_id_filter.is_some() { + &all_model_map } else { - model_map.clone() + &model_map }; let member_pushdown = member_pushdown_from_clause(query.clause.as_ref(), &model_map); let target_limit = query.pagination.as_ref().map_or(100, |pagination| { @@ -2832,10 +3461,13 @@ impl EcsService { let mut next_cursor = String::new(); let mut candidate_cursor = cursor; + let mut conn = self.acquire_query_connection("load_entity_page").await?; + loop { let keys_start = Instant::now(); let mut candidate_keys = self .load_entity_page_keys( + &mut conn, kind, &world_filters, table_id_filter, @@ -2865,38 +3497,42 @@ impl EcsService { .map(|(_, entity_id)| entity_id.trim_start_matches("0x").to_string()) .unwrap_or_default(); - let meta_start = Instant::now(); - let candidate_meta = self - .load_entity_meta_for_keys(kind, None, candidate_keys.as_slice()) - .await?; - ::metrics::histogram!("torii_ecs_load_entity_page_meta_rows_seconds") - .record(meta_start.elapsed().as_secs_f64()); - for chunk in candidate_keys.chunks(256) { let rows_start = Instant::now(); let rows = self - .load_entity_page_rows(kind, &world_filters, None, chunk) + .load_entity_page_rows(&mut conn, kind, &world_filters, None, chunk) .await?; ::metrics::histogram!("torii_ecs_load_entity_page_rows_seconds") .record(rows_start.elapsed().as_secs_f64()); ::metrics::counter!("torii_ecs_load_entity_page_rows_total") .increment(rows.len() as u64); - let row_map = rows.into_iter().fold( - HashMap::<(String, String), Vec<(String, String)>>::new(), - |mut acc, row| { - acc.entry((row.world_address, row.entity_id)) - .or_default() - .push((row.table_id, row.row_json)); - acc - }, - ); + + let mut row_map = HashMap::<(String, String), Vec<(String, String)>>::new(); + let mut meta_map = + HashMap::<(String, String), HashMap>::new(); + for row in rows { + let key = (row.world_address.clone(), row.entity_id.clone()); + meta_map.entry(key.clone()).or_default().insert( + row.table_id.clone(), + EntityMetaRow { + created_at: row.created_at, + updated_at: row.updated_at, + executed_at: row.executed_at, + deleted: false, + }, + ); + row_map + .entry(key) + .or_default() + .push((row.table_id, row.row_json)); + } for key in chunk { - let Some(meta) = candidate_meta.get(key) else { + let Some(meta) = meta_map.get(key) else { continue; }; let Some(entity) = - build_entity_from_snapshot(key, meta, row_map.get(key), &all_model_map)? + build_entity_from_snapshot(key, meta, row_map.get(key), models_for_entity)? else { continue; }; @@ -3115,6 +3751,7 @@ impl EcsService { async fn load_entity_page_keys( &self, + conn: &mut PoolConnection, kind: TableKind, world_filters: &[Felt], table_ids: Option<&[String]>, @@ -3139,8 +3776,11 @@ impl EcsService { "ASC" }; + let single_table = table_ids.is_some_and(|ids| ids.len() == 1); + let needs_group_by = table_ids.is_none_or(|ids| ids.len() > 1); + let mut builder = QueryBuilder::::new( - "SELECT DISTINCT m.world_address, m.entity_id \ + "SELECT m.world_address, m.entity_id \ FROM torii_ecs_entity_meta m \ WHERE m.kind = ", ); @@ -3182,6 +3822,9 @@ impl EcsService { if let Some(member_pushdown) = member_pushdown { append_member_pushdown_sql(&mut builder, self.state.backend, member_pushdown); } + if needs_group_by && !single_table { + builder.push(" GROUP BY m.world_address, m.entity_id"); + } builder.push(" ORDER BY m.entity_id "); builder.push(order); builder.push(", m.world_address "); @@ -3189,7 +3832,13 @@ impl EcsService { builder.push(" LIMIT "); builder.push_bind(limit as i64); - let rows = builder.build().fetch_all(&self.state.pool).await?; + let start = Instant::now(); + let rows = builder.build().fetch_all(&mut **conn).await?; + ::metrics::histogram!( + "torii_ecs_load_entity_page_keys_query_seconds", + "backend" => self.state.backend.as_str() + ) + .record(start.elapsed().as_secs_f64()); rows.into_iter() .map(|row| { Ok(( @@ -3200,29 +3849,36 @@ impl EcsService { .collect() } - async fn load_entity_meta_for_keys( + async fn load_entity_page_rows( &self, + conn: &mut PoolConnection, kind: TableKind, + _world_filters: &[Felt], table_ids: Option<&[String]>, entity_keys: &[(String, String)], - ) -> Result>> { - if table_ids.is_some_and(<[String]>::is_empty) || entity_keys.is_empty() { - return Ok(HashMap::new()); + ) -> Result> { + if entity_keys.is_empty() || table_ids.is_some_and(<[String]>::is_empty) { + return Ok(Vec::new()); } let deleted_sql = match self.state.backend { - DbBackend::Sqlite => "CAST(m.deleted AS INTEGER)", - DbBackend::Postgres => "CASE WHEN m.deleted THEN 1 ELSE 0 END", + DbBackend::Sqlite => "m.deleted = 0", + DbBackend::Postgres => "m.deleted = FALSE", }; let mut builder = QueryBuilder::::new(format!( - "SELECT m.world_address, m.table_id, m.entity_id, m.created_at, m.updated_at, m.executed_at, \ - {deleted_sql} AS deleted \ - FROM torii_ecs_entity_meta m WHERE m.kind = " + "SELECT em.world_address, em.table_id, em.entity_id, em.row_json, \ + m.created_at, m.updated_at, m.executed_at \ + FROM torii_ecs_entity_models em \ + JOIN torii_ecs_entity_meta m \ + ON m.kind = em.kind AND m.world_address = em.world_address \ + AND m.table_id = em.table_id AND m.entity_id = em.entity_id \ + AND {deleted_sql} \ + WHERE em.kind = " )); builder.push_bind(kind.as_str()); if let Some(table_ids) = table_ids { - builder.push(" AND m.table_id IN ("); + builder.push(" AND em.table_id IN ("); { let mut separated = builder.separated(", "); for table_id in table_ids { @@ -3231,163 +3887,61 @@ impl EcsService { } builder.push(")"); } + append_entity_key_pairs_filter( + &mut builder, + "em.world_address", + "em.entity_id", + entity_keys, + ); - let world_addresses = entity_keys - .iter() - .map(|(world_address, _)| world_address.clone()) - .collect::>(); - let entity_ids = entity_keys - .iter() - .map(|(_, entity_id)| entity_id.clone()) - .collect::>(); - let allowed = entity_keys.iter().cloned().collect::>(); - - builder.push(" AND m.world_address IN ("); - { - let mut separated = builder.separated(", "); - for world_address in &world_addresses { - separated.push_bind(world_address); - } - } - builder.push(")"); - builder.push(" AND m.entity_id IN ("); - { - let mut separated = builder.separated(", "); - for entity_id in &entity_ids { - separated.push_bind(entity_id); - } + let start = Instant::now(); + let rows = builder.build().fetch_all(&mut **conn).await?; + ::metrics::histogram!( + "torii_ecs_load_entity_page_rows_query_seconds", + "backend" => self.state.backend.as_str() + ) + .record(start.elapsed().as_secs_f64()); + let mut output = Vec::new(); + for row in rows { + output.push(EntityModelRow { + world_address: row.try_get("world_address")?, + table_id: row.try_get("table_id")?, + entity_id: row.try_get("entity_id")?, + row_json: row.try_get("row_json")?, + created_at: row.try_get::("created_at")? as u64, + updated_at: row.try_get::("updated_at")? as u64, + executed_at: row.try_get::("executed_at")? as u64, + }); } - builder.push(")"); - match self.state.backend { + Ok(output) + } + + async fn load_entity_snapshot( + &self, + kind: TableKind, + world_address: Felt, + entity_id: Felt, + ) -> Result> { + let sql = match self.state.backend { DbBackend::Sqlite => { - builder.push(" AND m.deleted = 0"); + "SELECT table_id, row_json + FROM torii_ecs_entity_models + WHERE kind = ?1 AND world_address = ?2 AND entity_id = ?3" } DbBackend::Postgres => { - builder.push(" AND m.deleted = FALSE"); + "SELECT table_id, row_json + FROM torii_ecs_entity_models + WHERE kind = $1 AND world_address = $2 AND entity_id = $3" } - } - - let rows = builder.build().fetch_all(&self.state.pool).await?; - let mut meta = HashMap::new(); - for row in rows { - let world_address = row.try_get::("world_address")?; - let entity_id = row.try_get::("entity_id")?; - if !allowed.contains(&(world_address.clone(), entity_id.clone())) { - continue; - } - meta.entry((world_address, entity_id)) - .or_insert_with(HashMap::new) - .insert( - row.try_get::("table_id")?, - EntityMetaRow { - created_at: row.try_get::("created_at")? as u64, - updated_at: row.try_get::("updated_at")? as u64, - executed_at: row.try_get::("executed_at")? as u64, - deleted: row.try_get::("deleted")? != 0, - }, - ); - } - Ok(meta) - } - - async fn load_entity_page_rows( - &self, - kind: TableKind, - _world_filters: &[Felt], - table_ids: Option<&[String]>, - entity_keys: &[(String, String)], - ) -> Result> { - if entity_keys.is_empty() || table_ids.is_some_and(<[String]>::is_empty) { - return Ok(Vec::new()); - } - - let mut builder = QueryBuilder::::new( - "SELECT world_address, table_id, entity_id, row_json \ - FROM torii_ecs_entity_models WHERE kind = ", - ); - builder.push_bind(kind.as_str()); - if let Some(table_ids) = table_ids { - builder.push(" AND table_id IN ("); - { - let mut separated = builder.separated(", "); - for table_id in table_ids { - separated.push_bind(table_id); - } - } - builder.push(")"); - } - - let world_addresses = entity_keys - .iter() - .map(|(world_address, _)| world_address.clone()) - .collect::>(); - let entity_ids = entity_keys - .iter() - .map(|(_, entity_id)| entity_id.clone()) - .collect::>(); - let allowed = entity_keys.iter().cloned().collect::>(); - - builder.push(" AND world_address IN ("); - { - let mut separated = builder.separated(", "); - for world_address in &world_addresses { - separated.push_bind(world_address); - } - } - builder.push(")"); - builder.push(" AND entity_id IN ("); - { - let mut separated = builder.separated(", "); - for entity_id in &entity_ids { - separated.push_bind(entity_id); - } - } - builder.push(")"); - - let rows = builder.build().fetch_all(&self.state.pool).await?; - let mut output = Vec::new(); - for row in rows { - let world_address: String = row.try_get("world_address")?; - let entity_id: String = row.try_get("entity_id")?; - if !allowed.contains(&(world_address.clone(), entity_id.clone())) { - continue; - } - output.push(EntityModelRow { - world_address, - table_id: row.try_get("table_id")?, - entity_id, - row_json: row.try_get("row_json")?, - }); - } - Ok(output) - } - - async fn load_entity_snapshot( - &self, - kind: TableKind, - world_address: Felt, - entity_id: Felt, - ) -> Result> { - let sql = match self.state.backend { - DbBackend::Sqlite => { - "SELECT table_id, row_json - FROM torii_ecs_entity_models - WHERE kind = ?1 AND world_address = ?2 AND entity_id = ?3" - } - DbBackend::Postgres => { - "SELECT table_id, row_json - FROM torii_ecs_entity_models - WHERE kind = $1 AND world_address = $2 AND entity_id = $3" - } - }; - let rows = sqlx::query(sql) - .bind(kind.as_str()) - .bind(felt_hex(world_address)) - .bind(felt_hex(entity_id)) - .fetch_all(&self.state.pool) - .await?; - if rows.is_empty() { - return Ok(None); + }; + let rows = sqlx::query(sql) + .bind(kind.as_str()) + .bind(felt_hex(world_address)) + .bind(felt_hex(entity_id)) + .fetch_all(&self.state.pool) + .await?; + if rows.is_empty() { + return Ok(None); } let meta = self @@ -3413,11 +3967,7 @@ impl EcsService { } let row_json: String = row.try_get("row_json")?; - let value: Value = serde_json::from_str(&row_json)?; - let object = value - .as_object() - .cloned() - .ok_or_else(|| anyhow!("entity snapshot row must be an object"))?; + let object: Map = serde_json::from_str(&row_json)?; aggregate .models .push(row_to_model_struct(&table.table, &object)?); @@ -3641,19 +4191,39 @@ impl EcsService { } } +fn record_active_entity_subscriptions(kind: TableKind, count: usize) { + match kind { + TableKind::Entity => { + ::metrics::gauge!("torii_ecs_entity_subscriptions_active").set(count as f64); + } + TableKind::EventMessage => { + ::metrics::gauge!("torii_ecs_event_message_subscriptions_active").set(count as f64); + } + } +} + +fn record_active_entity_subscription_groups(kind: TableKind, count: usize) { + match kind { + TableKind::Entity => { + ::metrics::gauge!("torii_ecs_entity_subscription_groups_active").set(count as f64); + } + TableKind::EventMessage => { + ::metrics::gauge!("torii_ecs_event_message_subscription_groups_active") + .set(count as f64); + } + } +} + #[tonic::async_trait] impl World for EcsService { - type SubscribeContractsStream = ReceiverStream>; - type SubscribeEntitiesStream = ReceiverStream>; - type SubscribeEventMessagesStream = ReceiverStream>; - type SubscribeTokenBalancesStream = - ReceiverStream>; - type SubscribeTokensStream = ReceiverStream>; - type SubscribeTokenTransfersStream = - ReceiverStream>; - type SubscribeEventsStream = ReceiverStream>; - type SubscribeTransactionsStream = - ReceiverStream>; + type SubscribeContractsStream = ManagedSubscriptionStream; + type SubscribeEntitiesStream = ManagedSubscriptionStream; + type SubscribeEventMessagesStream = ManagedSubscriptionStream; + type SubscribeTokenBalancesStream = ManagedSubscriptionStream; + type SubscribeTokensStream = ManagedSubscriptionStream; + type SubscribeTokenTransfersStream = ManagedSubscriptionStream; + type SubscribeEventsStream = ManagedSubscriptionStream; + type SubscribeTransactionsStream = ManagedSubscriptionStream; async fn subscribe_contracts( &self, @@ -3671,11 +4241,18 @@ impl World for EcsService { })) .await; } - self.state.contract_subscriptions.lock().await.insert( - Self::next_subscription_id().await, - ContractSubscription { query, sender }, - ); - Ok(Response::new(ReceiverStream::new(receiver))) + let subscription_id = Self::next_subscription_id().await; + self.state + .contract_subscriptions + .lock() + .await + .insert(subscription_id, ContractSubscription { query, sender }); + Ok(Response::new(ManagedSubscriptionStream::new( + Arc::clone(&self.state), + SubscriptionKind::Contract, + subscription_id, + receiver, + ))) } async fn worlds( @@ -3701,25 +4278,36 @@ impl World for EcsService { &self, request: Request, ) -> Result, Status> { + let start = Instant::now(); + ::metrics::counter!("torii_ecs_subscribe_entities_requests_total").increment(1); let request = request.into_inner(); let subscription_id = Self::next_subscription_id().await; - let (sender, receiver) = mpsc::channel(256); + let (sender, receiver) = mpsc::channel(2096); sender - .send(Ok(SubscribeEntityResponse { + .try_send(Ok(SubscribeEntityResponse { entity: None, subscription_id, })) - .await .map_err(|_| Status::internal("subscription setup failed"))?; - self.state.entity_subscriptions.lock().await.insert( + self.state + .insert_entity_subscription( + TableKind::Entity, + subscription_id, + EntitySubscription { + clause: request.clause, + world_addresses: request.world_addresses.into_iter().collect(), + sender, + }, + ) + .await; + ::metrics::histogram!("torii_ecs_subscribe_entities_setup_seconds") + .record(start.elapsed().as_secs_f64()); + Ok(Response::new(ManagedSubscriptionStream::new( + Arc::clone(&self.state), + SubscriptionKind::Entity(TableKind::Entity), subscription_id, - EntitySubscription { - clause: request.clause, - world_addresses: request.world_addresses.into_iter().collect(), - sender, - }, - ); - Ok(Response::new(ReceiverStream::new(receiver))) + receiver, + ))) } async fn update_entities_subscription( @@ -3727,16 +4315,12 @@ impl World for EcsService { request: Request, ) -> Result, Status> { let request = request.into_inner(); - if let Some(subscription) = self - .state - .entity_subscriptions - .lock() - .await - .get_mut(&request.subscription_id) - { - subscription.clause = request.clause; - subscription.world_addresses = request.world_addresses.into_iter().collect(); - } + let subscription_id = request.subscription_id; + let clause = request.clause; + let world_addresses = request.world_addresses.into_iter().collect(); + self.state + .update_entity_subscription(TableKind::Entity, subscription_id, clause, world_addresses) + .await; Ok(Response::new(())) } @@ -3765,25 +4349,36 @@ impl World for EcsService { &self, request: Request, ) -> Result, Status> { + let start = Instant::now(); + ::metrics::counter!("torii_ecs_subscribe_event_messages_requests_total").increment(1); let request = request.into_inner(); let subscription_id = Self::next_subscription_id().await; let (sender, receiver) = mpsc::channel(256); sender - .send(Ok(SubscribeEntityResponse { + .try_send(Ok(SubscribeEntityResponse { entity: None, subscription_id, })) - .await .map_err(|_| Status::internal("subscription setup failed"))?; - self.state.event_message_subscriptions.lock().await.insert( + self.state + .insert_entity_subscription( + TableKind::EventMessage, + subscription_id, + EntitySubscription { + clause: request.clause, + world_addresses: request.world_addresses.into_iter().collect(), + sender, + }, + ) + .await; + ::metrics::histogram!("torii_ecs_subscribe_event_messages_setup_seconds") + .record(start.elapsed().as_secs_f64()); + Ok(Response::new(ManagedSubscriptionStream::new( + Arc::clone(&self.state), + SubscriptionKind::Entity(TableKind::EventMessage), subscription_id, - EntitySubscription { - clause: request.clause, - world_addresses: request.world_addresses.into_iter().collect(), - sender, - }, - ); - Ok(Response::new(ReceiverStream::new(receiver))) + receiver, + ))) } async fn update_event_messages_subscription( @@ -3791,16 +4386,17 @@ impl World for EcsService { request: Request, ) -> Result, Status> { let request = request.into_inner(); - if let Some(subscription) = self - .state - .event_message_subscriptions - .lock() - .await - .get_mut(&request.subscription_id) - { - subscription.clause = request.clause; - subscription.world_addresses = request.world_addresses.into_iter().collect(); - } + let subscription_id = request.subscription_id; + let clause = request.clause; + let world_addresses = request.world_addresses.into_iter().collect(); + self.state + .update_entity_subscription( + TableKind::EventMessage, + subscription_id, + clause, + world_addresses, + ) + .await; Ok(Response::new(())) } @@ -3839,8 +4435,8 @@ impl World for EcsService { })) .await .map_err(|_| Status::internal("subscription setup failed"))?; - let seen = self - .snapshot_token_balance_seen( + let latest = self + .snapshot_token_balance_state( &request.account_addresses, &request.contract_addresses, &request.token_ids, @@ -3853,12 +4449,17 @@ impl World for EcsService { contract_addresses: request.contract_addresses, account_addresses: request.account_addresses, token_ids: request.token_ids, - seen, + latest, sender, }, ); self.spawn_token_balance_poll(subscription_id); - Ok(Response::new(ReceiverStream::new(receiver))) + Ok(Response::new(ManagedSubscriptionStream::new( + Arc::clone(&self.state), + SubscriptionKind::TokenBalance, + subscription_id, + receiver, + ))) } async fn update_token_balances_subscription( @@ -3866,8 +4467,8 @@ impl World for EcsService { request: Request, ) -> Result, Status> { let request = request.into_inner(); - let seen = self - .snapshot_token_balance_seen( + let latest = self + .snapshot_token_balance_state( &request.account_addresses, &request.contract_addresses, &request.token_ids, @@ -3884,7 +4485,7 @@ impl World for EcsService { subscription.contract_addresses = request.contract_addresses; subscription.account_addresses = request.account_addresses; subscription.token_ids = request.token_ids; - subscription.seen = seen; + subscription.latest = latest; } Ok(Response::new(())) } @@ -3917,7 +4518,12 @@ impl World for EcsService { }, ); self.spawn_token_poll(subscription_id); - Ok(Response::new(ReceiverStream::new(receiver))) + Ok(Response::new(ManagedSubscriptionStream::new( + Arc::clone(&self.state), + SubscriptionKind::Token, + subscription_id, + receiver, + ))) } async fn update_tokens_subscription( @@ -3976,7 +4582,12 @@ impl World for EcsService { }, ); self.spawn_token_transfer_poll(subscription_id); - Ok(Response::new(ReceiverStream::new(receiver))) + Ok(Response::new(ManagedSubscriptionStream::new( + Arc::clone(&self.state), + SubscriptionKind::TokenTransfer, + subscription_id, + receiver, + ))) } async fn update_token_transfers_subscription( @@ -4033,19 +4644,25 @@ impl World for EcsService { request: Request, ) -> Result, Status> { let request = request.into_inner(); + let subscription_id = Self::next_subscription_id().await; let (sender, receiver) = mpsc::channel(256); sender .send(Ok(SubscribeEventsResponse { event: None })) .await .map_err(|_| Status::internal("subscription setup failed"))?; self.state.event_subscriptions.lock().await.insert( - Self::next_subscription_id().await, + subscription_id, EventSubscription { keys: request.keys, sender, }, ); - Ok(Response::new(ReceiverStream::new(receiver))) + Ok(Response::new(ManagedSubscriptionStream::new( + Arc::clone(&self.state), + SubscriptionKind::Event, + subscription_id, + receiver, + ))) } async fn retrieve_tokens( @@ -4085,14 +4702,17 @@ impl World for EcsService { &self, request: Request, ) -> Result, Status> { + let start = Instant::now(); let query = request .into_inner() .query .ok_or_else(|| Status::invalid_argument("Missing query argument"))?; - let (balances, next_cursor) = self - .load_token_balances(&query) - .await - .map_err(internal_status)?; + let result = self.load_token_balances(&query).await; + ::metrics::histogram!("torii_ecs_retrieve_token_balances_latency_seconds") + .record(start.elapsed().as_secs_f64()); + let (balances, next_cursor) = result.map_err(internal_status)?; + ::metrics::counter!("torii_ecs_retrieve_token_balances_returned_total") + .increment(balances.len() as u64); Ok(Response::new(RetrieveTokenBalancesResponse { next_cursor, balances, @@ -4142,7 +4762,12 @@ impl World for EcsService { }, ); self.spawn_transaction_poll(subscription_id); - Ok(Response::new(ReceiverStream::new(receiver))) + Ok(Response::new(ManagedSubscriptionStream::new( + Arc::clone(&self.state), + SubscriptionKind::Transaction, + subscription_id, + receiver, + ))) } async fn retrieve_contracts( @@ -4228,6 +4853,9 @@ struct EntityModelRow { table_id: String, entity_id: String, row_json: String, + created_at: u64, + updated_at: u64, + executed_at: u64, } struct SnapshotJsonSerializer; @@ -4236,6 +4864,30 @@ fn internal_status(error: anyhow::Error) -> Status { Status::internal(error.to_string()) } +fn append_entity_key_pairs_filter<'a>( + builder: &mut QueryBuilder<'a, Any>, + world_column: &str, + entity_column: &str, + entity_keys: &'a [(String, String)], +) { + builder.push(" AND ("); + builder.push(world_column); + builder.push(", "); + builder.push(entity_column); + builder.push(") IN ("); + for (i, (world_address, entity_id)) in entity_keys.iter().enumerate() { + if i > 0 { + builder.push(", "); + } + builder.push("("); + builder.push_bind(world_address); + builder.push(", "); + builder.push_bind(entity_id); + builder.push(")"); + } + builder.push(")"); +} + fn sql_hex_blob_expr(backend: DbBackend, column: &str, width: usize) -> String { match backend { DbBackend::Sqlite => { @@ -4259,11 +4911,11 @@ fn sql_hex_literal_expr(backend: DbBackend, value: &str, width: usize) -> String } async fn attach_sqlite_databases( - conn: &mut PoolConnection, + conn: &mut AnyConnection, erc20_url: Option<&str>, erc721_url: Option<&str>, erc1155_url: Option<&str>, -) -> Result<()> { +) -> sqlx::Result<()> { for (schema, url) in [ ("erc20", erc20_url), ("erc721", erc721_url), @@ -4277,14 +4929,14 @@ async fn attach_sqlite_databases( } async fn attach_sqlite_database( - conn: &mut PoolConnection, + conn: &mut AnyConnection, schema: &str, url: &str, -) -> Result<()> { +) -> sqlx::Result<()> { let attached = sqlx::query_scalar::("SELECT 1 FROM pragma_database_list WHERE name = ? LIMIT 1") .bind(schema) - .fetch_optional(&mut **conn) + .fetch_optional(&mut *conn) .await? .is_some(); if attached { @@ -4292,12 +4944,22 @@ async fn attach_sqlite_database( } let path = sqlite_db_path(url).replace('\'', "''"); + // sqlite-dynamic-ok: ATTACH requires the database path and schema identifier in SQL text. sqlx::query(&format!("ATTACH DATABASE '{path}' AS {schema}")) - .execute(&mut **conn) + .execute(&mut *conn) .await?; Ok(()) } +fn sqlite_master_preview_sql(schema: &str) -> &'static str { + match schema { + "erc20" => "SELECT name FROM erc20.sqlite_master WHERE type='table' LIMIT 5", + "erc721" => "SELECT name FROM erc721.sqlite_master WHERE type='table' LIMIT 5", + "erc1155" => "SELECT name FROM erc1155.sqlite_master WHERE type='table' LIMIT 5", + _ => "SELECT name FROM sqlite_master WHERE 1 = 0", + } +} + fn sqlite_db_path(url: &str) -> String { let path = url .strip_prefix("sqlite://") @@ -4454,7 +5116,7 @@ fn row_to_model_struct(table: &DojoTable, row: &Map) -> Result = serde_json::from_str(row_json)?; aggregate .models .push(row_to_model_struct(&table.table, &object)?); + ::metrics::histogram!("torii_ecs_entity_snapshot_row_decode_seconds") + .record(row_decode_start.elapsed().as_secs_f64()); if aggregate.created_at == 0 || table_meta.created_at < aggregate.created_at { aggregate.created_at = table_meta.created_at; } @@ -4565,6 +5227,8 @@ fn build_entity_from_snapshot( if aggregate.models.is_empty() { return Ok(None); } + ::metrics::histogram!("torii_ecs_build_entity_from_snapshot_seconds") + .record(build_start.elapsed().as_secs_f64()); Ok(Some(aggregate)) } @@ -4634,7 +5298,7 @@ fn is_json_text_type(type_def: &TypeDef) -> bool { ) } -fn type_to_proto_value(type_def: &TypeDef, value: Value) -> Result { +fn type_to_proto_value(type_def: &TypeDef, value: &Value) -> Result { use types::ty::TyType; let ty_type = match type_def { @@ -4659,14 +5323,14 @@ fn type_to_proto_value(type_def: &TypeDef, value: Value) -> Result { )), }), TypeDef::U64 => TyType::Primitive(types::Primitive { - primitive_type: Some(types::primitive::PrimitiveType::U64(value_as_u64(&value))), + primitive_type: Some(types::primitive::PrimitiveType::U64(value_as_u64(value))), }), TypeDef::Felt252 | TypeDef::ClassHash | TypeDef::ContractAddress | TypeDef::StorageAddress | TypeDef::StorageBaseAddress => { - let bytes = value_as_felt_bytes(&value)?; + let bytes = value_as_felt_bytes(value)?; let primitive = match type_def { TypeDef::ClassHash => types::primitive::PrimitiveType::ClassHash(bytes), TypeDef::ContractAddress @@ -4682,27 +5346,27 @@ fn type_to_proto_value(type_def: &TypeDef, value: Value) -> Result { } TypeDef::EthAddress => TyType::Primitive(types::Primitive { primitive_type: Some(types::primitive::PrimitiveType::EthAddress( - value_as_fixed_unsigned_bytes(&value, 20)?, + value_as_fixed_unsigned_bytes(value, 20)?, )), }), TypeDef::U128 => TyType::Primitive(types::Primitive { primitive_type: Some(types::primitive::PrimitiveType::U128( - value_as_fixed_unsigned_bytes(&value, 16)?, + value_as_fixed_unsigned_bytes(value, 16)?, )), }), TypeDef::I128 => TyType::Primitive(types::Primitive { primitive_type: Some(types::primitive::PrimitiveType::I128( - value_as_fixed_signed_bytes(&value, 16)?, + value_as_fixed_signed_bytes(value, 16)?, )), }), TypeDef::U256 => TyType::Primitive(types::Primitive { primitive_type: Some(types::primitive::PrimitiveType::U256( - value_as_fixed_unsigned_bytes(&value, 32)?, + value_as_fixed_unsigned_bytes(value, 32)?, )), }), TypeDef::U512 => TyType::Primitive(types::Primitive { primitive_type: Some(types::primitive::PrimitiveType::U256( - value_as_fixed_unsigned_bytes(&value, 64)?, + value_as_fixed_unsigned_bytes(value, 64)?, )), }), TypeDef::Utf8String @@ -4710,21 +5374,21 @@ fn type_to_proto_value(type_def: &TypeDef, value: Value) -> Result { | TypeDef::ByteArray | TypeDef::ByteArrayEncoded(_) | TypeDef::Bytes31 - | TypeDef::Bytes31Encoded(_) => TyType::Bytearray(value_as_string(&value)), + | TypeDef::Bytes31Encoded(_) => TyType::Bytearray(value_as_string(value)), TypeDef::Struct(def) => { - let object = value.as_object().cloned().unwrap_or_default(); + let object = value.as_object(); TyType::Struct(types::Struct { name: def.name.clone(), children: def .members .iter() .map(|member| { + let member_value = object + .and_then(|object| object.get(&member.name)) + .unwrap_or(&Value::Null); Ok(types::Member { name: member.name.clone(), - ty: Some(type_to_proto_value( - &member.type_def, - object.get(&member.name).cloned().unwrap_or(Value::Null), - )?), + ty: Some(type_to_proto_value(&member.type_def, member_value)?), key: member.attributes.has_attribute("key"), }) }) @@ -4734,9 +5398,8 @@ fn type_to_proto_value(type_def: &TypeDef, value: Value) -> Result { TypeDef::Tuple(def) => TyType::Tuple(types::Array { children: value .as_array() - .cloned() - .unwrap_or_default() - .into_iter() + .map_or(&[] as &[Value], Vec::as_slice) + .iter() .zip(def.elements.iter()) .map(|(item, inner)| type_to_proto_value(inner, item)) .collect::>>()?, @@ -4744,26 +5407,24 @@ fn type_to_proto_value(type_def: &TypeDef, value: Value) -> Result { TypeDef::Array(def) => TyType::Array(types::Array { children: value .as_array() - .cloned() - .unwrap_or_default() - .into_iter() + .map_or(&[] as &[Value], Vec::as_slice) + .iter() .map(|item| type_to_proto_value(&def.type_def, item)) .collect::>>()?, }), TypeDef::FixedArray(def) => TyType::FixedSizeArray(types::FixedSizeArray { children: value .as_array() - .cloned() - .unwrap_or_default() - .into_iter() + .map_or(&[] as &[Value], Vec::as_slice) + .iter() .map(|item| type_to_proto_value(&def.type_def, item)) .collect::>>()?, size: def.size, }), TypeDef::Enum(def) => { - let object = value.as_object().cloned().unwrap_or_default(); + let object = value.as_object(); let variant_name = object - .get("option") + .and_then(|object| object.get("option")) .and_then(Value::as_str) .unwrap_or_default() .to_string(); @@ -4776,8 +5437,7 @@ fn type_to_proto_value(type_def: &TypeDef, value: Value) -> Result { }) .map(|(index, variant)| { let ty = object - .get(&variant.name) - .cloned() + .and_then(|object| object.get(&variant.name)) .map(|value| type_to_proto_value(&variant.type_def, value)) .transpose()? .unwrap_or_default(); @@ -4824,17 +5484,17 @@ fn type_to_proto_value(type_def: &TypeDef, value: Value) -> Result { } } TypeDef::Result(def) => { - let object = value.as_object().cloned().unwrap_or_default(); - if let Some(ok) = object.get("Ok").cloned() { + let object = value.as_object(); + if let Some(ok) = object.and_then(|object| object.get("Ok")) { return type_to_proto_value(&def.ok, ok); } - if let Some(err) = object.get("Err").cloned() { + if let Some(err) = object.and_then(|object| object.get("Err")) { return type_to_proto_value(&def.err, err); } - TyType::Bytearray(value_as_string(&value)) + TyType::Bytearray(value_as_string(value)) } TypeDef::None | TypeDef::Felt252Dict(_) | TypeDef::Ref(_) | TypeDef::Custom(_) => { - TyType::Bytearray(value_as_string(&value)) + TyType::Bytearray(value_as_string(value)) } }; @@ -4870,10 +5530,17 @@ fn value_as_felt_bytes(value: &Value) -> Result> { return Ok(Felt::ZERO.to_bytes_be().to_vec()); } - Ok(felt_from_hex(trimmed)?.to_bytes_be().to_vec()) + felt_like_string_to_bytes(trimmed) } fn value_as_fixed_unsigned_bytes(value: &Value, width: usize) -> Result> { + if let Some(value) = value.as_str() { + let trimmed = value.trim(); + if let Some(bytes) = decode_prefixed_hex_to_padded_bytes(trimmed, width)? { + return Ok(bytes); + } + } + let unsigned = value_as_big_uint(value)?; let bytes = unsigned.to_bytes_be(); if bytes.len() > width { @@ -5001,6 +5668,41 @@ fn parse_big_uint_with_radix(raw: &str, radix: u32) -> Result { .ok_or_else(|| anyhow!("failed to parse integer value: {raw}")) } +fn felt_like_string_to_bytes(raw: &str) -> Result> { + let trimmed = raw.trim(); + if let Some(bytes) = decode_prefixed_hex_to_padded_bytes(trimmed, 32)? { + return Ok(bytes); + } + + Ok(felt_from_hex(trimmed)?.to_bytes_be().to_vec()) +} + +fn decode_prefixed_hex_to_padded_bytes(raw: &str, width: usize) -> Result>> { + let Some(hex) = raw.strip_prefix("0x").or_else(|| raw.strip_prefix("0X")) else { + return Ok(None); + }; + + if hex.is_empty() { + return Ok(Some(vec![0_u8; width])); + } + + let mut normalized = String::with_capacity(hex.len() + (hex.len() % 2)); + if hex.len() % 2 != 0 { + normalized.push('0'); + } + normalized.push_str(hex); + + let bytes = hex::decode(&normalized) + .map_err(|err| anyhow!("failed to parse hex value {raw}: {err}"))?; + if bytes.len() > width { + anyhow::bail!("value does not fit in {width} bytes"); + } + + let mut out = vec![0_u8; width]; + out[width - bytes.len()..].copy_from_slice(&bytes); + Ok(Some(out)) +} + fn push_blob_in_filter( builder: &mut QueryBuilder<'_, Any>, column: &str, @@ -5020,6 +5722,52 @@ fn push_blob_in_filter( true } +fn push_blob_in_filter_with_mode( + builder: &mut QueryBuilder<'_, Any>, + column: &str, + values: &[Vec], + has_where: bool, +) -> bool { + if values.is_empty() { + return has_where; + } + if has_where { + builder.push(" AND "); + } else { + builder.push(" WHERE "); + } + builder.push(column); + builder.push(" IN ("); + { + let mut separated = builder.separated(", "); + for value in values { + separated.push_bind(value.clone()); + } + } + builder.push(")"); + true +} + +fn normalize_token_filter_values(values: &[Vec]) -> Vec> { + values + .iter() + .map(|value| { + let first_non_zero = value.iter().position(|byte| *byte != 0); + match first_non_zero { + Some(index) => value[index..].to_vec(), + None => vec![0], + } + }) + .collect() +} + +fn sql_blob_numeric_order_expr(backend: DbBackend, column: &str) -> String { + match backend { + DbBackend::Sqlite => format!("length({column})"), + DbBackend::Postgres => format!("octet_length({column})"), + } +} + fn push_transfer_filters( builder: &mut QueryBuilder<'_, Any>, query: &types::TokenTransferQuery, @@ -5091,7 +5839,7 @@ fn token_subscription_key(token: &types::Token) -> String { ) } -fn token_balance_subscription_key(balance: &types::TokenBalance) -> String { +fn token_balance_identity_key(balance: &types::TokenBalance) -> String { format!( "{}:{}:{}", hex::encode(&balance.contract_address), @@ -5104,6 +5852,22 @@ fn token_balance_subscription_key(balance: &types::TokenBalance) -> String { ) } +fn token_balance_state_fingerprint(balance: &types::TokenBalance) -> Vec { + balance.balance.clone() +} + +fn token_balance_subscription_state(balances: &[types::TokenBalance]) -> HashMap> { + balances + .iter() + .map(|balance| { + ( + token_balance_identity_key(balance), + token_balance_state_fingerprint(balance), + ) + }) + .collect() +} + fn canonical_felt_bytes_from_db(bytes: &[u8]) -> Result> { Ok(felt_from_bytes(bytes)?.to_bytes_be().to_vec()) } @@ -5612,24 +6376,124 @@ mod tests { .collect() } - fn member_bool_clause(model: &str, member: &str, value: bool) -> types::Clause { - types::Clause { - clause_type: Some(ClauseType::Member(types::MemberClause { - model: model.to_string(), - member: member.to_string(), - operator: ComparisonOperator::Eq as i32, - value: Some(types::MemberValue { - value_type: Some(ValueType::Primitive(types::Primitive { - primitive_type: Some(types::primitive::PrimitiveType::Bool(value)), - })), - }), - })), - } - } - + async fn explain_query_plan_details(conn: &mut PoolConnection, sql: &str) -> Vec { + // sqlite-dynamic-ok: test-only helper that needs to wrap arbitrary SQL in EXPLAIN QUERY PLAN. + sqlx::query(&format!("EXPLAIN QUERY PLAN {sql}")) + .fetch_all(&mut **conn) + .await + .expect("explain query plan") + .into_iter() + .map(|row| row.try_get::("detail").expect("detail column")) + .collect() + } + + async fn wait_for_entity_subscription_count( + service: &EcsService, + kind: TableKind, + expected: usize, + ) { + timeout(Duration::from_secs(1), async { + loop { + let count = service + .state + .entity_subscription_registry(kind) + .read() + .await + .subscription_count(); + if count == expected { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .expect("subscription count timeout"); + } + + async fn wait_for_entity_subscription_group_count( + service: &EcsService, + kind: TableKind, + expected: usize, + ) { + timeout(Duration::from_secs(1), async { + loop { + let count = service + .state + .entity_subscription_registry(kind) + .read() + .await + .group_count(); + if count == expected { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .expect("subscription group count timeout"); + } + + async fn wait_for_subscription_count( + service: &EcsService, + kind: SubscriptionKind, + expected: usize, + ) { + timeout(Duration::from_secs(1), async { + loop { + let count = match kind { + SubscriptionKind::Entity(kind) => service + .state + .entity_subscription_registry(kind) + .read() + .await + .subscription_count(), + SubscriptionKind::Event => service.state.event_subscriptions.lock().await.len(), + SubscriptionKind::Contract => { + service.state.contract_subscriptions.lock().await.len() + } + SubscriptionKind::Token => service.state.token_subscriptions.lock().await.len(), + SubscriptionKind::TokenBalance => { + service.state.token_balance_subscriptions.lock().await.len() + } + SubscriptionKind::TokenTransfer => service + .state + .token_transfer_subscriptions + .lock() + .await + .len(), + SubscriptionKind::Transaction => { + service.state.transaction_subscriptions.lock().await.len() + } + }; + if count == expected { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .expect("subscription count timeout"); + } + + fn member_bool_clause(model: &str, member: &str, value: bool) -> types::Clause { + types::Clause { + clause_type: Some(ClauseType::Member(types::MemberClause { + model: model.to_string(), + member: member.to_string(), + operator: ComparisonOperator::Eq as i32, + value: Some(types::MemberValue { + value_type: Some(ValueType::Primitive(types::Primitive { + primitive_type: Some(types::primitive::PrimitiveType::Bool(value)), + })), + }), + })), + } + } + #[test] fn serializes_u128_as_fixed_width_u128() { - let ty = type_to_proto_value(&TypeDef::U128, serde_json::json!(10000_u64)).expect("u128"); + let input = serde_json::json!(10000_u64); + let ty = type_to_proto_value(&TypeDef::U128, &input).expect("u128"); let types::ty::TyType::Primitive(primitive) = ty.ty_type.expect("primitive") else { panic!("expected primitive"); @@ -5647,11 +6511,8 @@ mod tests { #[test] fn serializes_eth_address_as_20_bytes() { - let ty = type_to_proto_value( - &TypeDef::EthAddress, - serde_json::json!("0x11223344556677889900aabbccddeeff00112233"), - ) - .expect("eth address"); + let input = serde_json::json!("0x11223344556677889900aabbccddeeff00112233"); + let ty = type_to_proto_value(&TypeDef::EthAddress, &input).expect("eth address"); let types::ty::TyType::Primitive(primitive) = ty.ty_type.expect("primitive") else { panic!("expected primitive"); @@ -5670,7 +6531,8 @@ mod tests { #[test] fn serializes_u256_as_fixed_width_u256() { - let ty = type_to_proto_value(&TypeDef::U256, serde_json::json!("0x1234")).expect("u256"); + let input = serde_json::json!("0x1234"); + let ty = type_to_proto_value(&TypeDef::U256, &input).expect("u256"); let types::ty::TyType::Primitive(primitive) = ty.ty_type.expect("primitive") else { panic!("expected primitive"); @@ -5691,6 +6553,19 @@ mod tests { assert_eq!(value.to_string(), "1549526502204757800000"); } + #[test] + fn seen_cache_caps_growth() { + let mut cache = SeenCache::with_capacity(2); + cache.extend(["first".to_string(), "second".to_string()]); + assert!(cache.contains("first")); + assert!(cache.contains("second")); + + cache.extend(["third".to_string()]); + assert!(!cache.contains("first")); + assert!(cache.contains("second")); + assert!(cache.contains("third")); + } + async fn seed_entity( service: &EcsService, kind: TableKind, @@ -6056,6 +6931,172 @@ mod tests { assert!(filtered.is_err(), "entity should be filtered out"); } + #[tokio::test] + async fn subscribe_entities_drop_cleans_up_subscription() { + let db_path = test_db_path("entities-drop-cleanup"); + let service = EcsService::new(&db_path, Some(1), None, None, None) + .await + .expect("service init"); + let world = Felt::from(42_u64); + + let response = service + .subscribe_entities(Request::new(SubscribeEntitiesRequest { + clause: None, + world_addresses: vec![world.to_bytes_be().to_vec()], + })) + .await + .expect("subscribe entities"); + let mut stream = response.into_inner(); + + let _setup = timeout(Duration::from_secs(1), stream.next()) + .await + .expect("setup timeout") + .expect("setup frame") + .expect("setup ok"); + wait_for_entity_subscription_count(&service, TableKind::Entity, 1).await; + + drop(stream); + + wait_for_entity_subscription_count(&service, TableKind::Entity, 0).await; + } + + #[tokio::test] + async fn subscribe_entities_repeated_drop_keeps_registry_flat() { + let db_path = test_db_path("entities-drop-loop"); + let service = EcsService::new(&db_path, Some(1), None, None, None) + .await + .expect("service init"); + let world = Felt::from(43_u64); + + for _ in 0..5 { + let response = service + .subscribe_entities(Request::new(SubscribeEntitiesRequest { + clause: None, + world_addresses: vec![world.to_bytes_be().to_vec()], + })) + .await + .expect("subscribe entities"); + let mut stream = response.into_inner(); + + let _setup = timeout(Duration::from_secs(1), stream.next()) + .await + .expect("setup timeout") + .expect("setup frame") + .expect("setup ok"); + wait_for_entity_subscription_count(&service, TableKind::Entity, 1).await; + + drop(stream); + + wait_for_entity_subscription_count(&service, TableKind::Entity, 0).await; + } + } + + #[tokio::test] + async fn subscribe_entities_identical_filters_share_group() { + let db_path = test_db_path("entities-shared-group"); + let service = EcsService::new(&db_path, Some(1), None, None, None) + .await + .expect("service init"); + let world = Felt::from(44_u64); + let clause = member_bool_clause("test-Lobby", "open", true); + + let response_a = service + .subscribe_entities(Request::new(SubscribeEntitiesRequest { + clause: Some(clause.clone()), + world_addresses: vec![world.to_bytes_be().to_vec()], + })) + .await + .expect("subscribe entities a"); + let response_b = service + .subscribe_entities(Request::new(SubscribeEntitiesRequest { + clause: Some(clause), + world_addresses: vec![world.to_bytes_be().to_vec()], + })) + .await + .expect("subscribe entities b"); + + let mut stream_a = response_a.into_inner(); + let mut stream_b = response_b.into_inner(); + + let _setup_a = timeout(Duration::from_secs(1), stream_a.next()) + .await + .expect("setup timeout a") + .expect("setup frame a") + .expect("setup ok a"); + let _setup_b = timeout(Duration::from_secs(1), stream_b.next()) + .await + .expect("setup timeout b") + .expect("setup frame b") + .expect("setup ok b"); + + wait_for_entity_subscription_count(&service, TableKind::Entity, 2).await; + wait_for_entity_subscription_group_count(&service, TableKind::Entity, 1).await; + + drop(stream_a); + wait_for_entity_subscription_count(&service, TableKind::Entity, 1).await; + wait_for_entity_subscription_group_count(&service, TableKind::Entity, 1).await; + + drop(stream_b); + wait_for_entity_subscription_count(&service, TableKind::Entity, 0).await; + wait_for_entity_subscription_group_count(&service, TableKind::Entity, 0).await; + } + + #[tokio::test] + async fn update_entities_subscription_moves_between_groups() { + let db_path = test_db_path("entities-update-groups"); + let service = EcsService::new(&db_path, Some(1), None, None, None) + .await + .expect("service init"); + let world = Felt::from(45_u64); + + let response_a = service + .subscribe_entities(Request::new(SubscribeEntitiesRequest { + clause: Some(member_bool_clause("test-Lobby", "open", true)), + world_addresses: vec![world.to_bytes_be().to_vec()], + })) + .await + .expect("subscribe entities a"); + let response_b = service + .subscribe_entities(Request::new(SubscribeEntitiesRequest { + clause: Some(member_bool_clause("test-Lobby", "open", true)), + world_addresses: vec![world.to_bytes_be().to_vec()], + })) + .await + .expect("subscribe entities b"); + + let mut stream_a = response_a.into_inner(); + let mut stream_b = response_b.into_inner(); + let setup_a = timeout(Duration::from_secs(1), stream_a.next()) + .await + .expect("setup timeout a") + .expect("setup frame a") + .expect("setup ok a"); + let _setup_b = timeout(Duration::from_secs(1), stream_b.next()) + .await + .expect("setup timeout b") + .expect("setup frame b") + .expect("setup ok b"); + + wait_for_entity_subscription_group_count(&service, TableKind::Entity, 1).await; + + service + .update_entities_subscription(Request::new(UpdateEntitiesSubscriptionRequest { + subscription_id: setup_a.subscription_id, + clause: Some(member_bool_clause("test-Lobby", "open", false)), + world_addresses: vec![world.to_bytes_be().to_vec()], + })) + .await + .expect("update entities subscription"); + + wait_for_entity_subscription_count(&service, TableKind::Entity, 2).await; + wait_for_entity_subscription_group_count(&service, TableKind::Entity, 2).await; + + drop(stream_a); + drop(stream_b); + wait_for_entity_subscription_count(&service, TableKind::Entity, 0).await; + wait_for_entity_subscription_group_count(&service, TableKind::Entity, 0).await; + } + #[tokio::test] async fn subscribe_event_messages_update_flow() { let db_path = test_db_path("event-messages-sub"); @@ -6122,6 +7163,35 @@ mod tests { assert!(filtered.is_err(), "event message should be filtered out"); } + #[tokio::test] + async fn subscribe_event_messages_drop_cleans_up_subscription() { + let db_path = test_db_path("event-messages-drop-cleanup"); + let service = EcsService::new(&db_path, Some(1), None, None, None) + .await + .expect("service init"); + let world = Felt::from(44_u64); + + let response = service + .subscribe_event_messages(Request::new(SubscribeEntitiesRequest { + clause: None, + world_addresses: vec![world.to_bytes_be().to_vec()], + })) + .await + .expect("subscribe event messages"); + let mut stream = response.into_inner(); + + let _setup = timeout(Duration::from_secs(1), stream.next()) + .await + .expect("setup timeout") + .expect("setup frame") + .expect("setup ok"); + wait_for_entity_subscription_count(&service, TableKind::EventMessage, 1).await; + + drop(stream); + + wait_for_entity_subscription_count(&service, TableKind::EventMessage, 0).await; + } + #[tokio::test] async fn subscribe_events_keys_filter() { let db_path = test_db_path("events-sub"); @@ -6187,6 +7257,31 @@ mod tests { ); } + #[tokio::test] + async fn subscribe_events_drop_cleans_up_subscription() { + let db_path = test_db_path("events-drop-cleanup"); + let service = EcsService::new(&db_path, Some(1), None, None, None) + .await + .expect("service init"); + + let response = service + .subscribe_events(Request::new(SubscribeEventsRequest { keys: vec![] })) + .await + .expect("subscribe events"); + let mut stream = response.into_inner(); + + let _setup = timeout(Duration::from_secs(1), stream.next()) + .await + .expect("setup timeout") + .expect("setup frame") + .expect("setup ok"); + wait_for_subscription_count(&service, SubscriptionKind::Event, 1).await; + + drop(stream); + + wait_for_subscription_count(&service, SubscriptionKind::Event, 0).await; + } + #[tokio::test] async fn subscribe_contracts_query_filter() { let db_path = test_db_path("contracts-sub"); @@ -6234,57 +7329,365 @@ mod tests { assert_eq!(contract.contract_type, ContractType::World as i32); } - #[test] - fn entity_keys_clause_scopes_key_extraction_to_requested_models() { - let entity = types::Entity { - hashed_keys: vec![], - models: vec![ - types::Struct { - name: "NUMS-Other".to_string(), - children: vec![keyed_member("other_id", Felt::from(0x111_u64))], - }, - types::Struct { - name: "NUMS-Config".to_string(), - children: vec![keyed_member("config_id", Felt::from(0x222_u64))], - }, - ], - created_at: 0, - updated_at: 0, - executed_at: 0, - world_address: vec![], - }; - let clause = types::Clause { - clause_type: Some(ClauseType::Keys(types::KeysClause { - models: vec!["NUMS-Config".to_string()], - pattern_matching: PatternMatching::VariableLen as i32, - keys: vec![Felt::from(0x222_u64).to_bytes_be().to_vec()], - })), - }; + #[tokio::test] + async fn subscribe_contracts_drop_cleans_up_subscription() { + let db_path = test_db_path("contracts-drop-cleanup"); + let service = EcsService::new(&db_path, Some(1), None, None, None) + .await + .expect("service init"); - assert!(entity_matches_clause(&entity, &clause)); + let response = service + .subscribe_contracts(Request::new(SubscribeContractsRequest { + query: Some(types::ContractQuery { + contract_addresses: vec![], + contract_types: vec![], + }), + })) + .await + .expect("subscribe contracts"); + let stream = response.into_inner(); + + wait_for_subscription_count(&service, SubscriptionKind::Contract, 1).await; + + drop(stream); + + wait_for_subscription_count(&service, SubscriptionKind::Contract, 0).await; } #[tokio::test] - async fn retrieve_entities_without_filters_uses_bounded_query() { - let db_path = test_db_path("retrieve-entities"); + async fn subscribe_transactions_drop_cleans_up_subscription() { + let db_path = test_db_path("transactions-drop-cleanup"); let service = EcsService::new(&db_path, Some(1), None, None, None) .await .expect("service init"); - let world = Felt::from(900_u64); - seed_entity( - &service, - TableKind::Entity, - world, - Felt::from(901_u64), - "test-ModelA", - Felt::from(1001_u64), - true, - ) - .await; - seed_entity( - &service, - TableKind::Entity, + let response = service + .subscribe_transactions(Request::new(SubscribeTransactionsRequest { filter: None })) + .await + .expect("subscribe transactions"); + let mut stream = response.into_inner(); + + let _setup = timeout(Duration::from_secs(1), stream.next()) + .await + .expect("setup timeout") + .expect("setup frame") + .expect("setup ok"); + wait_for_subscription_count(&service, SubscriptionKind::Transaction, 1).await; + + drop(stream); + + wait_for_subscription_count(&service, SubscriptionKind::Transaction, 0).await; + } + + #[tokio::test] + async fn subscribe_tokens_drop_cleans_up_subscription() { + let db_path = test_db_path("tokens-drop-cleanup"); + let erc20_url = temp_sqlite_url("erc20-tokens-drop-cleanup"); + let service = EcsService::new(&db_path, Some(1), Some(&erc20_url), None, None) + .await + .expect("service init"); + service + .attach_erc_databases() + .await + .expect("attach erc dbs"); + + sqlx::query( + "CREATE TABLE erc20.token_metadata ( + token BLOB PRIMARY KEY, + name TEXT, + symbol TEXT, + decimals TEXT, + total_supply BLOB + )", + ) + .execute(&service.state.pool) + .await + .expect("create erc20 token metadata"); + + let response = service + .subscribe_tokens(Request::new(SubscribeTokensRequest { + contract_addresses: vec![], + token_ids: vec![], + })) + .await + .expect("subscribe tokens"); + let mut stream = response.into_inner(); + + let _setup = timeout(Duration::from_secs(1), stream.next()) + .await + .expect("setup timeout") + .expect("setup frame") + .expect("setup ok"); + wait_for_subscription_count(&service, SubscriptionKind::Token, 1).await; + + drop(stream); + + wait_for_subscription_count(&service, SubscriptionKind::Token, 0).await; + } + + #[tokio::test] + async fn subscribe_token_balances_emits_updates_for_existing_balance_rows() { + let db_path = test_db_path("token-balances-live-updates"); + let erc20_url = temp_sqlite_url("erc20-token-balances-live-updates"); + let service = EcsService::new(&db_path, Some(1), Some(&erc20_url), None, None) + .await + .expect("service init"); + service + .attach_erc_databases() + .await + .expect("attach erc dbs"); + + sqlx::query( + "CREATE TABLE erc20.balances ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + token BLOB NOT NULL, + wallet BLOB NOT NULL, + balance BLOB NOT NULL, + last_block TEXT NOT NULL, + last_tx_hash BLOB NOT NULL + )", + ) + .execute(&service.state.pool) + .await + .expect("create erc20 balances"); + + let token = Felt::from(0x99_u64).to_bytes_be().to_vec(); + let wallet = Felt::from(0x55_u64).to_bytes_be().to_vec(); + let original_balance = u256_bytes_from_u64(1); + let updated_balance = u256_bytes_from_u64(5); + + sqlx::query( + "INSERT INTO erc20.balances (token, wallet, balance, last_block, last_tx_hash) + VALUES (?1, ?2, ?3, ?4, ?5)", + ) + .bind(&token) + .bind(&wallet) + .bind(&original_balance) + .bind("1") + .bind(vec![0_u8; 32]) + .execute(&service.state.pool) + .await + .expect("insert erc20 balance"); + + let response = service + .subscribe_token_balances(Request::new(SubscribeTokenBalancesRequest { + account_addresses: vec![wallet.clone()], + contract_addresses: vec![token.clone()], + token_ids: vec![], + })) + .await + .expect("subscribe token balances"); + let mut stream = response.into_inner(); + + let setup = timeout(Duration::from_secs(1), stream.next()) + .await + .expect("setup timeout") + .expect("setup frame") + .expect("setup ok"); + let subscription_id = setup.subscription_id; + assert!(setup.balance.is_none()); + + sqlx::query( + "UPDATE erc20.balances + SET balance = ?1, last_block = ?2, last_tx_hash = ?3 + WHERE token = ?4 AND wallet = ?5", + ) + .bind(&updated_balance) + .bind("2") + .bind(vec![1_u8; 32]) + .bind(&token) + .bind(&wallet) + .execute(&service.state.pool) + .await + .expect("update erc20 balance"); + + let first_update = timeout(Duration::from_secs(3), stream.next()) + .await + .expect("first update timeout") + .expect("first update frame") + .expect("first update ok"); + assert_eq!(first_update.subscription_id, subscription_id); + assert_eq!( + first_update.balance.expect("first balance payload").balance, + updated_balance + ); + + sqlx::query( + "UPDATE erc20.balances + SET balance = ?1, last_block = ?2, last_tx_hash = ?3 + WHERE token = ?4 AND wallet = ?5", + ) + .bind(&original_balance) + .bind("3") + .bind(vec![2_u8; 32]) + .bind(&token) + .bind(&wallet) + .execute(&service.state.pool) + .await + .expect("restore erc20 balance"); + + let second_update = timeout(Duration::from_secs(3), stream.next()) + .await + .expect("second update timeout") + .expect("second update frame") + .expect("second update ok"); + assert_eq!(second_update.subscription_id, subscription_id); + assert_eq!( + second_update + .balance + .expect("second balance payload") + .balance, + original_balance + ); + } + + #[tokio::test] + async fn subscribe_token_balances_drop_cleans_up_subscription() { + let db_path = test_db_path("token-balances-drop-cleanup"); + let erc20_url = temp_sqlite_url("erc20-token-balances-drop-cleanup"); + let service = EcsService::new(&db_path, Some(1), Some(&erc20_url), None, None) + .await + .expect("service init"); + service + .attach_erc_databases() + .await + .expect("attach erc dbs"); + + sqlx::query( + "CREATE TABLE erc20.balances ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + token BLOB NOT NULL, + wallet BLOB NOT NULL, + balance BLOB NOT NULL, + last_block TEXT NOT NULL, + last_tx_hash BLOB NOT NULL + )", + ) + .execute(&service.state.pool) + .await + .expect("create erc20 balances"); + + let response = service + .subscribe_token_balances(Request::new(SubscribeTokenBalancesRequest { + account_addresses: vec![], + contract_addresses: vec![], + token_ids: vec![], + })) + .await + .expect("subscribe token balances"); + let mut stream = response.into_inner(); + + let _setup = timeout(Duration::from_secs(1), stream.next()) + .await + .expect("setup timeout") + .expect("setup frame") + .expect("setup ok"); + wait_for_subscription_count(&service, SubscriptionKind::TokenBalance, 1).await; + + drop(stream); + + wait_for_subscription_count(&service, SubscriptionKind::TokenBalance, 0).await; + } + + #[tokio::test] + async fn subscribe_token_transfers_drop_cleans_up_subscription() { + let db_path = test_db_path("token-transfers-drop-cleanup"); + let erc20_url = temp_sqlite_url("erc20-token-transfers-drop-cleanup"); + let service = EcsService::new(&db_path, Some(1), Some(&erc20_url), None, None) + .await + .expect("service init"); + service + .attach_erc_databases() + .await + .expect("attach erc dbs"); + + sqlx::query( + "CREATE TABLE erc20.transfers ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + token BLOB NOT NULL, + from_addr BLOB NOT NULL, + to_addr BLOB NOT NULL, + amount BLOB NOT NULL, + timestamp TEXT + )", + ) + .execute(&service.state.pool) + .await + .expect("create erc20 transfers"); + + let response = service + .subscribe_token_transfers(Request::new(SubscribeTokenTransfersRequest { + account_addresses: vec![], + contract_addresses: vec![], + token_ids: vec![], + })) + .await + .expect("subscribe token transfers"); + let mut stream = response.into_inner(); + + let _setup = timeout(Duration::from_secs(1), stream.next()) + .await + .expect("setup timeout") + .expect("setup frame") + .expect("setup ok"); + wait_for_subscription_count(&service, SubscriptionKind::TokenTransfer, 1).await; + + drop(stream); + + wait_for_subscription_count(&service, SubscriptionKind::TokenTransfer, 0).await; + } + + #[test] + fn entity_keys_clause_scopes_key_extraction_to_requested_models() { + let entity = types::Entity { + hashed_keys: vec![], + models: vec![ + types::Struct { + name: "NUMS-Other".to_string(), + children: vec![keyed_member("other_id", Felt::from(0x111_u64))], + }, + types::Struct { + name: "NUMS-Config".to_string(), + children: vec![keyed_member("config_id", Felt::from(0x222_u64))], + }, + ], + created_at: 0, + updated_at: 0, + executed_at: 0, + world_address: vec![], + }; + let clause = types::Clause { + clause_type: Some(ClauseType::Keys(types::KeysClause { + models: vec!["NUMS-Config".to_string()], + pattern_matching: PatternMatching::VariableLen as i32, + keys: vec![Felt::from(0x222_u64).to_bytes_be().to_vec()], + })), + }; + + assert!(entity_matches_clause(&entity, &clause)); + } + + #[tokio::test] + async fn retrieve_entities_without_filters_uses_bounded_query() { + let db_path = test_db_path("retrieve-entities"); + let service = EcsService::new(&db_path, Some(1), None, None, None) + .await + .expect("service init"); + let world = Felt::from(900_u64); + + seed_entity( + &service, + TableKind::Entity, + world, + Felt::from(901_u64), + "test-ModelA", + Felt::from(1001_u64), + true, + ) + .await; + seed_entity( + &service, + TableKind::Entity, world, Felt::from(902_u64), "test-ModelB", @@ -6380,6 +7783,69 @@ mod tests { assert!(model_names.contains("NUMS-QuestAssociation")); } + #[tokio::test] + async fn retrieve_event_messages_includes_co_located_models_for_matching_entity() { + let db_path = test_db_path("retrieve-event-messages-colocated"); + let service = EcsService::new(&db_path, Some(1), None, None, None) + .await + .expect("service init"); + let world = Felt::from(910_u64); + let entity = Felt::from(10_110_u64); + + seed_model_with_bool( + &service, + TableKind::EventMessage, + world, + Felt::from(9_111_u64), + "NUMS-QuestDefinition", + entity, + "enabled", + true, + ) + .await; + seed_model_with_bool( + &service, + TableKind::EventMessage, + world, + Felt::from(9_112_u64), + "NUMS-QuestAssociation", + entity, + "linked", + true, + ) + .await; + + let response = service + .retrieve_event_messages(Request::new(RetrieveEntitiesRequest { + query: Some(types::Query { + pagination: Some(types::Pagination { + cursor: String::new(), + limit: 10, + direction: PaginationDirection::Forward as i32, + order_by: vec![], + }), + world_addresses: vec![world.to_bytes_be().to_vec()], + models: vec![], + clause: Some(member_bool_clause("NUMS-QuestDefinition", "enabled", true)), + no_hashed_keys: false, + historical: false, + }), + })) + .await + .expect("retrieve event messages") + .into_inner(); + + assert_eq!(response.entities.len(), 1); + let model_names = response.entities[0] + .models + .iter() + .map(|model| model.name.clone()) + .collect::>(); + assert_eq!(model_names.len(), 2); + assert!(model_names.contains("NUMS-QuestDefinition")); + assert!(model_names.contains("NUMS-QuestAssociation")); + } + #[tokio::test] async fn retrieve_entities_keys_clause_fills_first_page_with_matches() { let db_path = test_db_path("retrieve-entities-keys-page"); @@ -6772,20 +8238,142 @@ mod tests { }), })) .await - .expect("retrieve token balances") - .into_inner(); + .expect("retrieve token balances") + .into_inner(); + + assert_eq!(response.balances.len(), 1); + assert_eq!( + response.balances[0].contract_address, + Felt::from(0x99_u64).to_bytes_be().to_vec() + ); + assert_eq!( + response.balances[0].account_address, + Felt::from(0x55_u64).to_bytes_be().to_vec() + ); + assert_eq!(response.balances[0].balance, u256_bytes_from_u64(1)); + assert_eq!(response.balances[0].token_id, None); + } + + #[tokio::test] + async fn retrieve_token_balances_erc721_query_plan_uses_ordered_index() { + let db_path = test_db_path("token-balances-erc721-plan"); + let erc721_url = temp_sqlite_url("erc721-token-balances-plan"); + let service = EcsService::new(&db_path, Some(1), None, Some(&erc721_url), None) + .await + .expect("service init"); + service + .attach_erc_databases() + .await + .expect("attach erc dbs"); + + let mut conn = service + .acquire_initialized_connection() + .await + .expect("setup connection"); + sqlx::query( + "CREATE TABLE erc721.nft_ownership ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + token BLOB NOT NULL, + token_id BLOB NOT NULL, + owner BLOB NOT NULL + )", + ) + .execute(&mut *conn) + .await + .expect("create ownership"); + sqlx::query( + "CREATE INDEX erc721.idx_nft_ownership_owner_token_token_id_ord + ON nft_ownership(owner, token, length(token_id), token_id)", + ) + .execute(&mut *conn) + .await + .expect("create ordered ownership index"); + + let details = explain_query_plan_details( + &mut conn, + "SELECT owner, token, token_id + FROM erc721.nft_ownership + WHERE owner IN (x'01') AND token IN (x'02') + ORDER BY token ASC, owner ASC, length(token_id) ASC, token_id ASC + LIMIT 10", + ) + .await; + let combined = details.join(" | "); + + assert!( + details + .iter() + .any(|detail| detail.contains("idx_nft_ownership_owner_token_token_id_ord")), + "expected ordered ownership index in plan, got: {combined}" + ); + assert!( + !details + .iter() + .any(|detail| detail.contains("USE TEMP B-TREE")), + "expected planner to avoid temp sort, got: {combined}" + ); + } + + #[tokio::test] + async fn retrieve_token_balances_erc1155_query_plan_uses_ordered_index() { + let db_path = test_db_path("token-balances-erc1155-plan"); + let erc1155_url = temp_sqlite_url("erc1155-token-balances-plan"); + let service = EcsService::new(&db_path, Some(1), None, None, Some(&erc1155_url)) + .await + .expect("service init"); + service + .attach_erc_databases() + .await + .expect("attach erc dbs"); - assert_eq!(response.balances.len(), 1); - assert_eq!( - response.balances[0].contract_address, - Felt::from(0x99_u64).to_bytes_be().to_vec() + let mut conn = service + .acquire_initialized_connection() + .await + .expect("setup connection"); + sqlx::query( + "CREATE TABLE erc1155.erc1155_balances ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + contract BLOB NOT NULL, + wallet BLOB NOT NULL, + token_id BLOB NOT NULL, + balance BLOB NOT NULL, + last_block TEXT NOT NULL + )", + ) + .execute(&mut *conn) + .await + .expect("create erc1155 balances"); + sqlx::query( + "CREATE INDEX erc1155.idx_erc1155_balances_wallet_contract_token_id_ord + ON erc1155_balances(wallet, contract, length(token_id), token_id)", + ) + .execute(&mut *conn) + .await + .expect("create ordered erc1155 index"); + + let details = explain_query_plan_details( + &mut conn, + "SELECT wallet, contract, token_id, balance + FROM erc1155.erc1155_balances + WHERE wallet IN (x'01') AND contract IN (x'02') + ORDER BY contract ASC, wallet ASC, length(token_id) ASC, token_id ASC + LIMIT 10", + ) + .await; + let combined = details.join(" | "); + + assert!( + details + .iter() + .any(|detail| detail.contains("idx_erc1155_balances_wallet_contract_token_id_ord")), + "expected ordered erc1155 index in plan, got: {combined}" ); - assert_eq!( - response.balances[0].account_address, - Felt::from(0x55_u64).to_bytes_be().to_vec() + assert!( + !details + .iter() + .any(|detail| detail.contains("USE TEMP B-TREE")), + "expected planner to avoid temp sort, got: {combined}" ); - assert_eq!(response.balances[0].balance, u256_bytes_from_u64(1)); - assert_eq!(response.balances[0].token_id, None); } #[tokio::test] @@ -6932,6 +8520,222 @@ mod tests { assert_eq!(balance.token_id.as_ref(), Some(&expected_token_id)); } + #[tokio::test] + async fn retrieve_token_balances_limits_merged_results_in_sorted_order() { + let db_path = test_db_path("token-balances-limit-order"); + let erc20_url = temp_sqlite_url("erc20-token-balances-limit-order"); + let erc721_url = temp_sqlite_url("erc721-token-balances-limit-order"); + let erc1155_url = temp_sqlite_url("erc1155-token-balances-limit-order"); + let service = EcsService::new( + &db_path, + Some(1), + Some(&erc20_url), + Some(&erc721_url), + Some(&erc1155_url), + ) + .await + .expect("service init"); + service + .attach_erc_databases() + .await + .expect("attach erc dbs"); + + let mut conn = service + .acquire_initialized_connection() + .await + .expect("setup connection"); + sqlx::query( + "CREATE TABLE erc20.balances ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + token BLOB NOT NULL, + wallet BLOB NOT NULL, + balance BLOB NOT NULL, + last_block TEXT NOT NULL, + last_tx_hash BLOB NOT NULL + )", + ) + .execute(&mut *conn) + .await + .expect("create erc20 balances"); + sqlx::query( + "CREATE TABLE erc721.nft_ownership ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + token BLOB NOT NULL, + token_id BLOB NOT NULL, + owner BLOB NOT NULL + )", + ) + .execute(&mut *conn) + .await + .expect("create erc721 ownership"); + sqlx::query( + "CREATE TABLE erc1155.erc1155_balances ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + contract BLOB NOT NULL, + wallet BLOB NOT NULL, + token_id BLOB NOT NULL, + balance BLOB NOT NULL, + last_block TEXT NOT NULL + )", + ) + .execute(&mut *conn) + .await + .expect("create erc1155 balances"); + + sqlx::query( + "INSERT INTO erc20.balances (token, wallet, balance, last_block, last_tx_hash) + VALUES (?1, ?2, ?3, ?4, ?5)", + ) + .bind(Felt::from(0x30_u64).to_bytes_be().to_vec()) + .bind(Felt::from(0x10_u64).to_bytes_be().to_vec()) + .bind(vec![9_u8]) + .bind("1") + .bind(vec![0_u8; 32]) + .execute(&mut *conn) + .await + .expect("insert erc20 balance"); + sqlx::query( + "INSERT INTO erc721.nft_ownership (token, token_id, owner) + VALUES (?1, ?2, ?3)", + ) + .bind(Felt::from(0x10_u64).to_bytes_be().to_vec()) + .bind(vec![0x02_u8]) + .bind(Felt::from(0x20_u64).to_bytes_be().to_vec()) + .execute(&mut *conn) + .await + .expect("insert erc721 ownership"); + sqlx::query( + "INSERT INTO erc1155.erc1155_balances (contract, wallet, token_id, balance, last_block) + VALUES (?1, ?2, ?3, ?4, ?5)", + ) + .bind(Felt::from(0x20_u64).to_bytes_be().to_vec()) + .bind(Felt::from(0x05_u64).to_bytes_be().to_vec()) + .bind(vec![0x01_u8]) + .bind(vec![7_u8]) + .bind("1") + .execute(&mut *conn) + .await + .expect("insert erc1155 balance"); + drop(conn); + + let response = service + .retrieve_token_balances(Request::new(RetrieveTokenBalancesRequest { + query: Some(types::TokenBalanceQuery { + account_addresses: vec![], + contract_addresses: vec![], + token_ids: vec![], + pagination: Some(types::Pagination { + cursor: String::new(), + limit: 2, + direction: PaginationDirection::Forward as i32, + order_by: vec![], + }), + }), + })) + .await + .expect("retrieve token balances") + .into_inner(); + + assert_eq!(response.balances.len(), 2); + assert_eq!( + response.balances[0].contract_address, + Felt::from(0x10_u64).to_bytes_be().to_vec() + ); + assert_eq!( + response.balances[0].account_address, + Felt::from(0x20_u64).to_bytes_be().to_vec() + ); + assert_eq!( + response.balances[0].token_id.as_ref(), + Some(&u256_bytes_from_u64(2)) + ); + + assert_eq!( + response.balances[1].contract_address, + Felt::from(0x20_u64).to_bytes_be().to_vec() + ); + assert_eq!( + response.balances[1].account_address, + Felt::from(0x05_u64).to_bytes_be().to_vec() + ); + assert_eq!( + response.balances[1].token_id.as_ref(), + Some(&u256_bytes_from_u64(1)) + ); + } + + #[tokio::test] + async fn retrieve_token_balances_matches_canonical_token_id_filters() { + let db_path = test_db_path("token-balances-token-filter"); + let erc721_url = temp_sqlite_url("erc721-token-balances-token-filter"); + let service = EcsService::new(&db_path, Some(1), None, Some(&erc721_url), None) + .await + .expect("service init"); + service + .attach_erc_databases() + .await + .expect("attach erc dbs"); + + let mut conn = service + .acquire_initialized_connection() + .await + .expect("setup connection"); + sqlx::query( + "CREATE TABLE erc721.nft_ownership ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + token BLOB NOT NULL, + token_id BLOB NOT NULL, + owner BLOB NOT NULL + )", + ) + .execute(&mut *conn) + .await + .expect("create ownership"); + sqlx::query( + "INSERT INTO erc721.nft_ownership (token, token_id, owner) + VALUES (?1, ?2, ?3)", + ) + .bind(Felt::from(0x99_u64).to_bytes_be().to_vec()) + .bind(vec![0x04_u8]) + .bind(Felt::from(0x77_u64).to_bytes_be().to_vec()) + .execute(&mut *conn) + .await + .expect("insert ownership"); + drop(conn); + + let response = service + .retrieve_token_balances(Request::new(RetrieveTokenBalancesRequest { + query: Some(types::TokenBalanceQuery { + account_addresses: vec![], + contract_addresses: vec![], + token_ids: vec![u256_bytes_from_u64(4)], + pagination: Some(types::Pagination { + cursor: String::new(), + limit: 10, + direction: PaginationDirection::Forward as i32, + order_by: vec![], + }), + }), + })) + .await + .expect("retrieve token balances") + .into_inner(); + + assert_eq!(response.balances.len(), 1); + assert_eq!( + response.balances[0].contract_address, + Felt::from(0x99_u64).to_bytes_be().to_vec() + ); + assert_eq!( + response.balances[0].account_address, + Felt::from(0x77_u64).to_bytes_be().to_vec() + ); + assert_eq!( + response.balances[0].token_id.as_ref(), + Some(&u256_bytes_from_u64(4)) + ); + } + #[tokio::test] async fn retrieve_controllers_reads_backing_table() { let db_path = test_db_path("retrieve-controllers"); @@ -6991,4 +8795,82 @@ mod tests { assert_eq!(response.controllers[0].username, "alice"); assert_eq!(response.controllers[0].deployed_at_timestamp, 1_710_936_000); } + + #[tokio::test] + async fn retrieve_controllers_respects_requested_limit_above_shared_cap() { + let db_path = test_db_path("retrieve-controllers-limit"); + let service = EcsService::new(&db_path, Some(1), None, None, None) + .await + .expect("service init"); + + sqlx::query( + "CREATE TABLE controllers ( + id TEXT PRIMARY KEY, + address TEXT NOT NULL UNIQUE, + username TEXT NOT NULL, + deployed_at TEXT NOT NULL, + updated_at BIGINT NOT NULL + )", + ) + .execute(&service.state.pool) + .await + .expect("create controllers table"); + + let mut tx = service + .state + .pool + .begin() + .await + .expect("begin controller insert tx"); + for i in 0..1_500_u64 { + let controller = format!("{:#066x}", Felt::from(i + 1)); + sqlx::query( + "INSERT INTO controllers (id, address, username, deployed_at, updated_at) + VALUES (?1, ?2, ?3, ?4, ?5)", + ) + .bind(&controller) + .bind(&controller) + .bind(format!("user-{i}")) + .bind("2024-03-20T12:00:00Z") + .bind(i as i64) + .execute(&mut *tx) + .await + .expect("insert controller"); + } + tx.commit().await.expect("commit controller inserts"); + + let response = service + .retrieve_controllers(Request::new(RetrieveControllersRequest { + query: Some(types::ControllerQuery { + contract_addresses: Vec::new(), + usernames: Vec::new(), + pagination: Some(types::Pagination { + cursor: String::new(), + limit: 1_200, + direction: PaginationDirection::Forward as i32, + order_by: Vec::new(), + }), + }), + })) + .await + .expect("retrieve controllers") + .into_inner(); + + assert_eq!(response.controllers.len(), 1_200); + assert!(!response.next_cursor.is_empty()); + assert_eq!( + response + .controllers + .first() + .map(|controller| controller.address.clone()), + Some(Felt::from(1_u64).to_bytes_be().to_vec()) + ); + assert_eq!( + response + .controllers + .last() + .map(|controller| controller.address.clone()), + Some(Felt::from(1_200_u64).to_bytes_be().to_vec()) + ); + } } diff --git a/crates/torii-ecs-sink/tests/sqlite_prepared_statements.rs b/crates/torii-ecs-sink/tests/sqlite_prepared_statements.rs new file mode 100644 index 00000000..58487346 --- /dev/null +++ b/crates/torii-ecs-sink/tests/sqlite_prepared_statements.rs @@ -0,0 +1,63 @@ +use std::fs; +use std::path::PathBuf; + +fn workspace_root() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .parent() + .expect("crate dir has parent") + .parent() + .expect("workspace root") + .to_path_buf() +} + +fn has_allowed_dynamic_marker(lines: &[&str], index: usize) -> bool { + lines[index].contains("sqlite-dynamic-ok") + || index + .checked_sub(1) + .and_then(|prev| lines.get(prev)) + .is_some_and(|line| line.contains("sqlite-dynamic-ok")) +} + +#[test] +fn runtime_sqlite_queries_avoid_uncached_prepare_and_inline_format_sql() { + let root = workspace_root(); + let files = [ + "crates/torii-ecs-sink/src/grpc_service.rs", + "crates/torii-entities-historical-sink/src/lib.rs", + "crates/introspect-sqlite-sink/src/processor.rs", + "crates/torii-erc20/src/storage.rs", + "crates/torii-erc721/src/storage.rs", + "crates/torii-erc1155/src/storage.rs", + ]; + let forbidden = ["query(&format!", ".prepare("]; + let mut violations = Vec::new(); + + for relative_path in files { + let path = root.join(relative_path); + let source = fs::read_to_string(&path).unwrap_or_else(|err| { + panic!("failed to read {}: {err}", path.display()); + }); + let lines = source.lines().collect::>(); + + for (index, line) in lines.iter().enumerate() { + for pattern in forbidden { + if !line.contains(pattern) { + continue; + } + if pattern == ".prepare(" && line.contains("prepare_cached(") { + continue; + } + if has_allowed_dynamic_marker(&lines, index) { + continue; + } + violations.push(format!("{relative_path}:{}: {}", index + 1, line.trim())); + } + } + } + + assert!( + violations.is_empty(), + "found runtime SQLite queries that bypass cached prepares or inline format SQL:\n{}", + violations.join("\n") + ); +} diff --git a/crates/torii-entities-historical-sink/Cargo.toml b/crates/torii-entities-historical-sink/Cargo.toml index 162195c2..710230ef 100644 --- a/crates/torii-entities-historical-sink/Cargo.toml +++ b/crates/torii-entities-historical-sink/Cargo.toml @@ -15,6 +15,7 @@ sqlx = { workspace = true, features = ["any", "postgres", "sqlite", "runtime-tok starknet.workspace = true tokio.workspace = true torii.workspace = true +torii-runtime-common.workspace = true tracing.workspace = true thiserror.workspace = true introspect-types.workspace = true diff --git a/crates/torii-entities-historical-sink/src/lib.rs b/crates/torii-entities-historical-sink/src/lib.rs index b6c422e5..025d2182 100644 --- a/crates/torii-entities-historical-sink/src/lib.rs +++ b/crates/torii-entities-historical-sink/src/lib.rs @@ -13,12 +13,9 @@ use torii::etl::extractor::ExtractionBatch; use torii::etl::sink::{EventBus, Sink, SinkContext, TopicInfo}; use torii_introspect::events::{CreateTable, IntrospectBody, IntrospectMsg, UpdateTable}; use torii_introspect::schema::TableSchema; +use torii_runtime_common::database::DEFAULT_SQLITE_MAX_CONNECTIONS; const INTROSPECT_TYPE: TypeId = TypeId::new("introspect"); -const SQLITE_SCHEMA_STATE_TABLE: &str = "introspect_sink_schema_state"; -const PG_METADATA_SCHEMA: &str = "introspect"; -const PG_METADATA_TABLES: &str = "db_tables"; - #[derive(Clone, Copy, Debug, PartialEq, Eq)] enum DbBackend { Sqlite, @@ -96,6 +93,14 @@ struct TrackedTable { base_name: String, history_name: String, columns: Vec, + sqlite_queries: Option, +} + +#[derive(Clone, Debug)] +struct TrackedTableSqliteQueries { + next_revision: String, + copy_current_row: String, + insert_tombstone: String, } #[derive(Clone)] @@ -123,7 +128,7 @@ impl EntitiesHistoricalSink { }; let pool = AnyPoolOptions::new() .max_connections(max_connections.unwrap_or(if backend == DbBackend::Sqlite { - 1 + DEFAULT_SQLITE_MAX_CONNECTIONS } else { 5 })) @@ -156,9 +161,11 @@ impl EntitiesHistoricalSink { match self.backend { DbBackend::Sqlite => { - let rows = sqlx::query(&format!( - "SELECT table_schema_json FROM {SQLITE_SCHEMA_STATE_TABLE} WHERE alive != 0" - )) + let rows = sqlx::query( + "SELECT table_schema_json + FROM introspect_sink_schema_state + WHERE alive != 0", + ) .fetch_all(&self.pool) .await?; for row in rows { @@ -170,9 +177,11 @@ impl EntitiesHistoricalSink { } } DbBackend::Postgres => { - let rows = sqlx::query(&format!( - "SELECT id, name FROM {PG_METADATA_SCHEMA}.{PG_METADATA_TABLES} WHERE \"schema\" = $1" - )) + let rows = sqlx::query( + "SELECT id, name + FROM introspect.db_tables + WHERE \"schema\" = $1", + ) .bind(self.namespace.postgres_schema()) .fetch_all(&self.pool) .await?; @@ -216,10 +225,12 @@ impl EntitiesHistoricalSink { let (canonical_table_id, compact_table_id) = felt_hex_variants(table_id); match self.backend { DbBackend::Sqlite => { - let row = sqlx::query(&format!( - "SELECT table_schema_json FROM {SQLITE_SCHEMA_STATE_TABLE} \ - WHERE table_id = ?1 OR table_id = ?2 LIMIT 1" - )) + let row = sqlx::query( + "SELECT table_schema_json + FROM introspect_sink_schema_state + WHERE table_id = ?1 OR table_id = ?2 + LIMIT 1", + ) .bind(canonical_table_id) .bind(compact_table_id) .fetch_optional(&self.pool) @@ -232,10 +243,12 @@ impl EntitiesHistoricalSink { .transpose() } DbBackend::Postgres => { - let row = sqlx::query(&format!( - "SELECT name FROM {PG_METADATA_SCHEMA}.{PG_METADATA_TABLES} \ - WHERE \"schema\" = $1 AND (id::text = $2 OR id::text = $3) LIMIT 1" - )) + let row = sqlx::query( + "SELECT name + FROM introspect.db_tables + WHERE \"schema\" = $1 AND (id::text = $2 OR id::text = $3) + LIMIT 1", + ) .bind(self.namespace.postgres_schema()) .bind(canonical_table_id) .bind(compact_table_id) @@ -265,10 +278,13 @@ impl EntitiesHistoricalSink { "tracked table '{logical_name}' does not expose entity_id column" )); } + let sqlite_queries = (self.backend == DbBackend::Sqlite) + .then(|| build_tracked_table_sqlite_queries(&base_name, &history_name, &columns)); let tracked = TrackedTable { logical_name: logical_name.to_string(), base_name, history_name, + sqlite_queries, columns, }; self.ensure_history_table(&tracked).await?; @@ -282,6 +298,7 @@ impl EntitiesHistoricalSink { async fn load_source_columns(&self, base_name: &str) -> Result> { match self.backend { DbBackend::Sqlite => { + // sqlite-dynamic-ok: PRAGMA table_info requires the table identifier in SQL text. let rows = sqlx::query(&format!( "PRAGMA table_info({})", quote_sqlite_identifier(base_name) @@ -374,6 +391,7 @@ impl EntitiesHistoricalSink { ) -> Result> { let rows = match self.backend { DbBackend::Sqlite => { + // sqlite-dynamic-ok: PRAGMA table_info requires the table identifier in SQL text. sqlx::query(&format!( "PRAGMA table_info({})", quote_sqlite_identifier(&tracked.history_name) @@ -557,11 +575,12 @@ impl EntitiesHistoricalSink { compact_entity_id_hex: &str, ) -> Result { let sql = match self.backend { - DbBackend::Sqlite => format!( - "SELECT COALESCE(MAX(\"revision\"), 0) + 1 AS next_revision \ - FROM {} WHERE \"entity_id\" = ?1 OR \"entity_id\" = ?2", - quote_sqlite_identifier(&tracked.history_name) - ), + DbBackend::Sqlite => tracked + .sqlite_queries + .as_ref() + .expect("sqlite queries available for sqlite backend") + .next_revision + .clone(), DbBackend::Postgres => format!( "SELECT COALESCE(MAX(\"revision\"), 0) + 1 AS next_revision \ FROM {} WHERE \"entity_id\"::text = $1 OR \"entity_id\"::text = $2", @@ -634,46 +653,35 @@ impl EntitiesHistoricalSink { executed_at: u64, deleted: bool, ) -> Result { - let source_columns = tracked - .columns - .iter() - .map(|column| quote_ident(&column.name)) - .collect::>() - .join(", "); - let insert_columns = format!( - "{source_columns}, \"revision\", \"historical_deleted\", \ - \"historical_block_number\", \"historical_tx_hash\", \"historical_executed_at\"" - ); - let history_target = match self.backend { - DbBackend::Sqlite => quote_sqlite_identifier(&tracked.history_name), - DbBackend::Postgres => { - quote_pg_qualified(self.namespace.postgres_schema(), &tracked.history_name) - } - }; - let source_target = match self.backend { - DbBackend::Sqlite => quote_sqlite_identifier(&tracked.base_name), + let sql = match self.backend { + DbBackend::Sqlite => tracked + .sqlite_queries + .as_ref() + .expect("sqlite queries available for sqlite backend") + .copy_current_row + .clone(), DbBackend::Postgres => { - quote_pg_qualified(self.namespace.postgres_schema(), &tracked.base_name) + let source_columns = tracked + .columns + .iter() + .map(|column| quote_ident(&column.name)) + .collect::>() + .join(", "); + let insert_columns = format!( + "{source_columns}, \"revision\", \"historical_deleted\", \ + \"historical_block_number\", \"historical_tx_hash\", \"historical_executed_at\"" + ); + let history_target = + quote_pg_qualified(self.namespace.postgres_schema(), &tracked.history_name); + let source_target = + quote_pg_qualified(self.namespace.postgres_schema(), &tracked.base_name); + format!( + "INSERT INTO {history_target} ({insert_columns}) \ + SELECT {source_columns}, $1, $2, $3, $4, $5 \ + FROM {source_target} WHERE (\"entity_id\"::text = $6 OR \"entity_id\"::text = $7) LIMIT 1" + ) } }; - let where_entity = match self.backend { - DbBackend::Sqlite => "(\"entity_id\" = ?6 OR \"entity_id\" = ?7)", - DbBackend::Postgres => "(\"entity_id\"::text = $6 OR \"entity_id\"::text = $7)", - }; - let placeholder = match self.backend { - DbBackend::Sqlite => "?", - DbBackend::Postgres => "$", - }; - let revision_ph = format!("{placeholder}1"); - let deleted_ph = format!("{placeholder}2"); - let block_ph = format!("{placeholder}3"); - let tx_ph = format!("{placeholder}4"); - let executed_ph = format!("{placeholder}5"); - let sql = format!( - "INSERT INTO {history_target} ({insert_columns}) \ - SELECT {source_columns}, {revision_ph}, {deleted_ph}, {block_ph}, {tx_ph}, {executed_ph} \ - FROM {source_target} WHERE {where_entity} LIMIT 1" - ); let mut query = sqlx::query(&sql).bind(revision); query = match self.backend { DbBackend::Sqlite => query.bind(i64::from(deleted)), @@ -699,48 +707,54 @@ impl EntitiesHistoricalSink { tx_hash: Felt, executed_at: u64, ) -> Result<()> { - let mut columns = Vec::with_capacity(tracked.columns.len() + 5); - let mut values_sql = Vec::with_capacity(tracked.columns.len() + 5); + let sql = match self.backend { + DbBackend::Sqlite => tracked + .sqlite_queries + .as_ref() + .expect("sqlite queries available for sqlite backend") + .insert_tombstone + .clone(), + DbBackend::Postgres => { + let mut columns = Vec::with_capacity(tracked.columns.len() + 5); + let mut values_sql = Vec::with_capacity(tracked.columns.len() + 5); + + let mut bind_index = 1_usize; + for column in &tracked.columns { + columns.push(quote_ident(&column.name)); + if column.name == "entity_id" { + values_sql.push(self.entity_cast_placeholder(bind_index, &column.type_sql)); + bind_index += 1; + } else { + values_sql.push("NULL".to_string()); + } + } - let mut bind_index = 1_usize; - for column in &tracked.columns { - columns.push(quote_ident(&column.name)); - if column.name == "entity_id" { - values_sql.push(self.entity_cast_placeholder(bind_index, &column.type_sql)); + columns.extend([ + "\"revision\"".to_string(), + "\"historical_deleted\"".to_string(), + "\"historical_block_number\"".to_string(), + "\"historical_tx_hash\"".to_string(), + "\"historical_executed_at\"".to_string(), + ]); + values_sql.push(self.value_placeholder(bind_index)); bind_index += 1; - } else { - values_sql.push("NULL".to_string()); - } - } + values_sql.push(self.value_placeholder(bind_index)); + bind_index += 1; + values_sql.push(self.value_placeholder(bind_index)); + bind_index += 1; + values_sql.push(self.value_placeholder(bind_index)); + bind_index += 1; + values_sql.push(self.value_placeholder(bind_index)); - columns.extend([ - "\"revision\"".to_string(), - "\"historical_deleted\"".to_string(), - "\"historical_block_number\"".to_string(), - "\"historical_tx_hash\"".to_string(), - "\"historical_executed_at\"".to_string(), - ]); - values_sql.push(self.value_placeholder(bind_index)); - bind_index += 1; - values_sql.push(self.value_placeholder(bind_index)); - bind_index += 1; - values_sql.push(self.value_placeholder(bind_index)); - bind_index += 1; - values_sql.push(self.value_placeholder(bind_index)); - bind_index += 1; - values_sql.push(self.value_placeholder(bind_index)); - - let target = match self.backend { - DbBackend::Sqlite => quote_sqlite_identifier(&tracked.history_name), - DbBackend::Postgres => { - quote_pg_qualified(self.namespace.postgres_schema(), &tracked.history_name) + let target = + quote_pg_qualified(self.namespace.postgres_schema(), &tracked.history_name); + format!( + "INSERT INTO {target} ({}) VALUES ({})", + columns.join(", "), + values_sql.join(", ") + ) } }; - let sql = format!( - "INSERT INTO {target} ({}) VALUES ({})", - columns.join(", "), - values_sql.join(", ") - ); let mut query = sqlx::query(&sql).bind(entity_id_hex).bind(revision); query = match self.backend { @@ -919,6 +933,68 @@ fn quote_pg_qualified(schema: &str, table: &str) -> String { format!("{}.{}", quote_ident(schema), quote_ident(table)) } +fn build_tracked_table_sqlite_queries( + base_name: &str, + history_name: &str, + columns: &[HistoryColumn], +) -> TrackedTableSqliteQueries { + let history_target = quote_sqlite_identifier(history_name); + let source_target = quote_sqlite_identifier(base_name); + let source_columns = columns + .iter() + .map(|column| quote_ident(&column.name)) + .collect::>() + .join(", "); + let insert_columns = format!( + "{source_columns}, \"revision\", \"historical_deleted\", \ + \"historical_block_number\", \"historical_tx_hash\", \"historical_executed_at\"" + ); + + let mut tombstone_columns = Vec::with_capacity(columns.len() + 5); + let mut tombstone_values = Vec::with_capacity(columns.len() + 5); + for column in columns { + tombstone_columns.push(quote_ident(&column.name)); + if column.name == "entity_id" { + tombstone_values.push("?".to_string()); + } else { + tombstone_values.push("NULL".to_string()); + } + } + tombstone_columns.extend([ + "\"revision\"".to_string(), + "\"historical_deleted\"".to_string(), + "\"historical_block_number\"".to_string(), + "\"historical_tx_hash\"".to_string(), + "\"historical_executed_at\"".to_string(), + ]); + tombstone_values.extend([ + "?".to_string(), + "?".to_string(), + "?".to_string(), + "?".to_string(), + "?".to_string(), + ]); + + TrackedTableSqliteQueries { + next_revision: format!( + "SELECT COALESCE(MAX(\"revision\"), 0) + 1 AS next_revision \ + FROM {history_target} WHERE \"entity_id\" = ?1 OR \"entity_id\" = ?2" + ), + copy_current_row: format!( + "INSERT INTO {history_target} ({insert_columns}) \ + SELECT {source_columns}, ?1, ?2, ?3, ?4, ?5 \ + FROM {source_target} \ + WHERE (\"entity_id\" = ?6 OR \"entity_id\" = ?7) \ + LIMIT 1" + ), + insert_tombstone: format!( + "INSERT INTO {history_target} ({}) VALUES ({})", + tombstone_columns.join(", "), + tombstone_values.join(", ") + ), + } +} + fn felt_hex(value: Felt) -> String { format!("{value:#x}") } diff --git a/crates/torii-erc1155/src/identification.rs b/crates/torii-erc1155/src/identification.rs index bc7dfe26..80a8b44d 100644 --- a/crates/torii-erc1155/src/identification.rs +++ b/crates/torii-erc1155/src/identification.rs @@ -64,7 +64,7 @@ impl IdentificationRule for Erc1155Rule { // ERC1155 must have balance_of_batch (which ERC20/ERC721 don't have) // and at least one of the transfer events if has_balance_of_batch && (has_transfer_single || has_transfer_batch) { - tracing::trace!( + tracing::debug!( target: "torii_erc1155::identification", "Contract matches ERC1155 pattern" ); diff --git a/crates/torii-erc1155/src/storage.rs b/crates/torii-erc1155/src/storage.rs index bc6e4f22..1886977b 100644 --- a/crates/torii-erc1155/src/storage.rs +++ b/crates/torii-erc1155/src/storage.rs @@ -509,6 +509,18 @@ impl Erc1155Storage { [], )?; + conn.execute( + "CREATE INDEX IF NOT EXISTS idx_erc1155_balances_contract_wallet_token_id_ord + ON erc1155_balances(contract, wallet, length(token_id), token_id)", + [], + )?; + + conn.execute( + "CREATE INDEX IF NOT EXISTS idx_erc1155_balances_wallet_contract_token_id_ord + ON erc1155_balances(wallet, contract, length(token_id), token_id)", + [], + )?; + // Balance adjustments table for audit trail conn.execute( "CREATE TABLE IF NOT EXISTS erc1155_balance_adjustments ( @@ -835,7 +847,7 @@ impl Erc1155Storage { query.push_str(" ORDER BY t.block_number DESC, t.id DESC LIMIT ?"); params_vec.push(Box::new(limit as i64)); - let mut stmt = conn.prepare(&query)?; + let mut stmt = conn.prepare_cached(&query)?; let params_refs: Vec<&dyn rusqlite::ToSql> = params_vec.iter().map(std::convert::AsRef::as_ref).collect(); @@ -1453,7 +1465,7 @@ impl Erc1155Storage { let query = format!("SELECT token FROM token_metadata WHERE token IN ({placeholders})"); let token_blobs: Vec> = chunk.iter().map(|token| felt_to_blob(*token)).collect(); - let mut stmt = conn.prepare(&query)?; + let mut stmt = conn.prepare_cached(&query)?; let rows = stmt.query_map(params_from_iter(token_blobs.iter()), |row| { let token_bytes: Vec = row.get(0)?; Ok(blob_to_felt(&token_bytes)) @@ -1529,7 +1541,7 @@ impl Erc1155Storage { } let conn = self.conn.lock().unwrap(); let mut stmt = - conn.prepare("SELECT token, name, symbol, total_supply FROM token_metadata")?; + conn.prepare_cached("SELECT token, name, symbol, total_supply FROM token_metadata")?; let rows = stmt.query_map([], |row| { let token_bytes: Vec = row.get(0)?; let name: Option = row.get(1)?; @@ -1564,7 +1576,7 @@ impl Erc1155Storage { let mut out = if let Some(cursor_token) = cursor { let cursor_blob = felt_to_blob(cursor_token); - let mut stmt = conn.prepare( + let mut stmt = conn.prepare_cached( "SELECT token, name, symbol, total_supply FROM token_metadata WHERE token > ?1 @@ -1585,7 +1597,7 @@ impl Erc1155Storage { })?; rows.collect::, _>>()? } else { - let mut stmt = conn.prepare( + let mut stmt = conn.prepare_cached( "SELECT token, name, symbol, total_supply FROM token_metadata ORDER BY token ASC @@ -1658,7 +1670,7 @@ impl Erc1155Storage { } let params_refs: Vec<&dyn ToSql> = params_vec.iter().map(std::convert::AsRef::as_ref).collect(); - let mut stmt = conn.prepare(&query)?; + let mut stmt = conn.prepare_cached(&query)?; let rows = stmt.query_map(params_refs.as_slice(), |row| { let token_bytes: Vec = row.get(0)?; let token_id_bytes: Vec = row.get(1)?; @@ -1683,7 +1695,7 @@ impl Erc1155Storage { } let conn = self.conn.lock().unwrap(); let token_blob = felt_to_blob(token); - let mut stmt = conn.prepare( + let mut stmt = conn.prepare_cached( "SELECT token_id, uri, metadata_json FROM token_uris WHERE token = ?1 @@ -1727,7 +1739,7 @@ impl Erc1155Storage { } let params_refs: Vec<&dyn ToSql> = params_vec.iter().map(std::convert::AsRef::as_ref).collect(); - let mut stmt = conn.prepare(&query)?; + let mut stmt = conn.prepare_cached(&query)?; let rows = stmt.query_map(params_refs.as_slice(), |row| { let token_id_bytes: Vec = row.get(0)?; let uri: Option = row.get(1)?; @@ -1817,7 +1829,7 @@ impl Erc1155Storage { } let params_refs: Vec<&dyn ToSql> = params_vec.iter().map(std::convert::AsRef::as_ref).collect(); - let mut stmt = conn.prepare(&query)?; + let mut stmt = conn.prepare_cached(&query)?; let rows = stmt.query_map(params_refs.as_slice(), |row| row.get::<_, i64>(0))?; let value_ids = rows.collect::, _>>()?; if value_ids.is_empty() { @@ -1899,7 +1911,7 @@ impl Erc1155Storage { .iter() .map(std::convert::AsRef::as_ref) .collect(); - let mut stmt = conn.prepare(&page_query)?; + let mut stmt = conn.prepare_cached(&page_query)?; let rows = stmt.query_map(page_refs.as_slice(), |row| row.get::<_, Vec>(0))?; let mut token_ids = rows .collect::, _>>()? @@ -1948,7 +1960,7 @@ impl Erc1155Storage { .iter() .map(std::convert::AsRef::as_ref) .collect(); - let mut stmt = conn.prepare(&facet_query)?; + let mut stmt = conn.prepare_cached(&facet_query)?; let rows = stmt.query_map(facet_refs.as_slice(), |row| { Ok(AttributeFacetCount { key: row.get(0)?, @@ -3191,7 +3203,7 @@ fn sqlite_sync_facets_for_token( let token_id_blob = u256_to_blob(token_id); let existing_rows = { - let mut stmt = tx.prepare( + let mut stmt = tx.prepare_cached( "SELECT facet_value_id FROM facet_token_map WHERE token = ?1 AND token_id = ?2", )?; let rows = stmt.query_map(params![&token_blob, &token_id_blob], |row| { @@ -3360,7 +3372,7 @@ fn sqlite_token_uri_state_matches( return Ok(false); } - let mut stmt = conn.prepare( + let mut stmt = conn.prepare_cached( "SELECT key, value FROM token_attributes WHERE token = ?1 AND token_id = ?2 ORDER BY key ASC, value ASC", diff --git a/crates/torii-erc20/src/identification.rs b/crates/torii-erc20/src/identification.rs index b31b013d..2868ba3c 100644 --- a/crates/torii-erc20/src/identification.rs +++ b/crates/torii-erc20/src/identification.rs @@ -64,7 +64,7 @@ impl IdentificationRule for Erc20Rule { let has_transfer_event = abi.has_event("Transfer"); if has_transfer && has_balance_of && has_transfer_event { - tracing::trace!( + tracing::debug!( target: "torii_erc20::identification", "Contract matches ERC20 pattern" ); diff --git a/crates/torii-erc20/src/storage.rs b/crates/torii-erc20/src/storage.rs index 2810b3c2..82b0296e 100644 --- a/crates/torii-erc20/src/storage.rs +++ b/crates/torii-erc20/src/storage.rs @@ -780,7 +780,7 @@ impl Erc20Storage { params.push(wallet as &dyn ToSql); } - let mut stmt = conn.prepare(&sql)?; + let mut stmt = conn.prepare_cached(&sql)?; let rows = stmt.query_map(params_from_iter(params), |row| { let wallet: Vec = row.get(0)?; let balance: Vec = row.get(1)?; @@ -1229,7 +1229,7 @@ impl Erc20Storage { query.push_str(" ORDER BY t.block_number DESC, t.id DESC LIMIT ?"); params_vec.push(Box::new(limit as i64)); - let mut stmt = conn.prepare(&query)?; + let mut stmt = conn.prepare_cached(&query)?; let params_refs: Vec<&dyn rusqlite::ToSql> = params_vec.iter().map(std::convert::AsRef::as_ref).collect(); @@ -1371,7 +1371,7 @@ impl Erc20Storage { query.push_str(" ORDER BY a.block_number DESC, a.id DESC LIMIT ?"); params_vec.push(Box::new(limit as i64)); - let mut stmt = conn.prepare(&query)?; + let mut stmt = conn.prepare_cached(&query)?; let params_refs: Vec<&dyn rusqlite::ToSql> = params_vec.iter().map(std::convert::AsRef::as_ref).collect(); @@ -1551,7 +1551,7 @@ impl Erc20Storage { query.push_str(" ORDER BY id ASC LIMIT ?"); params_vec.push(Box::new(limit as i64)); - let mut stmt = conn.prepare(&query)?; + let mut stmt = conn.prepare_cached(&query)?; let params_refs: Vec<&dyn rusqlite::ToSql> = params_vec.iter().map(std::convert::AsRef::as_ref).collect(); @@ -1977,7 +1977,8 @@ impl Erc20Storage { let conn = self.conn.lock().unwrap(); let mut existing = HashSet::with_capacity(tokens.len()); - let mut stmt = conn.prepare("SELECT 1 FROM token_metadata WHERE token = ? LIMIT 1")?; + let mut stmt = + conn.prepare_cached("SELECT 1 FROM token_metadata WHERE token = ? LIMIT 1")?; for token in tokens { let mut rows = stmt.query(params![felt_to_blob(*token)])?; if rows.next()?.is_some() { @@ -2076,8 +2077,9 @@ impl Erc20Storage { )>, > { let conn = self.conn.lock().unwrap(); - let mut stmt = - conn.prepare("SELECT token, name, symbol, decimals, total_supply FROM token_metadata")?; + let mut stmt = conn.prepare_cached( + "SELECT token, name, symbol, decimals, total_supply FROM token_metadata", + )?; let rows = stmt.query_map([], |row| { let token_bytes: Vec = row.get(0)?; let name: Option = row.get(1)?; @@ -2120,7 +2122,7 @@ impl Erc20Storage { let mut out = if let Some(cursor_token) = cursor { let cursor_blob = felt_to_blob(cursor_token); - let mut stmt = conn.prepare( + let mut stmt = conn.prepare_cached( "SELECT token, name, symbol, decimals, total_supply FROM token_metadata WHERE token > ?1 @@ -2143,7 +2145,7 @@ impl Erc20Storage { })?; rows.collect::, _>>()? } else { - let mut stmt = conn.prepare( + let mut stmt = conn.prepare_cached( "SELECT token, name, symbol, decimals, total_supply FROM token_metadata ORDER BY token ASC diff --git a/crates/torii-erc721/src/identification.rs b/crates/torii-erc721/src/identification.rs index 7bda5908..13a7b22a 100644 --- a/crates/torii-erc721/src/identification.rs +++ b/crates/torii-erc721/src/identification.rs @@ -67,7 +67,7 @@ impl IdentificationRule for Erc721Rule { // ERC721 must have owner_of (which ERC20 doesn't have) if has_owner_of && has_balance_of && has_transfer_event { - tracing::trace!( + tracing::debug!( target: "torii_erc721::identification", "Contract matches ERC721 pattern" ); diff --git a/crates/torii-erc721/src/storage.rs b/crates/torii-erc721/src/storage.rs index 0c7bf958..658043fa 100644 --- a/crates/torii-erc721/src/storage.rs +++ b/crates/torii-erc721/src/storage.rs @@ -300,6 +300,18 @@ impl Erc721Storage { [], )?; + conn.execute( + "CREATE INDEX IF NOT EXISTS idx_nft_ownership_token_owner_token_id_ord + ON nft_ownership(token, owner, length(token_id), token_id)", + [], + )?; + + conn.execute( + "CREATE INDEX IF NOT EXISTS idx_nft_ownership_owner_token_token_id_ord + ON nft_ownership(owner, token, length(token_id), token_id)", + [], + )?; + // Transfer history conn.execute( "CREATE TABLE IF NOT EXISTS nft_transfers ( @@ -727,7 +739,7 @@ impl Erc721Storage { query.push_str(" ORDER BY t.block_number DESC, t.id DESC LIMIT ?"); params_vec.push(Box::new(limit as i64)); - let mut stmt = conn.prepare(&query)?; + let mut stmt = conn.prepare_cached(&query)?; let params_refs: Vec<&dyn rusqlite::ToSql> = params_vec.iter().map(std::convert::AsRef::as_ref).collect(); @@ -825,7 +837,7 @@ impl Erc721Storage { query.push_str(" ORDER BY block_number DESC, id DESC LIMIT ?"); params_vec.push(Box::new(limit as i64)); - let mut stmt = conn.prepare(&query)?; + let mut stmt = conn.prepare_cached(&query)?; let params_refs: Vec<&dyn rusqlite::ToSql> = params_vec.iter().map(std::convert::AsRef::as_ref).collect(); @@ -1020,7 +1032,7 @@ impl Erc721Storage { ); let token_blobs: Vec> = chunk.iter().map(|token| felt_to_blob(*token)).collect(); - let mut stmt = conn.prepare(&query)?; + let mut stmt = conn.prepare_cached(&query)?; let rows = stmt.query_map(params_from_iter(token_blobs.iter()), |row| { let token_bytes: Vec = row.get(0)?; Ok(blob_to_felt(&token_bytes)) @@ -1093,7 +1105,7 @@ impl Erc721Storage { ) -> Result, Option, Option)>> { let conn = self.conn.lock().unwrap(); let mut stmt = - conn.prepare("SELECT token, name, symbol, total_supply FROM token_metadata")?; + conn.prepare_cached("SELECT token, name, symbol, total_supply FROM token_metadata")?; let rows = stmt.query_map([], |row| { let token_bytes: Vec = row.get(0)?; let name: Option = row.get(1)?; @@ -1128,7 +1140,7 @@ impl Erc721Storage { let mut out = if let Some(cursor_token) = cursor { let cursor_blob = felt_to_blob(cursor_token); - let mut stmt = conn.prepare( + let mut stmt = conn.prepare_cached( "SELECT token, name, symbol, total_supply FROM token_metadata WHERE token > ?1 @@ -1149,7 +1161,7 @@ impl Erc721Storage { })?; rows.collect::, _>>()? } else { - let mut stmt = conn.prepare( + let mut stmt = conn.prepare_cached( "SELECT token, name, symbol, total_supply FROM token_metadata ORDER BY token ASC @@ -1224,7 +1236,7 @@ impl Erc721Storage { } let params_refs: Vec<&dyn ToSql> = params_vec.iter().map(std::convert::AsRef::as_ref).collect(); - let mut stmt = conn.prepare(&query)?; + let mut stmt = conn.prepare_cached(&query)?; let rows = stmt.query_map(params_refs.as_slice(), |row| { let token_bytes: Vec = row.get(0)?; let token_id_bytes: Vec = row.get(1)?; @@ -1249,7 +1261,7 @@ impl Erc721Storage { } let conn = self.conn.lock().unwrap(); let token_blob = felt_to_blob(token); - let mut stmt = conn.prepare( + let mut stmt = conn.prepare_cached( "SELECT token_id, uri, metadata_json FROM token_uris WHERE token = ?1 @@ -1293,7 +1305,7 @@ impl Erc721Storage { } let params_refs: Vec<&dyn ToSql> = params_vec.iter().map(std::convert::AsRef::as_ref).collect(); - let mut stmt = conn.prepare(&query)?; + let mut stmt = conn.prepare_cached(&query)?; let rows = stmt.query_map(params_refs.as_slice(), |row| { let token_id_bytes: Vec = row.get(0)?; let uri: Option = row.get(1)?; @@ -1383,7 +1395,7 @@ impl Erc721Storage { } let params_refs: Vec<&dyn ToSql> = params_vec.iter().map(std::convert::AsRef::as_ref).collect(); - let mut stmt = conn.prepare(&query)?; + let mut stmt = conn.prepare_cached(&query)?; let rows = stmt.query_map(params_refs.as_slice(), |row| row.get::<_, i64>(0))?; let value_ids = rows.collect::, _>>()?; if value_ids.is_empty() { @@ -1465,7 +1477,7 @@ impl Erc721Storage { .iter() .map(std::convert::AsRef::as_ref) .collect(); - let mut stmt = conn.prepare(&page_query)?; + let mut stmt = conn.prepare_cached(&page_query)?; let rows = stmt.query_map(page_refs.as_slice(), |row| row.get::<_, Vec>(0))?; let mut token_ids = rows .collect::, _>>()? @@ -1514,7 +1526,7 @@ impl Erc721Storage { .iter() .map(std::convert::AsRef::as_ref) .collect(); - let mut stmt = conn.prepare(&facet_query)?; + let mut stmt = conn.prepare_cached(&facet_query)?; let rows = stmt.query_map(facet_refs.as_slice(), |row| { Ok(AttributeFacetCount { key: row.get(0)?, @@ -2627,7 +2639,7 @@ fn sqlite_load_existing_facet_assignments( params.push(&entry.token_id_blob); } - let mut stmt = tx.prepare(&query)?; + let mut stmt = tx.prepare_cached(&query)?; let rows = stmt.query_map(params.as_slice(), |row| { Ok(( row.get::>(0)?, @@ -2954,7 +2966,7 @@ fn sqlite_token_uri_state_matches( return Ok(false); } - let mut stmt = conn.prepare( + let mut stmt = conn.prepare_cached( "SELECT key, value FROM token_attributes WHERE token = ?1 AND token_id = ?2 ORDER BY key ASC, value ASC", diff --git a/crates/torii-runtime-common/src/database.rs b/crates/torii-runtime-common/src/database.rs index cc1df161..ef7aeec0 100644 --- a/crates/torii-runtime-common/src/database.rs +++ b/crates/torii-runtime-common/src/database.rs @@ -1,6 +1,8 @@ use anyhow::{bail, Result}; use std::path::{Path, PathBuf}; +pub const DEFAULT_SQLITE_MAX_CONNECTIONS: u32 = 500; + #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum DatabaseBackend { Postgres, diff --git a/crates/torii-runtime-common/src/lib.rs b/crates/torii-runtime-common/src/lib.rs index 1e423133..eeb5a7bd 100644 --- a/crates/torii-runtime-common/src/lib.rs +++ b/crates/torii-runtime-common/src/lib.rs @@ -2,3 +2,4 @@ pub mod database; pub mod sink; +pub mod token_support; diff --git a/crates/torii-runtime-common/src/token_support.rs b/crates/torii-runtime-common/src/token_support.rs new file mode 100644 index 00000000..2a248d3c --- /dev/null +++ b/crates/torii-runtime-common/src/token_support.rs @@ -0,0 +1,70 @@ +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct InstalledTokenSupport { + pub erc20: bool, + pub erc721: bool, + pub erc1155: bool, +} + +impl InstalledTokenSupport { + pub const fn any(self) -> bool { + self.erc20 || self.erc721 || self.erc1155 + } +} + +pub const fn resolve_installed_token_support( + index_external_contracts: bool, + explicit_targets: InstalledTokenSupport, +) -> InstalledTokenSupport { + InstalledTokenSupport { + erc20: index_external_contracts || explicit_targets.erc20, + erc721: index_external_contracts || explicit_targets.erc721, + erc1155: index_external_contracts || explicit_targets.erc1155, + } +} + +#[cfg(test)] +mod tests { + use super::{resolve_installed_token_support, InstalledTokenSupport}; + + #[test] + fn installs_all_token_support_when_external_indexing_is_enabled() { + let support = resolve_installed_token_support(true, InstalledTokenSupport::default()); + assert_eq!( + support, + InstalledTokenSupport { + erc20: true, + erc721: true, + erc1155: true, + } + ); + assert!(support.any()); + } + + #[test] + fn installs_no_token_support_without_targets_or_external_indexing() { + let support = resolve_installed_token_support(false, InstalledTokenSupport::default()); + assert_eq!(support, InstalledTokenSupport::default()); + assert!(!support.any()); + } + + #[test] + fn installs_only_explicit_target_types_when_external_indexing_is_disabled() { + let support = resolve_installed_token_support( + false, + InstalledTokenSupport { + erc20: true, + erc721: false, + erc1155: true, + }, + ); + assert_eq!( + support, + InstalledTokenSupport { + erc20: true, + erc721: false, + erc1155: true, + } + ); + assert!(support.any()); + } +} diff --git a/scripts/test-ecs-subscriptions-smoke.sh b/scripts/test-ecs-subscriptions-smoke.sh index 3ceed1c5..5bcfefdf 100755 --- a/scripts/test-ecs-subscriptions-smoke.sh +++ b/scripts/test-ecs-subscriptions-smoke.sh @@ -44,7 +44,7 @@ require_cmd rg require_cmd mktemp list_out="$(mktemp)" -trap 'rm -f "$list_out" "$entities_out" "$entities_err" "$messages_out" "$messages_err" "$events_out" "$events_err" "$contracts_out" "$contracts_err"' EXIT +trap 'rm -f "$list_out" "$entities_out" "$entities_err" "$messages_out" "$messages_err" "$events_out" "$events_err" "$contracts_out" "$contracts_err" "$token_balances_query_out" "$token_balances_query_err" "$token_balances_out" "$token_balances_err"' EXIT grpcurl -plaintext "$GRPC_ADDR" list >"$list_out" rg -F -q 'world.World' "$list_out" || fail 'world.World service not exposed' @@ -53,6 +53,8 @@ entities_out="$(mktemp)"; entities_err="$(mktemp)" messages_out="$(mktemp)"; messages_err="$(mktemp)" events_out="$(mktemp)"; events_err="$(mktemp)" contracts_out="$(mktemp)"; contracts_err="$(mktemp)" +token_balances_query_out="$(mktemp)"; token_balances_query_err="$(mktemp)" +token_balances_out="$(mktemp)"; token_balances_err="$(mktemp)" log "gRPC target: $GRPC_ADDR" log "world filter: $WORLD_ADDRESS_B64" @@ -89,4 +91,44 @@ else log 'SubscribeContracts: no contract frame during window (inconclusive)' fi +set +e +grpcurl -plaintext -d '{"query":{"pagination":{"limit":1,"direction":"FORWARD"}}}' \ + "$GRPC_ADDR" world.World/RetrieveTokenBalances >"$token_balances_query_out" 2>"$token_balances_query_err" +token_balances_status=$? +set -e + +if [[ $token_balances_status -ne 0 ]]; then + fail "RetrieveTokenBalances failed: $(<"$token_balances_query_err")" +fi + +if rg -q '"balances": \[\]' "$token_balances_query_out"; then + log 'SubscribeTokenBalances: no balances available during window (inconclusive)' +else + token_balance_account="$( + sed -nE 's/.*"accountAddress": "([^"]+)".*/\1/p' "$token_balances_query_out" | head -n 1 + )" + token_balance_contract="$( + sed -nE 's/.*"contractAddress": "([^"]+)".*/\1/p' "$token_balances_query_out" | head -n 1 + )" + token_balance_id="$( + sed -nE 's/.*"tokenId": "([^"]+)".*/\1/p' "$token_balances_query_out" | head -n 1 + )" + + [[ -n "$token_balance_account" ]] || fail 'RetrieveTokenBalances returned a row without accountAddress' + [[ -n "$token_balance_contract" ]] || fail 'RetrieveTokenBalances returned a row without contractAddress' + + token_balances_payload="{\"account_addresses\":[\"$token_balance_account\"],\"contract_addresses\":[\"$token_balance_contract\"]" + if [[ -n "$token_balance_id" ]]; then + token_balances_payload="${token_balances_payload},\"token_ids\":[\"$token_balance_id\"]" + fi + token_balances_payload="${token_balances_payload}}" + + stream_call "world.World/SubscribeTokenBalances" "$token_balances_payload" "$token_balances_out" "$token_balances_err" + rg -q '"subscriptionId":' "$token_balances_out" || fail 'SubscribeTokenBalances missing subscriptionId frame' + if [[ "$REQUIRE_LIVE_UPDATES" == "1" ]]; then + rg -q '"balance": \{' "$token_balances_out" || fail 'SubscribeTokenBalances did not receive live update' + fi + log 'SubscribeTokenBalances setup frame: OK' +fi + log 'subscription smoke checks passed' diff --git a/src/etl/extractor/event.rs b/src/etl/extractor/event.rs index 48e7e8c3..0b6b4856 100644 --- a/src/etl/extractor/event.rs +++ b/src/etl/extractor/event.rs @@ -445,7 +445,7 @@ impl EventExtractor { .map(|state| { let range_end = state.range_end(self.config.block_batch_size, self.chain_head); if state.continuation_token.is_none() { - tracing::info!( + tracing::trace!( target: "torii::etl::event", contract = %state.state_key(), from_block = state.current_block, @@ -585,7 +585,7 @@ impl Extractor for EventExtractor { .map(|(_, r)| r) .collect(); - tracing::info!( + tracing::trace!( target: "torii::etl::event", contracts = requests.len(), chain_head = ?self.chain_head, @@ -649,7 +649,7 @@ impl Extractor for EventExtractor { state.advance_block_range(self.config.block_batch_size, self.chain_head); any_advanced = true; - tracing::info!( + tracing::trace!( target: "torii::etl::event", contract = %state.state_key(), range_from = completed_from, @@ -685,7 +685,7 @@ impl Extractor for EventExtractor { return Ok(ExtractionBatch::empty()); } - tracing::info!( + tracing::trace!( target: "torii::etl::event", events = all_events.len(), "Extracted events from batch" @@ -709,7 +709,7 @@ impl Extractor for EventExtractor { let filtered_events = Self::filter_events_by_tx_hashes(all_events, &successful_transaction_hashes); - tracing::info!( + tracing::trace!( target: "torii::etl::event", events_before_filter, events_after_filter = filtered_events.len(), diff --git a/src/lib.rs b/src/lib.rs index f75e62f6..5173bdaa 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -26,14 +26,17 @@ pub use tonic; pub use grpc::UpdateType; use axum::Router as AxumRouter; +use std::fs::File; +use std::io::{self, BufReader}; use std::net::SocketAddr; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::Arc; use std::time::Duration; use tokio_util::sync::CancellationToken; use tonic::codec::CompressionEncoding; use tonic::transport::Server; +use tower::Service; use tower_http::cors::{Any as CorsAny, CorsLayer}; use command::{CommandBus, CommandHandler}; @@ -189,6 +192,9 @@ pub struct ToriiConfig { /// Command bus queue size. pub command_bus_queue_size: usize, + + /// Optional TLS listener configuration. + pub tls: Option, } impl ToriiConfig { @@ -197,6 +203,35 @@ impl ToriiConfig { } } +#[derive(Debug, Clone)] +pub struct ToriiTlsConfig { + pub cert_path: PathBuf, + pub key_path: PathBuf, + pub alpn_protocols: Vec>, +} + +impl ToriiTlsConfig { + pub fn new(cert_path: impl Into, key_path: impl Into) -> Self { + Self { + cert_path: cert_path.into(), + key_path: key_path.into(), + alpn_protocols: vec![b"h2".to_vec(), b"http/1.1".to_vec()], + } + } + + pub fn with_alpn_protocols(mut self, alpn_protocols: Vec>) -> Self { + self.alpn_protocols = alpn_protocols; + self + } + + fn alpn_names(&self) -> Vec { + self.alpn_protocols + .iter() + .map(|protocol| String::from_utf8_lossy(protocol).into_owned()) + .collect() + } +} + /// Builder for ToriiConfig. #[derive(Default)] pub struct ToriiConfigBuilder { @@ -226,6 +261,7 @@ pub struct ToriiConfigBuilder { etl_concurrency: Option, command_handlers: Vec>, command_bus_queue_size: Option, + tls: Option, } impl ToriiConfigBuilder { @@ -540,6 +576,11 @@ impl ToriiConfigBuilder { self } + pub fn with_tls(mut self, tls: ToriiTlsConfig) -> Self { + self.tls = Some(tls); + self + } + /// Builds the Torii configuration. /// /// # Panics @@ -575,6 +616,7 @@ impl ToriiConfigBuilder { etl_concurrency: self.etl_concurrency.unwrap_or_default(), command_handlers: self.command_handlers, command_bus_queue_size: self.command_bus_queue_size.unwrap_or(4096), + tls: self.tls, } } } @@ -741,7 +783,17 @@ pub async fn run(config: ToriiConfig) -> Result<(), Box> .layer(cors); let addr: SocketAddr = format!("{}:{}", config.host, config.port).parse()?; + let tls_acceptor = config.tls.as_ref().map(build_tls_acceptor).transpose()?; tracing::info!(target: "torii::main", "Server listening on {}", addr); + if let Some(tls) = &config.tls { + tracing::info!( + target: "torii::main", + cert = %tls.cert_path.display(), + key = %tls.key_path.display(), + alpn = ?tls.alpn_names(), + "TLS enabled for listener" + ); + } tracing::info!(target: "torii::main", "gRPC Services:"); tracing::info!(target: "torii::main", " torii.Torii - Core service"); @@ -1154,7 +1206,62 @@ pub async fn run(config: ToriiConfig) -> Result<(), Box> }; let listener = tokio::net::TcpListener::bind(addr).await?; - let server = axum::serve(listener, app).with_graceful_shutdown(shutdown_signal); + let mut make_svc = app.into_make_service(); + let http = hyper_util::server::conn::auto::Builder::new(hyper_util::rt::TokioExecutor::new()); + + let server = async move { + tokio::pin!(shutdown_signal); + loop { + let (tcp, _remote_addr) = tokio::select! { + result = listener.accept() => match result { + Ok(conn) => conn, + Err(e) => { + tracing::error!(target: "torii::main", "Accept error: {}", e); + continue; + } + }, + () = &mut shutdown_signal => break, + }; + let tower_service = make_svc.call(()).await.expect("infallible"); + let hyper_service = hyper_util::service::TowerToHyperService::new(tower_service); + let builder = http.clone(); + let tls_acceptor = tls_acceptor.clone(); + tokio::spawn(async move { + if let Some(tls_acceptor) = tls_acceptor { + let tls_stream = match tls_acceptor.accept(tcp).await { + Ok(stream) => stream, + Err(err) => { + tracing::debug!( + target: "torii::main", + error = %err, + "TLS handshake failed" + ); + return; + } + }; + + if let Err(err) = builder + .serve_connection_with_upgrades( + hyper_util::rt::TokioIo::new(tls_stream), + hyper_service, + ) + .await + { + tracing::debug!(target: "torii::main", error = %err, "Connection ended"); + } + } else if let Err(err) = builder + .serve_connection_with_upgrades( + hyper_util::rt::TokioIo::new(tcp), + hyper_service, + ) + .await + { + tracing::debug!(target: "torii::main", error = %err, "Connection ended"); + } + }); + } + Ok::<_, std::io::Error>(()) + }; // Give active connections 15 seconds to close gracefully, then force shutdown. // This prevents hanging on long-lived gRPC streaming connections. @@ -1202,3 +1309,60 @@ pub async fn run(config: ToriiConfig) -> Result<(), Box> Ok(()) } + +fn build_tls_acceptor( + config: &ToriiTlsConfig, +) -> Result> { + ensure_rustls_crypto_provider(); + let cert_chain = load_cert_chain(&config.cert_path)?; + let private_key = load_private_key(&config.key_path)?; + + let mut server_config = tokio_rustls::rustls::ServerConfig::builder() + .with_no_client_auth() + .with_single_cert(cert_chain, private_key)?; + server_config + .alpn_protocols + .clone_from(&config.alpn_protocols); + + Ok(tokio_rustls::TlsAcceptor::from(Arc::new(server_config))) +} + +fn ensure_rustls_crypto_provider() { + if tokio_rustls::rustls::crypto::CryptoProvider::get_default().is_none() { + let _ = tokio_rustls::rustls::crypto::ring::default_provider().install_default(); + } +} + +fn load_cert_chain( + path: &Path, +) -> Result>, Box> +{ + let file = File::open(path)?; + let mut reader = BufReader::new(file); + let certs = rustls_pemfile::certs(&mut reader).collect::, io::Error>>()?; + + if certs.is_empty() { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("no certificates found in {}", path.display()), + ) + .into()); + } + + Ok(certs) +} + +fn load_private_key( + path: &Path, +) -> Result, Box> { + let file = File::open(path)?; + let mut reader = BufReader::new(file); + let private_key = rustls_pemfile::private_key(&mut reader)?.ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidData, + format!("no private key found in {}", path.display()), + ) + })?; + + Ok(private_key) +}