diff --git a/Cargo.lock b/Cargo.lock index 70bb0b26..a139cdb7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -917,6 +917,7 @@ dependencies = [ [[package]] name = "dojo-introspect" version = "0.1.0" +source = "git+https://github.com/dojoengine/dojo-introspect?rev=aadc3c9#aadc3c980706596a4a083413813a0a3ab01fded7" dependencies = [ "async-trait", "introspect-rust-macros", @@ -1863,6 +1864,7 @@ dependencies = [ [[package]] name = "introspect-events" version = "0.1.2" +source = "git+https://github.com/cartridge-gg/introspect?rev=34e93c1#34e93c10c867c53c622cce03abb6431c9dae0ef5" dependencies = [ "cainome-cairo-serde", "introspect-types", @@ -1874,6 +1876,7 @@ dependencies = [ [[package]] name = "introspect-rust-macros" version = "0.1.0" +source = "git+https://github.com/cartridge-gg/introspect?rev=34e93c1#34e93c10c867c53c622cce03abb6431c9dae0ef5" dependencies = [ "paste", "proc-macro2", @@ -1885,6 +1888,7 @@ dependencies = [ [[package]] name = "introspect-types" version = "0.1.2" +source = "git+https://github.com/cartridge-gg/introspect?rev=34e93c1#34e93c10c867c53c622cce03abb6431c9dae0ef5" dependencies = [ "blake3", "convert_case", @@ -4678,6 +4682,35 @@ dependencies = [ "tracing", ] +[[package]] +name = "torii-ecs-sink" +version = "0.1.0" +dependencies = [ + "anyhow", + "async-trait", + "chrono", + "dojo-introspect", + "futures", + "futures-util", + "hex", + "introspect-types", + "metrics", + "prost 0.13.5", + "prost-types 0.13.5", + "serde", + "serde_json", + "sqlx", + "starknet", + "tokio", + "tokio-stream", + "tonic", + "tonic-build", + "torii", + "torii-dojo", + "torii-introspect", + "tracing", +] + [[package]] name = "torii-erc1155" version = "0.1.0" @@ -4826,9 +4859,13 @@ dependencies = [ "sqlx", "starknet", "tokio", + "tonic", + "tonic-reflection", + "tonic-web", "torii", "torii-config-common", "torii-dojo", + "torii-ecs-sink", "torii-introspect-postgres-sink", "torii-introspect-sqlite-sink", "torii-sqlite", diff --git a/Cargo.toml b/Cargo.toml index 47989e6c..2fc08306 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,6 +6,7 @@ members = [ "crates/torii-runtime-common", "crates/torii-sql-sink", "crates/torii-log-sink", + "crates/torii-ecs-sink", "crates/torii-erc20", "crates/torii-erc721", "crates/torii-erc1155", @@ -126,6 +127,7 @@ torii-runtime-common = { path = "crates/torii-runtime-common" } # Internal crates torii-sql-sink.path = "crates/torii-sql-sink" torii-log-sink.path = "crates/torii-log-sink" +torii-ecs-sink.path = "crates/torii-ecs-sink" torii-erc20.path = "crates/torii-erc20" torii-erc721.path = "crates/torii-erc721" torii-erc1155.path = "crates/torii-erc1155" diff --git a/bins/torii-introspect-bin/Cargo.toml b/bins/torii-introspect-bin/Cargo.toml index e255c6c1..f2583c14 100644 --- a/bins/torii-introspect-bin/Cargo.toml +++ b/bins/torii-introspect-bin/Cargo.toml @@ -12,6 +12,7 @@ path = "src/main.rs" torii = { path = "../../" } torii-config-common.workspace = true torii-dojo = { path = "../../crates/dojo" } +torii-ecs-sink = { path = "../../crates/torii-ecs-sink" } torii-introspect-postgres-sink = { path = "../../crates/introspect-postgres-sink" } torii-introspect-sqlite-sink = { path = "../../crates/introspect-sqlite-sink" } torii-sqlite = { path = "../../crates/sqlite" } @@ -24,6 +25,9 @@ url = "2.5" clap = { version = "4.5", features = ["derive", "env"] } anyhow = "1.0" sqlx = { version = "0.8", features = ["postgres", "sqlite", "runtime-tokio-rustls", "any"] } +tonic.workspace = true +tonic-reflection.workspace = true +tonic-web.workspace = true [lints] workspace = true diff --git a/bins/torii-introspect-bin/src/config.rs b/bins/torii-introspect-bin/src/config.rs index 5b7bf658..e3cf6f0d 100644 --- a/bins/torii-introspect-bin/src/config.rs +++ b/bins/torii-introspect-bin/src/config.rs @@ -81,9 +81,6 @@ pub struct Config { pub max_db_connections: Option, /// Ignore persisted extractor state and force extraction from `from_block`. - /// - /// This binary no longer supports historical schema bootstrap, so resumed - /// runs must opt into a fresh extraction window explicitly. #[arg(long)] pub ignore_saved_state: bool, } @@ -133,31 +130,18 @@ mod tests { #[test] fn observability_defaults_to_disabled() { - let cfg = Config::parse_from([ - "torii-introspect", - "--contracts", - "0x1", - "--storage-database-url", - "postgres://localhost/torii", - ]); + let cfg = Config::parse_from(["torii-introspect", "--contracts", "0x1"]); assert!(!cfg.observability); } #[test] fn observability_flag_enables_metrics() { - let cfg = Config::parse_from([ - "torii-introspect", - "--contracts", - "0x1", - "--storage-database-url", - "postgres://localhost/torii", - "--observability", - ]); + let cfg = Config::parse_from(["torii-introspect", "--contracts", "0x1", "--observability"]); assert!(cfg.observability); } #[test] - fn storage_database_url_must_be_postgres() { + fn storage_database_url_accepts_sqlite() { let cfg = Config::parse_from([ "torii-introspect", "--contracts", @@ -182,13 +166,7 @@ mod tests { #[test] fn contract_addresses_parse_from_hex() { - let cfg = Config::parse_from([ - "torii-introspect", - "--contracts", - "0x1,0x2", - "--storage-database-url", - "postgres://localhost/torii", - ]); + let cfg = Config::parse_from(["torii-introspect", "--contracts", "0x1,0x2"]); let contracts = cfg.contract_addresses().unwrap(); assert_eq!(contracts.len(), 2); diff --git a/bins/torii-introspect-bin/src/main.rs b/bins/torii-introspect-bin/src/main.rs index 95c07861..96a9d615 100644 --- a/bins/torii-introspect-bin/src/main.rs +++ b/bins/torii-introspect-bin/src/main.rs @@ -2,16 +2,16 @@ mod config; -use anyhow::{bail, Result}; +use anyhow::Result; use clap::Parser; use config::{Config, StorageBackend}; use sqlx::postgres::PgPoolOptions; -use sqlx::sqlite::SqlitePoolOptions; +use sqlx::sqlite::{SqliteConnectOptions, SqlitePoolOptions}; use starknet::core::types::Felt; use std::path::Path; +use std::str::FromStr; use std::sync::Arc; use torii::etl::decoder::DecoderId; -use torii::etl::engine_db::{EngineDb, EngineDbConfig}; use torii::etl::extractor::{ ContractEventConfig, EventExtractor, EventExtractorConfig, RetryPolicy, }; @@ -19,9 +19,31 @@ use torii_config_common::apply_observability_env; use torii_dojo::decoder::DojoDecoder; use torii_dojo::store::postgres::PgStore; use torii_dojo::store::sqlite::SqliteStore; +use torii_ecs_sink::proto::world::world_server::WorldServer; +use torii_ecs_sink::{EcsSink, FILE_DESCRIPTOR_SET as ECS_DESCRIPTOR_SET}; use torii_introspect_postgres_sink::processor::IntrospectPgDb; use torii_introspect_sqlite_sink::processor::IntrospectSqliteDb; -use torii_sqlite::{is_sqlite_memory_path, sqlite_connect_options}; +use torii_sqlite::is_sqlite_memory_path; + +fn sqlite_connect_options(path: &str) -> Result { + if path == ":memory:" || path == "sqlite::memory:" { + return SqliteConnectOptions::from_str("sqlite::memory:") + .map_err(|err| anyhow::anyhow!("Failed to parse sqlite URL: {err}")); + } + + let options = if path.starts_with("sqlite:") { + SqliteConnectOptions::from_str(path) + .map_err(|err| anyhow::anyhow!("Failed to parse sqlite URL {path}: {err}"))? + } else { + SqliteConnectOptions::new().filename(path) + }; + + if path.starts_with("sqlite:") && path.contains("mode=") { + Ok(options) + } else { + Ok(options.create_if_missing(true)) + } +} #[tokio::main] async fn main() -> Result<()> { @@ -59,12 +81,13 @@ async fn run_indexer(config: Config) -> Result<()> { tracing::info!("Storage backend: {:?}", backend); tracing::info!("Engine database URL: {}", engine_database_url); tracing::info!("Storage database URL: {}", storage_database_url); + tracing::info!("Database backend: {:?}", backend); tracing::info!( "Saved state handling: {}", if config.ignore_saved_state { "ignoring persisted extractor state" } else { - "failing closed if persisted extractor state exists" + "resuming from persisted extractor state when available" } ); tracing::info!( @@ -106,12 +129,6 @@ async fn run_indexer(config: Config) -> Result<()> { tokio::fs::create_dir_all(db_dir).await?; } - let engine_db = EngineDb::new(EngineDbConfig { - path: engine_database_url.clone(), - }) - .await?; - ensure_no_saved_state(&engine_db, &contracts, config.ignore_saved_state).await?; - match backend { StorageBackend::Postgres => { run_with_postgres( @@ -160,17 +177,39 @@ async fn run_with_postgres( ); let decoder = DojoDecoder::, _>::new(pool.clone(), provider); + let sink = IntrospectPgDb::new(pool.clone(), ()); decoder.store.initialize().await?; + sink.initialize_introspect_pg_sink().await?; decoder.load_tables(&[]).await?; + sink.load_tables_no_commit(decoder.get_tables()?)?; let decoder: Arc = Arc::new(decoder); + let ecs_sink = EcsSink::new(storage_database_url, config.max_db_connections).await?; + let ecs_grpc_service = ecs_sink.get_grpc_service_impl(); + + let reflection = tonic_reflection::server::Builder::configure() + .register_encoded_file_descriptor_set(torii::TORII_DESCRIPTOR_SET) + .register_encoded_file_descriptor_set(ECS_DESCRIPTOR_SET) + .build_v1() + .expect("failed to build ECS reflection service"); + + let grpc_router = tonic::transport::Server::builder() + .accept_http1(true) + .add_service(tonic_web::enable(WorldServer::new( + (*ecs_grpc_service).clone(), + ))) + .add_service(tonic_web::enable(reflection)); + let mut torii_config = torii::ToriiConfig::builder() .port(config.port) + .with_grpc_router(grpc_router) + .with_custom_reflection(true) .engine_database_url(engine_database_url) .with_extractor(extractor) .add_decoder(decoder) - .add_sink_boxed(Box::new(IntrospectPgDb::new(pool.clone(), ()))); + .add_sink_boxed(Box::new(ecs_sink)) + .add_sink_boxed(Box::new(sink)); let decoder_id = DecoderId::new("dojo-introspect"); for contract in contracts { @@ -181,6 +220,7 @@ async fn run_with_postgres( tracing::info!("Torii configured, starting ETL pipeline..."); tracing::info!("gRPC service available on port {}", config.port); tracing::info!(" - torii.Torii (core subscriptions and metrics endpoint)"); + tracing::info!(" - world.World (legacy ECS gRPC service)"); torii::run(torii_config.build()) .await @@ -227,11 +267,30 @@ async fn run_with_sqlite( let decoder: Arc = Arc::new(decoder); + let ecs_sink = EcsSink::new(storage_database_url, config.max_db_connections).await?; + let ecs_grpc_service = ecs_sink.get_grpc_service_impl(); + + let reflection = tonic_reflection::server::Builder::configure() + .register_encoded_file_descriptor_set(torii::TORII_DESCRIPTOR_SET) + .register_encoded_file_descriptor_set(ECS_DESCRIPTOR_SET) + .build_v1() + .expect("failed to build ECS reflection service"); + + let grpc_router = tonic::transport::Server::builder() + .accept_http1(true) + .add_service(tonic_web::enable(WorldServer::new( + (*ecs_grpc_service).clone(), + ))) + .add_service(tonic_web::enable(reflection)); + let mut torii_config = torii::ToriiConfig::builder() .port(config.port) + .with_grpc_router(grpc_router) + .with_custom_reflection(true) .engine_database_url(engine_database_url) .with_extractor(extractor) .add_decoder(decoder) + .add_sink_boxed(Box::new(ecs_sink)) .add_sink_boxed(Box::new(IntrospectSqliteDb::new(pool.clone(), ()))); let decoder_id = DecoderId::new("dojo-introspect"); @@ -243,46 +302,10 @@ async fn run_with_sqlite( tracing::info!("Torii configured, starting ETL pipeline..."); tracing::info!("gRPC service available on port {}", config.port); tracing::info!(" - torii.Torii (core subscriptions and metrics endpoint)"); + tracing::info!(" - world.World (legacy ECS gRPC service)"); torii::run(torii_config.build()) .await .map_err(|e| anyhow::anyhow!("Torii error: {e}"))?; Ok(()) } - -async fn ensure_no_saved_state( - engine_db: &EngineDb, - contracts: &[Felt], - ignore_saved_state: bool, -) -> Result<()> { - if ignore_saved_state { - return Ok(()); - } - - let mut owners_with_saved_state = Vec::new(); - for owner in contracts { - let state_key = format!("{owner:#x}"); - if engine_db - .get_extractor_state("event", &state_key) - .await? - .is_some() - { - owners_with_saved_state.push(*owner); - } - } - - if owners_with_saved_state.is_empty() { - return Ok(()); - } - - bail!( - "persisted extractor state exists for contract(s): {}. \ - Historical schema bootstrap is not supported on this branch; \ - re-run with --ignore-saved-state or use a fresh database", - owners_with_saved_state - .iter() - .map(|owner| format!("{owner:#x}")) - .collect::>() - .join(", ") - ) -} diff --git a/crates/dojo/src/error.rs b/crates/dojo/src/error.rs index 35b13dea..911f7550 100644 --- a/crates/dojo/src/error.rs +++ b/crates/dojo/src/error.rs @@ -8,21 +8,21 @@ use starknet_types_core::felt::Felt; #[derive(Debug, thiserror::Error)] pub enum DojoToriiError { - #[error("Unknown Dojo Event selector {0}")] + #[error("Unknown Dojo Event selector {0:#066x}")] UnknownDojoEventSelector(Felt), #[error("Missing event selector")] MissingEventSelector, - #[error("Column {0} not found in table {1}")] + #[error("Column {0:#066x} not found in table {1}")] ColumnNotFound(Felt, String), - #[error("Failed to parse field {0} in table {1}")] + #[error("Failed to parse field {0:#066x} in table {1}")] FieldParseError(Felt, String), - #[error("Too many values provided for field {0}")] + #[error("Too many values provided for field {0:#066x}")] TooManyFieldValues(Felt), #[error("Failed to parse values for table {0}")] ParseValuesError(String), - #[error("Cannot add {2} table already exists with id {0} and name {1}")] + #[error("Cannot add {2} table already exists with id {0:#066x} and name {1}")] TableAlreadyExists(Felt, String, String), - #[error("Table not found with id {0}")] + #[error("Table not found with id {0:#066x}")] TableNotFoundById(Felt), #[error("Failed to acquire lock: {0}")] LockError(String), diff --git a/crates/dojo/src/table.rs b/crates/dojo/src/table.rs index 4c8d24d3..05048a88 100644 --- a/crates/dojo/src/table.rs +++ b/crates/dojo/src/table.rs @@ -22,6 +22,7 @@ pub struct DojoTable { pub legacy: bool, } +#[derive(Debug, serde::Serialize, serde::Deserialize, Clone)] pub struct DojoTableInfo { pub name: String, pub attributes: Vec, diff --git a/crates/introspect-postgres-sink/Cargo.toml b/crates/introspect-postgres-sink/Cargo.toml index 357b4029..1702540e 100644 --- a/crates/introspect-postgres-sink/Cargo.toml +++ b/crates/introspect-postgres-sink/Cargo.toml @@ -9,6 +9,7 @@ license = "Apache-2.0" [dependencies] sqlx = { workspace = true, features = [ "postgres", + "sqlite", "runtime-tokio-rustls", "macros", "migrate", diff --git a/crates/introspect-postgres-sink/src/processor.rs b/crates/introspect-postgres-sink/src/processor.rs index 053b2437..b98cca3c 100644 --- a/crates/introspect-postgres-sink/src/processor.rs +++ b/crates/introspect-postgres-sink/src/processor.rs @@ -365,105 +365,3 @@ pub struct MessageWithContext<'a, M> { pub msg: &'a M, pub context: &'a MetaData, } - -// #[cfg(test)] -// mod tests { -// use super::*; -// use introspect_types::{ColumnDef, PrimaryDef, PrimaryTypeDef, TypeDef}; -// use torii_introspect::{CreateTable, UpdateTable}; - -// fn primary() -> PrimaryDef { -// PrimaryDef { -// name: "entity_id".to_string(), -// attributes: vec![], -// type_def: PrimaryTypeDef::Felt252, -// } -// } - -// fn column(id: u64, name: &str) -> ColumnDef { -// ColumnDef { -// id: Felt::from(id), -// name: name.to_string(), -// attributes: vec![], -// type_def: TypeDef::U32, -// } -// } - -// fn create_table(id: u64, columns: Vec) -> CreateTable { -// CreateTable { -// id: Felt::from(id), -// name: "duel-state".to_string(), -// attributes: vec![], -// primary: primary(), -// columns, -// } -// } - -// fn update_table(id: u64, columns: Vec) -> UpdateTable { -// UpdateTable { -// id: Felt::from(id), -// name: "duel-state".to_string(), -// attributes: vec![], -// primary: primary(), -// columns, -// } -// } - -// // #[test] -// // fn update_table_creates_unknown_table() { -// // let mut tables = PostgresTables::default(); -// // let mut queries = Vec::new(); -// // let schema = PgSchema::Public; -// // tables -// // .update_table( -// // &schema, -// // update_table(1, vec![column(10, "duelist_count")]), -// // &mut queries, -// // ) -// // .unwrap(); - -// // assert!(!queries.is_empty()); -// // assert!(queries -// // .iter() -// // .any(|query| query.contains("CREATE TABLE IF NOT EXISTS"))); -// // assert!(tables.read().unwrap().contains_key(&Felt::from(1_u64))); -// // } - -// // #[test] -// // fn update_table_emits_alter_queries_for_new_columns() { -// // let mut tables = PostgresTables::default(); -// // let mut create_queries = Vec::new(); -// // let schema = PgSchema::Public; -// // tables -// // .create_table( -// // &schema, -// // create_table(1, vec![column(10, "duelist_count")]), -// // &mut create_queries, -// // ) -// // .unwrap(); - -// // let mut update_queries = Vec::new(); -// // tables -// // .update_table( -// // &schema, -// // update_table( -// // 1, -// // vec![ -// // column(10, "duelist_count"), -// // column(11, "alive_duelist_count"), -// // ], -// // ), -// // &EventContext::default(), -// // &mut update_queries, -// // ) -// // .unwrap(); - -// // assert!(update_queries -// // .iter() -// // .any(|query| query.contains("ALTER TABLE") && query.contains("alive_duelist_count"))); -// // assert!(tables.read().unwrap()[&Felt::from(1_u64)] -// // .columns -// // .values() -// // .any(|column| column.name == "alive_duelist_count")); -// // } -// } diff --git a/crates/introspect-sqlite-sink/migrations/002_schema_state.sql b/crates/introspect-sqlite-sink/migrations/002_schema_state.sql new file mode 100644 index 00000000..caa564d4 --- /dev/null +++ b/crates/introspect-sqlite-sink/migrations/002_schema_state.sql @@ -0,0 +1,6 @@ +CREATE TABLE IF NOT EXISTS introspect_sink_schema_state ( + table_id TEXT PRIMARY KEY, + table_schema_json TEXT NOT NULL, + alive INTEGER NOT NULL DEFAULT 1, + updated_at INTEGER NOT NULL DEFAULT (unixepoch()) +); diff --git a/crates/introspect-sqlite-sink/src/processor.rs b/crates/introspect-sqlite-sink/src/processor.rs index 12ff6e4a..3f6121b2 100644 --- a/crates/introspect-sqlite-sink/src/processor.rs +++ b/crates/introspect-sqlite-sink/src/processor.rs @@ -1,14 +1,15 @@ use crate::json::SqliteJsonSerializer; use crate::table::{SqliteTable, SqliteTableError}; use crate::INTROSPECT_SQLITE_SINK_MIGRATIONS; -use serde_json::Value; +use serde_json::{Serializer as JsonSerializer, Value}; +use sqlx::Row; use starknet_types_core::felt::Felt; use std::collections::HashMap; use std::fmt::Display; use std::ops::Deref; use std::sync::{PoisonError, RwLock}; -use torii::etl::MetaData; -use torii_introspect::events::IntrospectMsg; +use torii::etl::envelope::MetaData; +use torii_introspect::events::{IntrospectBody, IntrospectMsg}; use torii_introspect::schema::TableSchema; use torii_introspect::InsertsFields; use torii_sqlite::{SqliteConnection, SqlxError}; @@ -181,6 +182,70 @@ impl IntrospectSqliteDb { pub async fn initialize_introspect_sqlite_sink(&self) -> SqliteDbResult<()> { self.migrate(Some("introspect"), INTROSPECT_SQLITE_SINK_MIGRATIONS) .await?; + self.load_persisted_state().await?; + Ok(()) + } + + async fn load_persisted_state(&self) -> SqliteDbResult<()> { + let rows = sqlx::query( + r" + SELECT table_schema_json, alive + FROM introspect_sink_schema_state + ORDER BY updated_at ASC + ", + ) + .fetch_all(self.pool()) + .await?; + + let mut tables = self.tables.write()?; + for row in rows { + let schema_json: String = row.try_get("table_schema_json")?; + let alive: i64 = row.try_get("alive")?; + let table_schema: TableSchema = serde_json::from_str(&schema_json)?; + let (id, mut table) = + SqliteTable::new_from_table(self.namespace.prefix(), table_schema); + table.alive = alive != 0; + tables.insert(id, table); + } + + Ok(()) + } + + async fn persist_table_state(&self, table: &TableSchema, alive: bool) -> SqliteDbResult<()> { + let schema_json = serde_json::to_string(table)?; + let alive = i64::from(alive); + sqlx::query( + r" + INSERT INTO introspect_sink_schema_state (table_id, table_schema_json, alive, updated_at) + VALUES (?1, ?2, ?3, unixepoch()) + ON CONFLICT (table_id) + DO UPDATE SET + table_schema_json = excluded.table_schema_json, + alive = excluded.alive, + updated_at = unixepoch() + ", + ) + .bind(format!("{:#x}", table.id)) + .bind(schema_json) + .bind(alive) + .execute(self.pool()) + .await?; + Ok(()) + } + + async fn persist_alive_state(&self, table_id: Felt, alive: bool) -> SqliteDbResult<()> { + let alive = i64::from(alive); + sqlx::query( + r" + UPDATE introspect_sink_schema_state + SET alive = ?1, updated_at = unixepoch() + WHERE table_id = ?2 + ", + ) + .bind(alive) + .bind(format!("{table_id:#x}")) + .execute(self.pool()) + .await?; Ok(()) } @@ -202,13 +267,30 @@ impl IntrospectSqliteDb { IntrospectMsg::CreateTable(event) => { let (_, query) = self.tables.create_table(&self.namespace, event.clone())?; self.execute_queries(&[query]).await?; + self.persist_table_state(&event.clone().into(), true) + .await?; Ok(()) } - IntrospectMsg::UpdateTable(event) => self.tables.set_table_dead(&event.id), - IntrospectMsg::AddColumns(event) => self.tables.set_table_dead(&event.table), - IntrospectMsg::DropColumns(event) => self.tables.set_table_dead(&event.table), - IntrospectMsg::RetypeColumns(event) => self.tables.set_table_dead(&event.table), - IntrospectMsg::RetypePrimary(event) => self.tables.set_table_dead(&event.table), + IntrospectMsg::UpdateTable(event) => { + self.tables.set_table_dead(&event.id)?; + self.persist_alive_state(event.id, false).await + } + IntrospectMsg::AddColumns(event) => { + self.tables.set_table_dead(&event.table)?; + self.persist_alive_state(event.table, false).await + } + IntrospectMsg::DropColumns(event) => { + self.tables.set_table_dead(&event.table)?; + self.persist_alive_state(event.table, false).await + } + IntrospectMsg::RetypeColumns(event) => { + self.tables.set_table_dead(&event.table)?; + self.persist_alive_state(event.table, false).await + } + IntrospectMsg::RetypePrimary(event) => { + self.tables.set_table_dead(&event.table)?; + self.persist_alive_state(event.table, false).await + } IntrospectMsg::RenameTable(_) | IntrospectMsg::DropTable(_) | IntrospectMsg::RenameColumns(_) @@ -219,6 +301,18 @@ impl IntrospectSqliteDb { } } + pub async fn process_messages( + &self, + msgs: Vec<&IntrospectBody>, + ) -> SqliteDbResult>> { + let mut results = Vec::with_capacity(msgs.len()); + for body in msgs { + let (msg, metadata) = body.into(); + results.push(self.process_message(msg, metadata).await); + } + Ok(results) + } + async fn insert_fields( &self, event: &InsertsFields, @@ -269,10 +363,18 @@ impl IntrospectSqliteDb { update_columns ); + let mut bytes = Vec::new(); + let mut serializer = JsonSerializer::new(&mut bytes); + record_schema.parse_records_with_metadata( + &event.records, + &(), + &mut serializer, + &SqliteJsonSerializer, + )?; + let rows = serde_json::from_slice::>(&bytes)?; + let mut tx = self.begin().await?; - for record in &event.records { - let value = - serde_json::to_value(record_schema.to_frame(record, &(), &SqliteJsonSerializer))?; + for value in rows { let object = value.as_object().ok_or(SqliteDbError::InvalidRecordFrame)?; let mut query = sqlx::query(&sql); diff --git a/crates/introspect-sqlite-sink/src/sink.rs b/crates/introspect-sqlite-sink/src/sink.rs index fee2cc19..53de7d7c 100644 --- a/crates/introspect-sqlite-sink/src/sink.rs +++ b/crates/introspect-sqlite-sink/src/sink.rs @@ -31,6 +31,7 @@ impl Sink for IntrospectSqliteDb { let mut inserts_fields = 0usize; let mut inserted_records = 0usize; let mut delete_records = 0usize; + let mut msgs = Vec::with_capacity(envelopes.len()); for envelope in envelopes { if envelope.type_id == INTROSPECT_TYPE { if let Some(body) = envelope.downcast_ref::() { @@ -47,10 +48,11 @@ impl Sink for IntrospectSqliteDb { _ => {} } processed += 1; - self.process_message(&body.msg, &body.metadata).await?; + msgs.push(body); } } } + self.process_messages(msgs).await?; if processed > 0 { tracing::info!( diff --git a/crates/torii-ecs-sink/Cargo.toml b/crates/torii-ecs-sink/Cargo.toml new file mode 100644 index 00000000..0d402e5f --- /dev/null +++ b/crates/torii-ecs-sink/Cargo.toml @@ -0,0 +1,35 @@ +[package] +name = "torii-ecs-sink" +version = "0.1.0" +edition = "2021" + +[dependencies] +torii = { path = "../.." } +torii-dojo = { path = "../dojo" } +torii-introspect = { path = "../introspect" } +dojo-introspect.workspace = true +introspect-types.workspace = true + +anyhow.workspace = true +async-trait.workspace = true +chrono.workspace = true +futures.workspace = true +futures-util.workspace = true +prost.workspace = true +prost-types.workspace = true +serde.workspace = true +serde_json.workspace = true +sqlx = { workspace = true, features = ["runtime-tokio", "sqlite", "postgres", "any"] } +starknet.workspace = true +tokio.workspace = true +tokio-stream.workspace = true +tonic.workspace = true +tracing.workspace = true +metrics.workspace = true +hex.workspace = true + +[build-dependencies] +tonic-build.workspace = true + +[lints] +workspace = true diff --git a/crates/torii-ecs-sink/README.md b/crates/torii-ecs-sink/README.md new file mode 100644 index 00000000..21405d63 --- /dev/null +++ b/crates/torii-ecs-sink/README.md @@ -0,0 +1,88 @@ +# Torii ECS Sink + +`torii-ecs-sink` exposes the legacy `world.World` gRPC API on top of the data already indexed by `torii-introspect`. + +## What It Serves + +The sink mounts these legacy RPCs into `torii-introspect-bin` on the same gRPC port: + +- `Worlds` +- `RetrieveEntities` +- `RetrieveEventMessages` +- `RetrieveEvents` +- `SubscribeContracts` +- `SubscribeEntities` +- `SubscribeEventMessages` +- `SubscribeEvents` +- `UpdateEntitiesSubscription` +- `UpdateEventMessagesSubscription` + +The read path is backed by: + +- `torii_dojo_manager_state` for Dojo table metadata +- `torii_ecs_entity_meta` and `torii_ecs_entity_models` for entity and event-message snapshots +- `torii_ecs_events` for raw events +- `torii_ecs_table_kinds` for entity vs event-message classification + +## Manual Validation + +With `torii-introspect-bin` running on the default port: + +```bash +grpcurl -plaintext localhost:3000 list +grpcurl -plaintext localhost:3000 describe world.World +grpcurl -plaintext -d '{}' localhost:3000 world.World/Worlds +grpcurl -max-time 10 -plaintext -d '{"query":{"pagination":{"limit":1,"direction":"FORWARD"}}}' localhost:3000 world.World/RetrieveEvents +``` + +Important: + +- `RetrieveEntities` only returns models classified as `entity` +- `RetrieveEventMessages` only returns models classified as `event_message` +- a model such as `ARCADE-Sale` can be valid for `RetrieveEventMessages` and intentionally return nothing from `RetrieveEntities` + +## End-to-End Test Script + +The repository ships a runnable script at `./scripts/test-ecs-grpc-e2e.sh`. + +From the repository root: + +```bash +./scripts/test-ecs-grpc-e2e.sh +``` + +By default the script: + +- targets `localhost:3000` +- reads sqlite state from `./torii-data/introspect.db` +- autodiscovers one world that has both `entity` and `event_message` models +- validates reflection, `Worlds`, `RetrieveEntities`, `RetrieveEventMessages`, and `RetrieveEvents` + +For Postgres-backed deployments, set `DB_URL` or `DATABASE_URL` to the same storage database used by `torii-introspect-bin`. The script will use `psql` for metadata discovery instead of `sqlite3`. + +### Environment Overrides + +You can override discovery when needed: + +```bash +GRPC_ADDR=localhost:3000 \ +DB_PATH=./torii-data/introspect.db \ +WORLD_ADDRESS=0x7a079295990e43441a7389fdc3b9ba063c6cd6aee16fb846f598c42a9f04ff7 \ +ENTITY_MODEL=ARCADE-Order \ +EVENT_MESSAGE_MODEL=ARCADE-Sale \ +./scripts/test-ecs-grpc-e2e.sh +``` + +If `WORLD_ADDRESS`, `ENTITY_MODEL`, and `EVENT_MESSAGE_MODEL` are provided, the script does not need sqlite introspection metadata to choose test cases. + +Postgres example: + +```bash +GRPC_ADDR=localhost:3000 \ +DB_URL=postgres://torii:torii@localhost:5432/torii \ +./scripts/test-ecs-grpc-e2e.sh +``` + +## Kulala + +For editor-driven checks, the crate also includes `./crates/torii-ecs-sink/ecs-grpc.http`. diff --git a/crates/torii-ecs-sink/build.rs b/crates/torii-ecs-sink/build.rs new file mode 100644 index 00000000..47298bad --- /dev/null +++ b/crates/torii-ecs-sink/build.rs @@ -0,0 +1,15 @@ +use std::path::PathBuf; + +fn main() -> Result<(), Box> { + let out_dir = + PathBuf::from(std::env::var("OUT_DIR").expect("OUT_DIR environment variable not set")); + + tonic_build::configure() + .build_server(true) + .build_client(false) + .file_descriptor_set_path(out_dir.join("world_descriptor.bin")) + .compile_protos(&["proto/world.proto"], &["proto"])?; + + println!("cargo:rerun-if-changed=proto"); + Ok(()) +} diff --git a/crates/torii-ecs-sink/ecs-grpc.http b/crates/torii-ecs-sink/ecs-grpc.http new file mode 100644 index 00000000..ed96bfb8 --- /dev/null +++ b/crates/torii-ecs-sink/ecs-grpc.http @@ -0,0 +1,159 @@ +@address=localhost:3000 + +@world_1_hex=0x2d26295d6c541d64740e1ae56abc079b82b22c35ab83985ef8bd15dc0f9edfb +@world_1_b64=LSYpXWxUHWR0DhrlarwHm4KyLDWrg5he+L0V3A+e3w== + +@world_2_hex=0x02ef591697f0fd9adc0ba9dbe0ca04dabad80cf95f08ba02e435d9cb6698a28a +@world_2_b64=Au9ZFpfw/ZrcC6nb4MoE2rrYDPlfCLoC5DXZy2aYooo= + +@world_3_hex=0x8b4838140a3cbd36ebe64d4b5aaf56a30cc3753c928a79338bf56c53f506c5 +@world_3_b64=i0g4FAo8vTbr5k1LWq9WowzDdTySinkzi/VsU/UGxQ== + +@world_4_hex=0x07a079295990e43441a7389fdc3b9ba063c6cd6aee16fb846f598c42a9f04ff7 +@world_4_b64=B6B5KVmQ5DRBpzif3DuboGPGzWruFvuEb1mMQqnwT/c= + +@subscription_id=1 + +### Shared +# @grpc-plaintext +# @grpc-import-path proto +# @grpc-proto world.proto + +### List gRPC Services +GRPC {{address}} list + +### Describe ECS Service +GRPC {{address}} describe world.World + +### Describe RetrieveEntities +GRPC {{address}} describe world.World/RetrieveEntities + +### Worlds For First Indexed World +GRPC {{address}} world.World/Worlds +Content-Type: application/json + +{ + "world_addresses": ["{{world_1_b64}}"] +} + +### Worlds For All Indexed Worlds +GRPC {{address}} world.World/Worlds +Content-Type: application/json + +{ + "world_addresses": [ + "{{world_1_b64}}", + "{{world_2_b64}}", + "{{world_3_b64}}", + "{{world_4_b64}}" + ] +} + +### Subscribe Contracts For First World +GRPC {{address}} world.World/SubscribeContracts +Content-Type: application/json + +{ + "query": { + "contract_addresses": ["{{world_1_b64}}"], + "contract_types": ["WORLD"] + } +} + +### Retrieve Entities For First World +GRPC {{address}} world.World/RetrieveEntities +Content-Type: application/json + +{ + "query": { + "world_addresses": ["{{world_1_b64}}"], + "pagination": { + "limit": 5, + "direction": "FORWARD" + } + } +} + +### Retrieve Entities For First World And Specific Model +GRPC {{address}} world.World/RetrieveEntities +Content-Type: application/json + +{ + "query": { + "world_addresses": ["{{world_1_b64}}"], + "models": ["blob_arena-CombatantState"], + "pagination": { + "limit": 5, + "direction": "FORWARD" + } + } +} + +### Subscribe Entities For First World +GRPC {{address}} world.World/SubscribeEntities +Content-Type: application/json + +{ + "world_addresses": ["{{world_1_b64}}"] +} + +### Update Entity Subscription +GRPC {{address}} world.World/UpdateEntitiesSubscription +Content-Type: application/json + +{ + "subscription_id": {{subscription_id}}, + "world_addresses": ["{{world_1_b64}}"] +} + +### Retrieve Event Messages For First World +GRPC {{address}} world.World/RetrieveEventMessages +Content-Type: application/json + +{ + "query": { + "world_addresses": ["{{world_1_b64}}"], + "pagination": { + "limit": 5, + "direction": "FORWARD" + } + } +} + +### Subscribe Event Messages For First World +GRPC {{address}} world.World/SubscribeEventMessages +Content-Type: application/json + +{ + "world_addresses": ["{{world_1_b64}}"] +} + +### Update Event Message Subscription +GRPC {{address}} world.World/UpdateEventMessagesSubscription +Content-Type: application/json + +{ + "subscription_id": {{subscription_id}}, + "world_addresses": ["{{world_1_b64}}"] +} + +### Retrieve Events +GRPC {{address}} world.World/RetrieveEvents +Content-Type: application/json + +{ + "query": { + "pagination": { + "limit": 5, + "direction": "FORWARD" + } + } +} + +### Subscribe Events +GRPC {{address}} world.World/SubscribeEvents +Content-Type: application/json + +{ + "keys": [] +} diff --git a/crates/torii-ecs-sink/proto/schema.proto b/crates/torii-ecs-sink/proto/schema.proto new file mode 100644 index 00000000..676e9cb0 --- /dev/null +++ b/crates/torii-ecs-sink/proto/schema.proto @@ -0,0 +1,66 @@ +syntax = "proto3"; +package types; + +message EnumOption { + string name = 1; + Ty ty = 2; +} + +message Enum { + string name = 1; + uint32 option = 2; + repeated EnumOption options = 3; +} + +message Primitive { + oneof primitive_type { + int32 i8 = 1; + int32 i16 = 2; + int32 i32 = 3; + int64 i64 = 4; + bytes i128 = 5; + uint32 u8 = 6; + uint32 u16 = 7; + uint32 u32 = 8; + uint64 u64 = 9; + bytes u128 = 10; + bytes u256 = 11; + bool bool = 12; + bytes felt252 = 13; + bytes class_hash = 14; + bytes contract_address = 15; + bytes eth_address = 16; + } +} + +message Struct { + string name = 1; + repeated Member children = 2; +} + +message Array { + repeated Ty children = 1; +} + +message FixedSizeArray { + repeated Ty children = 1; + uint32 size = 2; +} + +message Ty { + oneof ty_type { + Primitive primitive = 2; + Enum enum = 3; + Struct struct = 4; + Array tuple = 5; + Array array = 6; + string bytearray = 7; + FixedSizeArray fixed_size_array = 8; + } +} + +message Member { + string name = 1; + Ty ty = 2; + bool key = 3; +} diff --git a/crates/torii-ecs-sink/proto/types.proto b/crates/torii-ecs-sink/proto/types.proto new file mode 100644 index 00000000..b3e6d98a --- /dev/null +++ b/crates/torii-ecs-sink/proto/types.proto @@ -0,0 +1,169 @@ +syntax = "proto3"; +package types; + +import "schema.proto"; + +message World { + string world_address = 1; + repeated Model models = 2; +} + +message Model { + bytes selector = 1; + string namespace = 2; + string name = 3; + uint32 packed_size = 4; + uint32 unpacked_size = 5; + bytes class_hash = 6; + bytes layout = 7; + bytes schema = 8; + bytes contract_address = 9; + bool use_legacy_store = 10; + bytes world_address = 11; +} + +message Entity { + bytes hashed_keys = 1; + repeated Struct models = 2; + uint64 created_at = 3; + uint64 updated_at = 4; + uint64 executed_at = 5; + bytes world_address = 6; +} + +message Event { + repeated bytes keys = 1; + repeated bytes data = 2; + bytes transaction_hash = 3; +} + +message Query { + Clause clause = 1; + bool no_hashed_keys = 2; + repeated string models = 3; + Pagination pagination = 4; + bool historical = 5; + repeated bytes world_addresses = 6; +} + +message EventQuery { + KeysClause keys = 1; + Pagination pagination = 2; +} + +message Clause { + oneof clause_type { + HashedKeysClause hashed_keys = 1; + KeysClause keys = 2; + MemberClause member = 3; + CompositeClause composite = 4; + } +} + +message KeysClause { + repeated bytes keys = 2; + PatternMatching pattern_matching = 3; + repeated string models = 4; +} + +message HashedKeysClause { + repeated bytes hashed_keys = 1; +} + +message MemberValue { + oneof value_type { + Primitive primitive = 1; + string string = 2; + MemberValueList list = 3; + } +} + +message MemberValueList { + repeated MemberValue values = 1; +} + +message MemberClause { + string model = 2; + string member = 3; + ComparisonOperator operator = 4; + MemberValue value = 5; +} + +message CompositeClause { + LogicalOperator operator = 3; + repeated Clause clauses = 4; +} + +enum PatternMatching { + FixedLen = 0; + VariableLen = 1; +} + +enum LogicalOperator { + AND = 0; + OR = 1; +} + +enum ComparisonOperator { + EQ = 0; + NEQ = 1; + GT = 2; + GTE = 3; + LT = 4; + LTE = 5; + IN = 6; + NOT_IN = 7; + CONTAINS = 8; + CONTAINS_ALL = 9; + CONTAINS_ANY = 10; + ARRAY_LENGTH_EQ = 11; + ARRAY_LENGTH_GT = 12; + ARRAY_LENGTH_LT = 13; +} + +message OrderBy { + string field = 1; + OrderDirection direction = 2; +} + +enum OrderDirection { + ASC = 0; + DESC = 1; +} + +enum PaginationDirection { + FORWARD = 0; + BACKWARD = 1; +} + +message Pagination { + string cursor = 1; + uint32 limit = 2; + PaginationDirection direction = 3; + repeated OrderBy order_by = 4; +} + +enum ContractType { + WORLD = 0; + ERC20 = 1; + ERC721 = 2; + ERC1155 = 3; + UDC = 4; + OTHER = 5; +} + +message Contract { + bytes contract_address = 1; + ContractType contract_type = 2; + optional uint64 head = 3; + optional uint64 tps = 4; + optional uint64 last_block_timestamp = 5; + optional bytes last_pending_block_tx = 6; + uint64 updated_at = 7; + uint64 created_at = 8; +} + +message ContractQuery { + repeated bytes contract_addresses = 1; + repeated ContractType contract_types = 2; +} diff --git a/crates/torii-ecs-sink/proto/world.proto b/crates/torii-ecs-sink/proto/world.proto new file mode 100644 index 00000000..803c59ae --- /dev/null +++ b/crates/torii-ecs-sink/proto/world.proto @@ -0,0 +1,76 @@ +syntax = "proto3"; +package world; + +import "types.proto"; +import "google/protobuf/empty.proto"; + +service World { + rpc SubscribeContracts (SubscribeContractsRequest) returns (stream SubscribeContractsResponse); + rpc Worlds (WorldsRequest) returns (WorldsResponse); + rpc SubscribeEntities (SubscribeEntitiesRequest) returns (stream SubscribeEntityResponse); + rpc UpdateEntitiesSubscription (UpdateEntitiesSubscriptionRequest) returns (google.protobuf.Empty); + rpc RetrieveEntities (RetrieveEntitiesRequest) returns (RetrieveEntitiesResponse); + rpc SubscribeEventMessages (SubscribeEntitiesRequest) returns (stream SubscribeEntityResponse); + rpc UpdateEventMessagesSubscription (UpdateEntitiesSubscriptionRequest) returns (google.protobuf.Empty); + rpc RetrieveEventMessages (RetrieveEntitiesRequest) returns (RetrieveEntitiesResponse); + rpc RetrieveEvents (RetrieveEventsRequest) returns (RetrieveEventsResponse); + rpc SubscribeEvents (SubscribeEventsRequest) returns (stream SubscribeEventsResponse); +} + +message SubscribeContractsRequest { + types.ContractQuery query = 1; +} + +message SubscribeContractsResponse { + types.Contract contract = 1; +} + +message WorldsRequest { + repeated bytes world_addresses = 1; +} + +message WorldsResponse { + repeated types.World worlds = 1; +} + +message SubscribeEntitiesRequest { + types.Clause clause = 1; + repeated bytes world_addresses = 2; +} + +message UpdateEntitiesSubscriptionRequest { + uint64 subscription_id = 1; + types.Clause clause = 2; + repeated bytes world_addresses = 3; +} + +message SubscribeEntityResponse { + types.Entity entity = 1; + uint64 subscription_id = 2; +} + +message RetrieveEntitiesRequest { + types.Query query = 1; +} + +message RetrieveEntitiesResponse { + string next_cursor = 1; + repeated types.Entity entities = 2; +} + +message RetrieveEventsRequest { + types.EventQuery query = 1; +} + +message RetrieveEventsResponse { + string next_cursor = 1; + repeated types.Event events = 2; +} + +message SubscribeEventsRequest { + repeated types.KeysClause keys = 1; +} + +message SubscribeEventsResponse { + types.Event event = 1; +} diff --git a/crates/torii-ecs-sink/src/grpc_service.rs b/crates/torii-ecs-sink/src/grpc_service.rs new file mode 100644 index 00000000..59014e16 --- /dev/null +++ b/crates/torii-ecs-sink/src/grpc_service.rs @@ -0,0 +1,3473 @@ +use std::collections::{HashMap, HashSet}; +use std::str::FromStr; +use std::sync::Arc; +use std::time::Instant; + +use anyhow::{anyhow, Result}; +use chrono::Utc; +use introspect_types::serialize::ToCairoDeSeFrom; +use introspect_types::serialize_def::CairoTypeSerialization; +use introspect_types::{ + Attributes, CairoDeserializer, ColumnDef, ColumnInfo, PrimaryTypeDef, ResultDef, TupleDef, + TypeDef, +}; +use serde::ser::SerializeMap; +use serde::Serializer; +use serde_json::{Map, Serializer as JsonSerializer, Value}; +use sqlx::{ + any::AnyPoolOptions, postgres::PgPoolOptions, sqlite::SqliteConnectOptions, + sqlite::SqlitePoolOptions, Any, ConnectOptions, Pool, QueryBuilder, Row, +}; +use starknet::core::types::Felt; +use tokio::sync::mpsc::error::TrySendError; +use tokio::sync::{mpsc, Mutex}; +use tokio_stream::wrappers::ReceiverStream; +use tonic::{Request, Response, Status}; +use torii_dojo::store::postgres::PgStore; +use torii_dojo::store::sqlite::SqliteStore; +use torii_dojo::store::DojoStoreTrait; +use torii_dojo::DojoTable; +use torii_introspect::events::{CreateTable, Record, UpdateTable}; +use torii_introspect::schema::TableSchema; + +use crate::proto::types::{ + self, clause::ClauseType, member_value::ValueType, ComparisonOperator, ContractType, + LogicalOperator, PaginationDirection, PatternMatching, +}; +use crate::proto::world::{ + world_server::World, RetrieveEntitiesRequest, RetrieveEntitiesResponse, RetrieveEventsRequest, + RetrieveEventsResponse, SubscribeContractsRequest, SubscribeContractsResponse, + SubscribeEntitiesRequest, SubscribeEntityResponse, SubscribeEventsRequest, + SubscribeEventsResponse, UpdateEntitiesSubscriptionRequest, WorldsRequest, WorldsResponse, +}; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum TableKind { + Entity, + EventMessage, +} + +impl TableKind { + fn as_str(self) -> &'static str { + match self { + Self::Entity => "entity", + Self::EventMessage => "event_message", + } + } + + fn from_str(value: &str) -> Option { + match value { + "entity" => Some(Self::Entity), + "event_message" => Some(Self::EventMessage), + _ => None, + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum DbBackend { + Sqlite, + Postgres, +} + +impl DbBackend { + fn detect(database_url: &str) -> Self { + if database_url.starts_with("postgres://") || database_url.starts_with("postgresql://") { + Self::Postgres + } else { + Self::Sqlite + } + } +} + +#[derive(Clone)] +pub struct EcsService { + state: Arc, +} + +struct EcsState { + pool: Pool, + backend: DbBackend, + database_url: String, + managed_tables: Mutex>>>, + entity_subscriptions: Mutex>, + event_message_subscriptions: Mutex>, + event_subscriptions: Mutex>, + contract_subscriptions: Mutex>, +} + +struct EntitySubscription { + clause: Option, + world_addresses: HashSet>, + sender: mpsc::Sender>, +} + +struct EventSubscription { + keys: Vec, + sender: mpsc::Sender>, +} + +struct ContractSubscription { + query: types::ContractQuery, + sender: mpsc::Sender>, +} + +#[derive(Clone)] +struct ManagedTable { + world_address: Felt, + table: DojoTable, + kind: TableKind, +} + +#[derive(Default)] +struct EntityAggregate { + world_address: Vec, + hashed_keys: Vec, + models: Vec, + created_at: u64, + updated_at: u64, + executed_at: u64, +} + +impl EntityAggregate { + fn into_proto(self) -> types::Entity { + types::Entity { + hashed_keys: self.hashed_keys, + models: self.models, + created_at: self.created_at, + updated_at: self.updated_at, + executed_at: self.executed_at, + world_address: self.world_address, + } + } +} + +impl EcsService { + pub async fn new(database_url: &str, max_connections: Option) -> Result { + sqlx::any::install_default_drivers(); + + let backend = DbBackend::detect(database_url); + let database_url = match backend { + DbBackend::Postgres => database_url.to_string(), + DbBackend::Sqlite => sqlite_url(database_url)?, + }; + + let pool = AnyPoolOptions::new() + .max_connections(max_connections.unwrap_or(if backend == DbBackend::Sqlite { + 1 + } else { + 5 + })) + .connect(&database_url) + .await?; + + let service = Self { + state: Arc::new(EcsState { + pool, + backend, + database_url, + managed_tables: Mutex::new(None), + entity_subscriptions: Mutex::new(HashMap::new()), + event_message_subscriptions: Mutex::new(HashMap::new()), + event_subscriptions: Mutex::new(HashMap::new()), + contract_subscriptions: Mutex::new(HashMap::new()), + }), + }; + service.initialize().await?; + Ok(service) + } + + async fn initialize(&self) -> Result<()> { + if self.state.backend == DbBackend::Sqlite { + sqlx::query("PRAGMA journal_mode=WAL") + .execute(&self.state.pool) + .await + .ok(); + } + + sqlx::query( + "CREATE TABLE IF NOT EXISTS torii_ecs_contracts ( + contract_address TEXT PRIMARY KEY, + contract_type INTEGER NOT NULL, + head BIGINT, + tps BIGINT, + last_block_timestamp BIGINT, + last_pending_block_tx TEXT, + updated_at BIGINT NOT NULL, + created_at BIGINT NOT NULL + )", + ) + .execute(&self.state.pool) + .await?; + + sqlx::query( + "CREATE TABLE IF NOT EXISTS torii_ecs_table_kinds ( + table_id TEXT PRIMARY KEY, + world_address TEXT NOT NULL, + kind TEXT NOT NULL, + updated_at BIGINT NOT NULL + )", + ) + .execute(&self.state.pool) + .await?; + + let entity_meta_sql = match self.state.backend { + DbBackend::Sqlite => { + "CREATE TABLE IF NOT EXISTS torii_ecs_entity_meta ( + kind TEXT NOT NULL, + world_address TEXT NOT NULL, + table_id TEXT NOT NULL, + entity_id TEXT NOT NULL, + created_at BIGINT NOT NULL, + updated_at BIGINT NOT NULL, + executed_at BIGINT NOT NULL, + deleted INTEGER NOT NULL DEFAULT 0, + PRIMARY KEY(kind, world_address, table_id, entity_id) + )" + } + DbBackend::Postgres => { + "CREATE TABLE IF NOT EXISTS torii_ecs_entity_meta ( + kind TEXT NOT NULL, + world_address TEXT NOT NULL, + table_id TEXT NOT NULL, + entity_id TEXT NOT NULL, + created_at BIGINT NOT NULL, + updated_at BIGINT NOT NULL, + executed_at BIGINT NOT NULL, + deleted BOOLEAN NOT NULL DEFAULT FALSE, + PRIMARY KEY(kind, world_address, table_id, entity_id) + )" + } + }; + sqlx::query(entity_meta_sql) + .execute(&self.state.pool) + .await?; + + sqlx::query( + "CREATE TABLE IF NOT EXISTS torii_ecs_events ( + event_id TEXT PRIMARY KEY, + world_address TEXT NOT NULL, + block_number BIGINT NOT NULL, + executed_at BIGINT NOT NULL, + transaction_hash TEXT NOT NULL, + keys_json TEXT NOT NULL, + data_json TEXT NOT NULL + )", + ) + .execute(&self.state.pool) + .await?; + + sqlx::query( + "CREATE TABLE IF NOT EXISTS torii_ecs_entity_models ( + kind TEXT NOT NULL, + world_address TEXT NOT NULL, + table_id TEXT NOT NULL, + entity_id TEXT NOT NULL, + row_json TEXT NOT NULL, + updated_at BIGINT NOT NULL, + PRIMARY KEY(kind, world_address, table_id, entity_id) + )", + ) + .execute(&self.state.pool) + .await?; + + sqlx::query( + "CREATE INDEX IF NOT EXISTS torii_ecs_entity_meta_lookup_idx + ON torii_ecs_entity_meta(kind, world_address, entity_id)", + ) + .execute(&self.state.pool) + .await?; + + sqlx::query( + "CREATE INDEX IF NOT EXISTS torii_ecs_entity_meta_table_lookup_idx + ON torii_ecs_entity_meta(kind, table_id, world_address, entity_id, deleted)", + ) + .execute(&self.state.pool) + .await?; + + sqlx::query( + "CREATE INDEX IF NOT EXISTS torii_ecs_entity_meta_keyset_idx + ON torii_ecs_entity_meta(kind, deleted, entity_id, world_address)", + ) + .execute(&self.state.pool) + .await?; + + sqlx::query( + "CREATE INDEX IF NOT EXISTS torii_ecs_entity_models_lookup_idx + ON torii_ecs_entity_models(kind, world_address, entity_id)", + ) + .execute(&self.state.pool) + .await?; + + sqlx::query( + "CREATE INDEX IF NOT EXISTS torii_ecs_entity_models_table_lookup_idx + ON torii_ecs_entity_models(kind, table_id, world_address, entity_id)", + ) + .execute(&self.state.pool) + .await?; + + sqlx::query( + "CREATE INDEX IF NOT EXISTS torii_ecs_entity_models_key_lookup_idx + ON torii_ecs_entity_models(kind, world_address, entity_id)", + ) + .execute(&self.state.pool) + .await?; + + sqlx::query( + "CREATE INDEX IF NOT EXISTS torii_ecs_events_cursor_idx + ON torii_ecs_events(event_id)", + ) + .execute(&self.state.pool) + .await?; + + Ok(()) + } + + pub async fn record_contract_progress( + &self, + contract_address: Felt, + contract_type: ContractType, + head: u64, + executed_at: u64, + last_pending_block_tx: Option, + ) -> Result<()> { + let now = Utc::now().timestamp() as u64; + let sql = match self.state.backend { + DbBackend::Sqlite => { + "INSERT INTO torii_ecs_contracts ( + contract_address, contract_type, head, tps, last_block_timestamp, + last_pending_block_tx, updated_at, created_at + ) VALUES (?1, ?2, ?3, NULL, ?4, ?5, ?6, ?6) + ON CONFLICT(contract_address) DO UPDATE SET + contract_type = excluded.contract_type, + head = excluded.head, + last_block_timestamp = excluded.last_block_timestamp, + last_pending_block_tx = excluded.last_pending_block_tx, + updated_at = excluded.updated_at" + } + DbBackend::Postgres => { + "INSERT INTO torii_ecs_contracts ( + contract_address, contract_type, head, tps, last_block_timestamp, + last_pending_block_tx, updated_at, created_at + ) VALUES ($1, $2, $3, NULL, $4, $5, $6, $6) + ON CONFLICT(contract_address) DO UPDATE SET + contract_type = EXCLUDED.contract_type, + head = EXCLUDED.head, + last_block_timestamp = EXCLUDED.last_block_timestamp, + last_pending_block_tx = EXCLUDED.last_pending_block_tx, + updated_at = EXCLUDED.updated_at" + } + }; + sqlx::query(sql) + .bind(felt_hex(contract_address)) + .bind(contract_type as i32) + .bind(head as i64) + .bind(executed_at as i64) + .bind(last_pending_block_tx.map(felt_hex)) + .bind(now as i64) + .execute(&self.state.pool) + .await?; + + if self.state.contract_subscriptions.lock().await.is_empty() { + return Ok(()); + } + + let contract = self + .load_contracts(&types::ContractQuery { + contract_addresses: vec![contract_address.to_bytes_be().to_vec()], + contract_types: vec![], + }) + .await? + .into_iter() + .next(); + + if let Some(contract) = contract { + self.publish_contract_update(contract).await; + } + Ok(()) + } + + pub async fn record_table_kind( + &self, + world_address: Felt, + table_id: Felt, + kind: TableKind, + ) -> Result<()> { + let now = Utc::now().timestamp(); + let sql = match self.state.backend { + DbBackend::Sqlite => { + "INSERT INTO torii_ecs_table_kinds (table_id, world_address, kind, updated_at) + VALUES (?1, ?2, ?3, ?4) + ON CONFLICT(table_id) DO UPDATE SET + world_address = excluded.world_address, + kind = excluded.kind, + updated_at = excluded.updated_at" + } + DbBackend::Postgres => { + "INSERT INTO torii_ecs_table_kinds (table_id, world_address, kind, updated_at) + VALUES ($1, $2, $3, $4) + ON CONFLICT(table_id) DO UPDATE SET + world_address = EXCLUDED.world_address, + kind = EXCLUDED.kind, + updated_at = EXCLUDED.updated_at" + } + }; + sqlx::query(sql) + .bind(felt_hex(table_id)) + .bind(felt_hex(world_address)) + .bind(kind.as_str()) + .bind(now) + .execute(&self.state.pool) + .await?; + self.update_managed_table_kind(table_id, world_address, kind) + .await; + Ok(()) + } + + pub async fn cache_created_table(&self, world_address: Felt, table: &CreateTable) { + let schema = TableSchema { + id: table.id, + name: table.name.clone(), + attributes: table.attributes.clone(), + primary: table.primary.clone(), + columns: table.columns.clone(), + }; + self.put_managed_table(world_address, DojoTable::from(schema)) + .await; + } + + pub async fn cache_updated_table(&self, world_address: Felt, table: &UpdateTable) { + let schema = TableSchema { + id: table.id, + name: table.name.clone(), + attributes: table.attributes.clone(), + primary: table.primary.clone(), + columns: table.columns.clone(), + }; + self.put_managed_table(world_address, DojoTable::from(schema)) + .await; + } + + pub async fn upsert_entity_meta( + &self, + kind: TableKind, + world_address: Felt, + table_id: Felt, + entity_id: Felt, + executed_at: u64, + deleted: bool, + ) -> Result<()> { + let sql = match self.state.backend { + DbBackend::Sqlite => { + "INSERT INTO torii_ecs_entity_meta ( + kind, world_address, table_id, entity_id, created_at, updated_at, executed_at, deleted + ) VALUES (?1, ?2, ?3, ?4, ?5, ?5, ?5, ?6) + ON CONFLICT(kind, world_address, table_id, entity_id) DO UPDATE SET + updated_at = excluded.updated_at, + executed_at = excluded.executed_at, + deleted = excluded.deleted" + } + DbBackend::Postgres => { + "INSERT INTO torii_ecs_entity_meta ( + kind, world_address, table_id, entity_id, created_at, updated_at, executed_at, deleted + ) VALUES ($1, $2, $3, $4, $5, $5, $5, $6) + ON CONFLICT(kind, world_address, table_id, entity_id) DO UPDATE SET + updated_at = EXCLUDED.updated_at, + executed_at = EXCLUDED.executed_at, + deleted = EXCLUDED.deleted" + } + }; + let mut query = sqlx::query(sql) + .bind(kind.as_str()) + .bind(felt_hex(world_address)) + .bind(felt_hex(table_id)) + .bind(felt_hex(entity_id)) + .bind(executed_at as i64); + query = match self.state.backend { + DbBackend::Sqlite => query.bind(i64::from(deleted)), + DbBackend::Postgres => query.bind(deleted), + }; + query.execute(&self.state.pool).await?; + Ok(()) + } + + pub async fn upsert_entity_model( + &self, + kind: TableKind, + world_address: Felt, + table_id: Felt, + columns: &[Felt], + record: &Record, + executed_at: u64, + ) -> Result<()> { + let Some(table) = self.load_managed_table(table_id).await? else { + return Ok(()); + }; + let row = record_to_json_map(&table.table, columns, record)?; + let row_json = serde_json::to_string(&row)?; + let sql = match self.state.backend { + DbBackend::Sqlite => { + "INSERT INTO torii_ecs_entity_models ( + kind, world_address, table_id, entity_id, row_json, updated_at + ) VALUES (?1, ?2, ?3, ?4, ?5, ?6) + ON CONFLICT(kind, world_address, table_id, entity_id) DO UPDATE SET + row_json = excluded.row_json, + updated_at = excluded.updated_at" + } + DbBackend::Postgres => { + "INSERT INTO torii_ecs_entity_models ( + kind, world_address, table_id, entity_id, row_json, updated_at + ) VALUES ($1, $2, $3, $4, $5, $6) + ON CONFLICT(kind, world_address, table_id, entity_id) DO UPDATE SET + row_json = EXCLUDED.row_json, + updated_at = EXCLUDED.updated_at" + } + }; + sqlx::query(sql) + .bind(kind.as_str()) + .bind(felt_hex(world_address)) + .bind(felt_hex(table_id)) + .bind(felt_hex(Felt::from_bytes_be(&record.id))) + .bind(row_json) + .bind(executed_at as i64) + .execute(&self.state.pool) + .await?; + Ok(()) + } + + pub async fn delete_entity_model( + &self, + kind: TableKind, + world_address: Felt, + table_id: Felt, + entity_id: Felt, + ) -> Result<()> { + let sql = match self.state.backend { + DbBackend::Sqlite => { + "DELETE FROM torii_ecs_entity_models + WHERE kind = ?1 AND world_address = ?2 AND table_id = ?3 AND entity_id = ?4" + } + DbBackend::Postgres => { + "DELETE FROM torii_ecs_entity_models + WHERE kind = $1 AND world_address = $2 AND table_id = $3 AND entity_id = $4" + } + }; + sqlx::query(sql) + .bind(kind.as_str()) + .bind(felt_hex(world_address)) + .bind(felt_hex(table_id)) + .bind(felt_hex(entity_id)) + .execute(&self.state.pool) + .await?; + Ok(()) + } + + pub async fn table_kind(&self, table_id: Felt) -> Result { + let sql = match self.state.backend { + DbBackend::Sqlite => "SELECT kind FROM torii_ecs_table_kinds WHERE table_id = ?1", + DbBackend::Postgres => "SELECT kind FROM torii_ecs_table_kinds WHERE table_id = $1", + }; + let row = sqlx::query(sql) + .bind(felt_hex(table_id)) + .fetch_optional(&self.state.pool) + .await?; + Ok(row + .and_then(|row| row.try_get::("kind").ok()) + .and_then(|kind| TableKind::from_str(&kind)) + .unwrap_or(TableKind::Entity)) + } + + pub async fn store_event( + &self, + world_address: Felt, + transaction_hash: Felt, + block_number: u64, + executed_at: u64, + keys: &[Felt], + data: &[Felt], + ordinal: usize, + ) -> Result<()> { + let event_id = format!( + "{}:{}:{}", + felt_hex(world_address), + felt_hex(transaction_hash), + ordinal + ); + let keys_json = + serde_json::to_string(&keys.iter().map(|felt| felt_hex(*felt)).collect::>())?; + let data_json = + serde_json::to_string(&data.iter().map(|felt| felt_hex(*felt)).collect::>())?; + + let sql = match self.state.backend { + DbBackend::Sqlite => { + "INSERT INTO torii_ecs_events ( + event_id, world_address, block_number, executed_at, transaction_hash, keys_json, data_json + ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7) + ON CONFLICT(event_id) DO NOTHING" + } + DbBackend::Postgres => { + "INSERT INTO torii_ecs_events ( + event_id, world_address, block_number, executed_at, transaction_hash, keys_json, data_json + ) VALUES ($1, $2, $3, $4, $5, $6, $7) + ON CONFLICT(event_id) DO NOTHING" + } + }; + sqlx::query(sql) + .bind(event_id) + .bind(felt_hex(world_address)) + .bind(block_number as i64) + .bind(executed_at as i64) + .bind(felt_hex(transaction_hash)) + .bind(keys_json) + .bind(data_json) + .execute(&self.state.pool) + .await?; + + self.publish_event_update(types::Event { + keys: keys + .iter() + .map(|felt| felt.to_bytes_be().to_vec()) + .collect(), + data: data + .iter() + .map(|felt| felt.to_bytes_be().to_vec()) + .collect(), + transaction_hash: transaction_hash.to_bytes_be().to_vec(), + }) + .await; + + Ok(()) + } + + pub async fn publish_entity_update( + &self, + kind: TableKind, + world_address: Felt, + entity_id: Felt, + ) -> Result<()> { + let start = Instant::now(); + let subscriptions = match kind { + TableKind::Entity => &self.state.entity_subscriptions, + TableKind::EventMessage => &self.state.event_message_subscriptions, + }; + if subscriptions.lock().await.is_empty() { + return Ok(()); + } + + let entity = self + .load_entity_by_id(kind, world_address, entity_id) + .await? + .map(EntityAggregate::into_proto); + let Some(entity) = entity else { + return Ok(()); + }; + + let (checked, targets) = { + let subscriptions = subscriptions.lock().await; + let checked = subscriptions.len(); + let targets = subscriptions + .iter() + .filter_map(|(&subscription_id, subscription)| { + if !subscription.world_addresses.is_empty() + && !subscription.world_addresses.contains(&entity.world_address) + { + return None; + } + if let Some(clause) = &subscription.clause { + if !entity_matches_clause(&entity, clause) { + return None; + } + } + Some((subscription_id, subscription.sender.clone())) + }) + .collect::>(); + (checked, targets) + }; + let matched = targets.len(); + + let mut closed = Vec::new(); + for (subscription_id, sender) in targets { + if let Err(err) = sender.try_send(Ok(SubscribeEntityResponse { + entity: Some(entity.clone()), + subscription_id, + })) { + if matches!(err, TrySendError::Closed(_)) { + closed.push(subscription_id); + } + } + } + + if !closed.is_empty() { + let mut subscriptions = subscriptions.lock().await; + for subscription_id in closed { + subscriptions.remove(&subscription_id); + } + } + ::metrics::histogram!("torii_ecs_publish_entity_update_seconds") + .record(start.elapsed().as_secs_f64()); + ::metrics::counter!("torii_ecs_publish_entity_subscribers_checked_total") + .increment(checked as u64); + ::metrics::counter!("torii_ecs_publish_entity_subscribers_matched_total") + .increment(matched as u64); + Ok(()) + } + + async fn publish_event_update(&self, event: types::Event) { + let start = Instant::now(); + let (checked, targets) = { + let subscriptions = self.state.event_subscriptions.lock().await; + if subscriptions.is_empty() { + return; + } + let checked = subscriptions.len(); + let targets = subscriptions + .iter() + .filter_map(|(&subscription_id, subscription)| { + if !match_keys(&event.keys, &subscription.keys) { + return None; + } + Some((subscription_id, subscription.sender.clone())) + }) + .collect::>(); + (checked, targets) + }; + let matched = targets.len(); + + let mut closed = Vec::new(); + for (subscription_id, sender) in targets { + if let Err(err) = sender.try_send(Ok(SubscribeEventsResponse { + event: Some(event.clone()), + })) { + if matches!(err, TrySendError::Closed(_)) { + closed.push(subscription_id); + } + } + } + + if !closed.is_empty() { + let mut subscriptions = self.state.event_subscriptions.lock().await; + for subscription_id in closed { + subscriptions.remove(&subscription_id); + } + } + ::metrics::histogram!("torii_ecs_publish_event_update_seconds") + .record(start.elapsed().as_secs_f64()); + ::metrics::counter!("torii_ecs_publish_event_subscribers_checked_total") + .increment(checked as u64); + ::metrics::counter!("torii_ecs_publish_event_subscribers_matched_total") + .increment(matched as u64); + } + + async fn publish_contract_update(&self, contract: types::Contract) { + let start = Instant::now(); + let (checked, targets) = { + let subscriptions = self.state.contract_subscriptions.lock().await; + if subscriptions.is_empty() { + return; + } + let checked = subscriptions.len(); + let targets = subscriptions + .iter() + .filter_map(|(&subscription_id, subscription)| { + if !contract_matches_query(&contract, &subscription.query) { + return None; + } + Some((subscription_id, subscription.sender.clone())) + }) + .collect::>(); + (checked, targets) + }; + let matched = targets.len(); + + let mut closed = Vec::new(); + for (subscription_id, sender) in targets { + if let Err(err) = sender.try_send(Ok(SubscribeContractsResponse { + contract: Some(contract.clone()), + })) { + if matches!(err, TrySendError::Closed(_)) { + closed.push(subscription_id); + } + } + } + + if !closed.is_empty() { + let mut subscriptions = self.state.contract_subscriptions.lock().await; + for subscription_id in closed { + subscriptions.remove(&subscription_id); + } + } + ::metrics::histogram!("torii_ecs_publish_contract_update_seconds") + .record(start.elapsed().as_secs_f64()); + ::metrics::counter!("torii_ecs_publish_contract_subscribers_checked_total") + .increment(checked as u64); + ::metrics::counter!("torii_ecs_publish_contract_subscribers_matched_total") + .increment(matched as u64); + } + + async fn load_contracts(&self, query: &types::ContractQuery) -> Result> { + let mut builder = QueryBuilder::::new( + "SELECT contract_address, contract_type, head, tps, last_block_timestamp, \ + last_pending_block_tx, updated_at, created_at \ + FROM torii_ecs_contracts", + ); + if !query.contract_addresses.is_empty() || !query.contract_types.is_empty() { + builder.push(" WHERE "); + if !query.contract_addresses.is_empty() { + builder.push("contract_address IN ("); + { + let mut separated = builder.separated(", "); + for address in &query.contract_addresses { + separated.push_bind(felt_hex(felt_from_bytes(address)?)); + } + } + builder.push(")"); + if !query.contract_types.is_empty() { + builder.push(" AND "); + } + } + if !query.contract_types.is_empty() { + builder.push("contract_type IN ("); + { + let mut separated = builder.separated(", "); + for contract_type in &query.contract_types { + separated.push_bind(*contract_type); + } + } + builder.push(")"); + } + } + + let rows = builder.build().fetch_all(&self.state.pool).await?; + let mut contracts = Vec::new(); + for row in rows { + let contract_address = row.try_get::("contract_address")?; + let contract_type = row.try_get::("contract_type")?; + let contract = types::Contract { + contract_address: felt_from_hex(&contract_address)?.to_bytes_be().to_vec(), + contract_type, + head: row + .try_get::, _>("head")? + .map(|value| value as u64), + tps: row + .try_get::, _>("tps")? + .map(|value| value as u64), + last_block_timestamp: row + .try_get::, _>("last_block_timestamp")? + .map(|value| value as u64), + last_pending_block_tx: row + .try_get::, _>("last_pending_block_tx")? + .map(|value| felt_from_hex(&value)) + .transpose()? + .map(|felt| felt.to_bytes_be().to_vec()), + updated_at: row.try_get::("updated_at")? as u64, + created_at: row.try_get::("created_at")? as u64, + }; + contracts.push(contract); + } + Ok(contracts) + } + + async fn load_worlds(&self, requested: &[Felt]) -> Result> { + let requested = requested.iter().copied().collect::>(); + let mut by_world: HashMap, Vec> = HashMap::new(); + for table in self.load_managed_tables(None).await? { + if !requested.is_empty() && !requested.contains(&table.world_address) { + continue; + } + by_world + .entry(table.world_address.to_bytes_be().to_vec()) + .or_default() + .push(model_from_table(&table)); + } + + let mut worlds = by_world + .into_iter() + .map(|(world_address, models)| types::World { + world_address: felt_from_bytes(&world_address) + .unwrap_or_default() + .to_string(), + models, + }) + .collect::>(); + worlds.sort_by(|a, b| a.world_address.cmp(&b.world_address)); + Ok(worlds) + } + + async fn put_managed_table(&self, world_address: Felt, table: DojoTable) { + if self.state.managed_tables.lock().await.is_none() { + let _ = self.load_managed_table_map().await; + } + let mut managed_tables = self.state.managed_tables.lock().await; + if let Some(tables) = managed_tables.as_ref() { + let mut next = (**tables).clone(); + let key = felt_hex(table.id); + let kind = next.get(&key).map_or(TableKind::Entity, |table| table.kind); + next.insert( + key, + ManagedTable { + world_address, + table, + kind, + }, + ); + *managed_tables = Some(Arc::new(next)); + } + } + + async fn update_managed_table_kind( + &self, + table_id: Felt, + world_address: Felt, + kind: TableKind, + ) { + let mut managed_tables = self.state.managed_tables.lock().await; + let Some(tables) = managed_tables.as_ref() else { + return; + }; + let mut next = (**tables).clone(); + let key = felt_hex(table_id); + if let Some(table) = next.get_mut(&key) { + table.world_address = world_address; + table.kind = kind; + *managed_tables = Some(Arc::new(next)); + } else { + *managed_tables = None; + } + } + + async fn load_managed_table(&self, table_id: Felt) -> Result> { + let key = felt_hex(table_id); + Ok(self.load_managed_table_map().await?.get(&key).cloned()) + } + + async fn load_managed_table_map(&self) -> Result>> { + { + let managed_tables = self.state.managed_tables.lock().await; + if let Some(tables) = managed_tables.as_ref() { + return Ok(Arc::clone(tables)); + } + } + + let kinds = self.load_table_kind_rows().await?; + let tables = self.load_dojo_tables().await?; + let mut managed = HashMap::with_capacity(tables.len()); + for table in tables { + let key = felt_hex(table.id); + let (world_address, table_kind) = if let Some((world, kind_name)) = kinds.get(&key) { + let kind = TableKind::from_str(kind_name).unwrap_or(TableKind::Entity); + (felt_from_hex(world)?, kind) + } else { + (Felt::ZERO, TableKind::Entity) + }; + managed.insert( + key, + ManagedTable { + world_address, + table, + kind: table_kind, + }, + ); + } + + let mut managed_tables = self.state.managed_tables.lock().await; + let managed = Arc::new(managed); + *managed_tables = Some(Arc::clone(&managed)); + Ok(managed) + } + + async fn load_table_kind_rows(&self) -> Result> { + let table_kind_sql = "SELECT table_id, world_address, kind FROM torii_ecs_table_kinds"; + let kind_rows = sqlx::query(table_kind_sql) + .fetch_all(&self.state.pool) + .await?; + let mut kinds = HashMap::new(); + for row in kind_rows { + kinds.insert( + row.try_get::("table_id")?, + ( + row.try_get::("world_address")?, + row.try_get::("kind")?, + ), + ); + } + Ok(kinds) + } + + async fn load_dojo_tables(&self) -> Result> { + match self.state.backend { + DbBackend::Sqlite => { + let pool = SqlitePoolOptions::new() + .max_connections(1) + .connect_with(SqliteConnectOptions::from_str(&self.state.database_url)?) + .await?; + let store = SqliteStore(Arc::new(pool)); + Ok(store.read_tables(&[]).await?) + } + DbBackend::Postgres => { + let pool = PgPoolOptions::new() + .max_connections(1) + .connect(&self.state.database_url) + .await?; + let store = PgStore(Arc::new(pool)); + Ok(store.read_tables(&[]).await?) + } + } + } + + async fn load_managed_tables(&self, kind: Option) -> Result> { + Ok(self + .load_managed_table_map() + .await? + .values() + .filter(|table| kind.is_none_or(|required| table.kind == required)) + .cloned() + .collect()) + } + + async fn load_entity_page( + &self, + kind: TableKind, + query: &types::Query, + ) -> Result<(Vec, String)> { + let world_filters = query + .world_addresses + .iter() + .map(|bytes| felt_from_bytes(bytes)) + .collect::>>()?; + let world_filter_set = world_filters.iter().copied().collect::>(); + let model_filter_set = query + .models + .iter() + .map(String::as_str) + .collect::>(); + let models = self + .load_managed_tables(Some(kind)) + .await? + .into_iter() + .filter(|table| { + (world_filter_set.is_empty() || world_filter_set.contains(&table.world_address)) + && (model_filter_set.is_empty() + || model_filter_set.contains(table.table.name.as_str())) + }) + .collect::>(); + if models.is_empty() { + return Ok((Vec::new(), String::new())); + } + + let model_map = models + .into_iter() + .map(|table| (felt_hex(table.table.id), table)) + .collect::>(); + let table_ids = model_map.keys().cloned().collect::>(); + let table_id_filter = if query.models.is_empty() { + None + } else { + Some(table_ids.as_slice()) + }; + let member_pushdown = member_pushdown_from_clause(query.clause.as_ref(), &model_map); + let target_limit = query.pagination.as_ref().map_or(100, |pagination| { + if pagination.limit == 0 { + 100 + } else { + pagination.limit as usize + } + }); + + let direction = query + .pagination + .as_ref() + .map_or(PaginationDirection::Forward as i32, |pagination| { + pagination.direction + }); + let cursor = query + .pagination + .as_ref() + .map_or(String::new(), |pagination| pagination.cursor.clone()); + + let keys_start = Instant::now(); + let mut candidate_keys = self + .load_entity_page_keys( + kind, + &world_filters, + table_id_filter, + member_pushdown.as_ref(), + &cursor, + direction, + target_limit.saturating_add(1), + ) + .await?; + ::metrics::histogram!("torii_ecs_load_entity_page_meta_seconds") + .record(keys_start.elapsed().as_secs_f64()); + + let has_more = target_limit != usize::MAX && candidate_keys.len() > target_limit; + if has_more { + candidate_keys.truncate(target_limit); + } + + let meta_start = Instant::now(); + let candidate_meta = self + .load_entity_meta_for_keys(kind, table_id_filter, candidate_keys.as_slice()) + .await?; + ::metrics::histogram!("torii_ecs_load_entity_page_meta_rows_seconds") + .record(meta_start.elapsed().as_secs_f64()); + ::metrics::gauge!("torii_ecs_load_entity_page_candidates").set(candidate_keys.len() as f64); + + let mut items = Vec::new(); + for chunk in candidate_keys.chunks(256) { + let rows_start = Instant::now(); + let rows = self + .load_entity_page_rows(kind, &world_filters, table_id_filter, chunk) + .await?; + ::metrics::histogram!("torii_ecs_load_entity_page_rows_seconds") + .record(rows_start.elapsed().as_secs_f64()); + ::metrics::counter!("torii_ecs_load_entity_page_rows_total") + .increment(rows.len() as u64); + let row_map = rows.into_iter().fold( + HashMap::<(String, String), Vec<(String, String)>>::new(), + |mut acc, row| { + acc.entry((row.world_address, row.entity_id)) + .or_default() + .push((row.table_id, row.row_json)); + acc + }, + ); + + for key in chunk { + let Some(meta) = candidate_meta.get(key) else { + continue; + }; + let Some(entity) = + build_entity_from_snapshot(key, meta, row_map.get(key), &model_map)? + else { + continue; + }; + let entity = entity.into_proto(); + if query + .clause + .as_ref() + .is_some_and(|clause| !entity_matches_clause(&entity, clause)) + { + continue; + } + items.push(entity); + if items.len() >= target_limit { + break; + } + } + + if items.len() >= target_limit { + break; + } + } + + let next_cursor = if has_more { + candidate_keys + .last() + .map(|(_, entity_id)| entity_id.trim_start_matches("0x").to_string()) + .unwrap_or_default() + } else { + String::new() + }; + + if query.no_hashed_keys { + for entity in &mut items { + entity.hashed_keys.clear(); + } + } + Ok((items, next_cursor)) + } + + #[allow(dead_code)] + async fn load_entity_page_from_storage( + &self, + kind: TableKind, + query: &types::Query, + ) -> Result<(Vec, String)> { + let world_filters = query + .world_addresses + .iter() + .map(|bytes| felt_from_bytes(bytes)) + .collect::>>()?; + let world_filter_set = world_filters.iter().copied().collect::>(); + let model_filter_set = query + .models + .iter() + .map(String::as_str) + .collect::>(); + let models = self + .load_managed_tables(Some(kind)) + .await? + .into_iter() + .filter(|table| { + (world_filter_set.is_empty() || world_filter_set.contains(&table.world_address)) + && (model_filter_set.is_empty() + || model_filter_set.contains(table.table.name.as_str())) + }) + .collect::>(); + + let mut entities: HashMap<(Vec, Vec), EntityAggregate> = HashMap::new(); + for table in models { + let rows = self.fetch_table_rows(&table).await?; + let meta = self + .load_entity_meta(table.kind, table.world_address, table.table.id) + .await?; + for row in rows { + let entity_id = row + .get(&table.table.primary.name) + .cloned() + .unwrap_or(Value::Null); + let entity_felt = value_to_primary_felt(&entity_id, &table.table.primary.type_def) + .unwrap_or(Felt::ZERO); + let entity_key = entity_felt.to_bytes_be().to_vec(); + if meta + .get(&felt_hex(entity_felt)) + .is_some_and(|meta| meta.deleted) + { + continue; + } + + let model = row_to_model_struct(&table.table, &row)?; + let aggregate = entities + .entry(( + table.world_address.to_bytes_be().to_vec(), + entity_key.clone(), + )) + .or_insert_with(|| EntityAggregate { + world_address: table.world_address.to_bytes_be().to_vec(), + hashed_keys: entity_key.clone(), + ..EntityAggregate::default() + }); + aggregate.models.push(model); + if let Some(meta) = meta.get(&felt_hex(entity_felt)) { + if aggregate.created_at == 0 || meta.created_at < aggregate.created_at { + aggregate.created_at = meta.created_at; + } + aggregate.updated_at = aggregate.updated_at.max(meta.updated_at); + aggregate.executed_at = aggregate.executed_at.max(meta.executed_at); + } + } + } + + let mut items = entities + .into_values() + .map(EntityAggregate::into_proto) + .filter(|entity| { + query + .clause + .as_ref() + .is_none_or(|clause| entity_matches_clause(entity, clause)) + }) + .collect::>(); + items.sort_by(|a, b| a.hashed_keys.cmp(&b.hashed_keys)); + if query + .pagination + .as_ref() + .is_some_and(|p| p.direction == PaginationDirection::Backward as i32) + { + items.reverse(); + } + + if let Some(pagination) = &query.pagination { + if !pagination.cursor.is_empty() { + items.retain(|entity| { + let key = hex::encode(&entity.hashed_keys); + if pagination.direction == PaginationDirection::Backward as i32 { + key < pagination.cursor + } else { + key > pagination.cursor + } + }); + } + let limit = if pagination.limit == 0 { + 100 + } else { + pagination.limit as usize + }; + let next_cursor = items + .get(limit.saturating_sub(1)) + .map(|entity| hex::encode(&entity.hashed_keys)) + .unwrap_or_default(); + items.truncate(limit); + if query.no_hashed_keys { + for entity in &mut items { + entity.hashed_keys.clear(); + } + } + return Ok((items, next_cursor)); + } + + if query.no_hashed_keys { + for entity in &mut items { + entity.hashed_keys.clear(); + } + } + Ok((items, String::new())) + } + + async fn load_entity_by_id( + &self, + kind: TableKind, + world_address: Felt, + entity_id: Felt, + ) -> Result> { + if let Some(entity) = self + .load_entity_snapshot(kind, world_address, entity_id) + .await? + { + return Ok(Some(entity)); + } + + let query = types::Query { + clause: Some(types::Clause { + clause_type: Some(ClauseType::HashedKeys(types::HashedKeysClause { + hashed_keys: vec![entity_id.to_bytes_be().to_vec()], + })), + }), + no_hashed_keys: false, + models: vec![], + pagination: None, + historical: false, + world_addresses: vec![world_address.to_bytes_be().to_vec()], + }; + Ok(self + .load_entity_page(kind, &query) + .await? + .0 + .into_iter() + .next() + .map(|entity| EntityAggregate { + world_address: entity.world_address, + hashed_keys: entity.hashed_keys, + models: entity.models, + created_at: entity.created_at, + updated_at: entity.updated_at, + executed_at: entity.executed_at, + })) + } + + async fn load_entity_page_keys( + &self, + kind: TableKind, + world_filters: &[Felt], + table_ids: Option<&[String]>, + member_pushdown: Option<&MemberPushdown>, + cursor: &str, + direction: i32, + limit: usize, + ) -> Result> { + if table_ids.is_some_and(<[String]>::is_empty) { + return Ok(Vec::new()); + } + let limit = limit.clamp(1, 200); + + let cursor_op = if direction == PaginationDirection::Backward as i32 { + "<" + } else { + ">" + }; + let order = if direction == PaginationDirection::Backward as i32 { + "DESC" + } else { + "ASC" + }; + + let mut builder = QueryBuilder::::new( + "SELECT DISTINCT m.world_address, m.entity_id \ + FROM torii_ecs_entity_meta m \ + WHERE m.kind = ", + ); + builder.push_bind(kind.as_str()); + if let Some(table_ids) = table_ids { + builder.push(" AND m.table_id IN ("); + { + let mut separated = builder.separated(", "); + for table_id in table_ids { + separated.push_bind(table_id); + } + } + builder.push(")"); + } + if !world_filters.is_empty() { + builder.push(" AND m.world_address IN ("); + { + let mut separated = builder.separated(", "); + for world_address in world_filters { + separated.push_bind(felt_hex(*world_address)); + } + } + builder.push(")"); + } + match self.state.backend { + DbBackend::Sqlite => { + builder.push(" AND m.deleted = 0"); + } + DbBackend::Postgres => { + builder.push(" AND m.deleted = FALSE"); + } + } + if !cursor.is_empty() { + builder.push(" AND m.entity_id "); + builder.push(cursor_op); + builder.push(" "); + builder.push_bind(format!("0x{cursor}")); + } + if let Some(member_pushdown) = member_pushdown { + append_member_pushdown_sql(&mut builder, self.state.backend, member_pushdown); + } + builder.push(" ORDER BY m.entity_id "); + builder.push(order); + builder.push(", m.world_address "); + builder.push(order); + builder.push(" LIMIT "); + builder.push_bind(limit as i64); + + let rows = builder.build().fetch_all(&self.state.pool).await?; + rows.into_iter() + .map(|row| { + Ok(( + row.try_get::("world_address")?, + row.try_get::("entity_id")?, + )) + }) + .collect() + } + + async fn load_entity_meta_for_keys( + &self, + kind: TableKind, + table_ids: Option<&[String]>, + entity_keys: &[(String, String)], + ) -> Result>> { + if table_ids.is_some_and(<[String]>::is_empty) || entity_keys.is_empty() { + return Ok(HashMap::new()); + } + + let deleted_sql = match self.state.backend { + DbBackend::Sqlite => "CAST(m.deleted AS INTEGER)", + DbBackend::Postgres => "CASE WHEN m.deleted THEN 1 ELSE 0 END", + }; + + let mut builder = QueryBuilder::::new(format!( + "SELECT m.world_address, m.table_id, m.entity_id, m.created_at, m.updated_at, m.executed_at, \ + {deleted_sql} AS deleted \ + FROM torii_ecs_entity_meta m WHERE m.kind = " + )); + builder.push_bind(kind.as_str()); + if let Some(table_ids) = table_ids { + builder.push(" AND m.table_id IN ("); + { + let mut separated = builder.separated(", "); + for table_id in table_ids { + separated.push_bind(table_id); + } + } + builder.push(")"); + } + + let world_addresses = entity_keys + .iter() + .map(|(world_address, _)| world_address.clone()) + .collect::>(); + let entity_ids = entity_keys + .iter() + .map(|(_, entity_id)| entity_id.clone()) + .collect::>(); + let allowed = entity_keys.iter().cloned().collect::>(); + + builder.push(" AND m.world_address IN ("); + { + let mut separated = builder.separated(", "); + for world_address in &world_addresses { + separated.push_bind(world_address); + } + } + builder.push(")"); + builder.push(" AND m.entity_id IN ("); + { + let mut separated = builder.separated(", "); + for entity_id in &entity_ids { + separated.push_bind(entity_id); + } + } + builder.push(")"); + match self.state.backend { + DbBackend::Sqlite => { + builder.push(" AND m.deleted = 0"); + } + DbBackend::Postgres => { + builder.push(" AND m.deleted = FALSE"); + } + } + + let rows = builder.build().fetch_all(&self.state.pool).await?; + let mut meta = HashMap::new(); + for row in rows { + let world_address = row.try_get::("world_address")?; + let entity_id = row.try_get::("entity_id")?; + if !allowed.contains(&(world_address.clone(), entity_id.clone())) { + continue; + } + meta.entry((world_address, entity_id)) + .or_insert_with(HashMap::new) + .insert( + row.try_get::("table_id")?, + EntityMetaRow { + created_at: row.try_get::("created_at")? as u64, + updated_at: row.try_get::("updated_at")? as u64, + executed_at: row.try_get::("executed_at")? as u64, + deleted: row.try_get::("deleted")? != 0, + }, + ); + } + Ok(meta) + } + + async fn load_entity_page_rows( + &self, + kind: TableKind, + _world_filters: &[Felt], + table_ids: Option<&[String]>, + entity_keys: &[(String, String)], + ) -> Result> { + if entity_keys.is_empty() || table_ids.is_some_and(<[String]>::is_empty) { + return Ok(Vec::new()); + } + + let mut builder = QueryBuilder::::new( + "SELECT world_address, table_id, entity_id, row_json \ + FROM torii_ecs_entity_models WHERE kind = ", + ); + builder.push_bind(kind.as_str()); + if let Some(table_ids) = table_ids { + builder.push(" AND table_id IN ("); + { + let mut separated = builder.separated(", "); + for table_id in table_ids { + separated.push_bind(table_id); + } + } + builder.push(")"); + } + + let world_addresses = entity_keys + .iter() + .map(|(world_address, _)| world_address.clone()) + .collect::>(); + let entity_ids = entity_keys + .iter() + .map(|(_, entity_id)| entity_id.clone()) + .collect::>(); + let allowed = entity_keys.iter().cloned().collect::>(); + + builder.push(" AND world_address IN ("); + { + let mut separated = builder.separated(", "); + for world_address in &world_addresses { + separated.push_bind(world_address); + } + } + builder.push(")"); + builder.push(" AND entity_id IN ("); + { + let mut separated = builder.separated(", "); + for entity_id in &entity_ids { + separated.push_bind(entity_id); + } + } + builder.push(")"); + + let rows = builder.build().fetch_all(&self.state.pool).await?; + let mut output = Vec::new(); + for row in rows { + let world_address: String = row.try_get("world_address")?; + let entity_id: String = row.try_get("entity_id")?; + if !allowed.contains(&(world_address.clone(), entity_id.clone())) { + continue; + } + output.push(EntityModelRow { + world_address, + table_id: row.try_get("table_id")?, + entity_id, + row_json: row.try_get("row_json")?, + }); + } + Ok(output) + } + + async fn load_entity_snapshot( + &self, + kind: TableKind, + world_address: Felt, + entity_id: Felt, + ) -> Result> { + let sql = match self.state.backend { + DbBackend::Sqlite => { + "SELECT table_id, row_json + FROM torii_ecs_entity_models + WHERE kind = ?1 AND world_address = ?2 AND entity_id = ?3" + } + DbBackend::Postgres => { + "SELECT table_id, row_json + FROM torii_ecs_entity_models + WHERE kind = $1 AND world_address = $2 AND entity_id = $3" + } + }; + let rows = sqlx::query(sql) + .bind(kind.as_str()) + .bind(felt_hex(world_address)) + .bind(felt_hex(entity_id)) + .fetch_all(&self.state.pool) + .await?; + if rows.is_empty() { + return Ok(None); + } + + let meta = self + .load_entity_meta_rows(kind, world_address, entity_id) + .await?; + let managed_tables = self.load_managed_table_map().await?; + let mut aggregate = EntityAggregate { + world_address: world_address.to_bytes_be().to_vec(), + hashed_keys: entity_id.to_bytes_be().to_vec(), + ..EntityAggregate::default() + }; + + for row in rows { + let table_id = row.try_get::("table_id")?; + let Some(table) = managed_tables.get(&table_id) else { + continue; + }; + let Some(table_meta) = meta.get(&table_id) else { + continue; + }; + if table_meta.deleted { + continue; + } + + let row_json: String = row.try_get("row_json")?; + let value: Value = serde_json::from_str(&row_json)?; + let object = value + .as_object() + .cloned() + .ok_or_else(|| anyhow!("entity snapshot row must be an object"))?; + aggregate + .models + .push(row_to_model_struct(&table.table, &object)?); + if aggregate.created_at == 0 || table_meta.created_at < aggregate.created_at { + aggregate.created_at = table_meta.created_at; + } + aggregate.updated_at = aggregate.updated_at.max(table_meta.updated_at); + aggregate.executed_at = aggregate.executed_at.max(table_meta.executed_at); + } + + if aggregate.models.is_empty() { + return Ok(None); + } + Ok(Some(aggregate)) + } + + async fn load_entity_meta_rows( + &self, + kind: TableKind, + world_address: Felt, + entity_id: Felt, + ) -> Result> { + let sql = match self.state.backend { + DbBackend::Sqlite => "SELECT table_id, created_at, updated_at, executed_at, + CAST(deleted AS INTEGER) AS deleted + FROM torii_ecs_entity_meta + WHERE kind = ? AND world_address = ? AND entity_id = ?" + .to_string(), + DbBackend::Postgres => "SELECT table_id, created_at, updated_at, executed_at, + CASE WHEN deleted THEN 1 ELSE 0 END AS deleted + FROM torii_ecs_entity_meta + WHERE kind = $1 AND world_address = $2 AND entity_id = $3" + .to_string(), + }; + let rows = sqlx::query(&sql) + .bind(kind.as_str()) + .bind(felt_hex(world_address)) + .bind(felt_hex(entity_id)) + .fetch_all(&self.state.pool) + .await?; + let mut meta = HashMap::new(); + for row in rows { + meta.insert( + row.try_get::("table_id")?, + EntityMetaRow { + created_at: row.try_get::("created_at")? as u64, + updated_at: row.try_get::("updated_at")? as u64, + executed_at: row.try_get::("executed_at")? as u64, + deleted: row.try_get::("deleted")? != 0, + }, + ); + } + Ok(meta) + } + + #[allow(dead_code)] + async fn load_entity_meta( + &self, + kind: TableKind, + world_address: Felt, + table_id: Felt, + ) -> Result> { + let sql = match self.state.backend { + DbBackend::Sqlite => "SELECT entity_id, created_at, updated_at, executed_at, + CAST(deleted AS INTEGER) AS deleted + FROM torii_ecs_entity_meta + WHERE kind = ? AND world_address = ? AND table_id = ?" + .to_string(), + DbBackend::Postgres => "SELECT entity_id, created_at, updated_at, executed_at, + CASE WHEN deleted THEN 1 ELSE 0 END AS deleted + FROM torii_ecs_entity_meta + WHERE kind = $1 AND world_address = $2 AND table_id = $3" + .to_string(), + }; + let rows = sqlx::query(&sql) + .bind(kind.as_str()) + .bind(felt_hex(world_address)) + .bind(felt_hex(table_id)) + .fetch_all(&self.state.pool) + .await?; + let mut meta = HashMap::new(); + for row in rows { + meta.insert( + row.try_get::("entity_id")?, + EntityMetaRow { + created_at: row.try_get::("created_at")? as u64, + updated_at: row.try_get::("updated_at")? as u64, + executed_at: row.try_get::("executed_at")? as u64, + deleted: row.try_get::("deleted")? != 0, + }, + ); + } + Ok(meta) + } + + #[allow(dead_code)] + async fn fetch_table_rows(&self, table: &ManagedTable) -> Result>> { + match self.state.backend { + DbBackend::Sqlite => { + let sql = format!( + "SELECT * FROM \"{}\"", + table.table.name.replace('"', "\"\"") + ); + let rows = sqlx::query(&sql).fetch_all(&self.state.pool).await?; + rows.into_iter() + .map(|row| any_row_to_json_map(&row, &table.table)) + .collect() + } + DbBackend::Postgres => { + let sql = format!( + "SELECT row_to_json(t)::text AS data FROM (SELECT * FROM \"{}\") t", + table.table.name.replace('"', "\"\"") + ); + let rows = sqlx::query(&sql).fetch_all(&self.state.pool).await?; + rows.into_iter() + .map(|row| { + let data: String = row.try_get("data")?; + let value: Value = serde_json::from_str(&data)?; + value + .as_object() + .cloned() + .ok_or_else(|| anyhow!("row_to_json must return an object")) + }) + .collect() + } + } + } + + async fn load_events_page( + &self, + query: &types::EventQuery, + ) -> Result<(Vec, String)> { + let pagination = query.pagination.clone().unwrap_or(types::Pagination { + cursor: String::new(), + limit: 100, + direction: PaginationDirection::Forward as i32, + order_by: Vec::new(), + }); + let limit = if pagination.limit == 0 { + 100 + } else { + pagination.limit as usize + }; + let direction_is_backward = pagination.direction == PaginationDirection::Backward as i32; + let order_sql = if direction_is_backward { "DESC" } else { "ASC" }; + + let mut builder = QueryBuilder::::new( + "SELECT event_id, transaction_hash, keys_json, data_json \ + FROM torii_ecs_events", + ); + if !pagination.cursor.is_empty() { + builder.push(" WHERE event_id "); + builder.push(if direction_is_backward { "< " } else { "> " }); + builder.push_bind(pagination.cursor.clone()); + } + builder.push(" ORDER BY event_id "); + builder.push(order_sql); + + let fetch_limit = if query.keys.is_some() { + limit.saturating_mul(20).max(100) + } else { + limit + }; + builder.push(" LIMIT "); + builder.push_bind(fetch_limit as i64); + + let rows = builder.build().fetch_all(&self.state.pool).await?; + ::metrics::counter!("torii_ecs_retrieve_events_rows_fetched_total") + .increment(rows.len() as u64); + + let mut events = Vec::new(); + for row in rows { + let keys_hex: Vec = + serde_json::from_str(&row.try_get::("keys_json")?)?; + let data_hex: Vec = + serde_json::from_str(&row.try_get::("data_json")?)?; + let event = types::Event { + keys: keys_hex + .into_iter() + .map(|value| felt_from_hex(&value).map(|felt| felt.to_bytes_be().to_vec())) + .collect::>>()?, + data: data_hex + .into_iter() + .map(|value| felt_from_hex(&value).map(|felt| felt.to_bytes_be().to_vec())) + .collect::>>()?, + transaction_hash: felt_from_hex(&row.try_get::("transaction_hash")?)? + .to_bytes_be() + .to_vec(), + }; + if query + .keys + .as_ref() + .is_none_or(|keys| match_keys(&event.keys, std::slice::from_ref(keys))) + { + events.push((row.try_get::("event_id")?, event)); + } + if events.len() >= limit { + break; + } + } + ::metrics::counter!("torii_ecs_retrieve_events_rows_decoded_total") + .increment(events.len() as u64); + + if direction_is_backward { + events.reverse(); + } + let next_cursor = events + .last() + .map(|(cursor, _)| cursor.clone()) + .unwrap_or_default(); + Ok(( + events.into_iter().map(|(_, event)| event).collect(), + next_cursor, + )) + } + + async fn next_subscription_id() -> u64 { + use std::sync::atomic::{AtomicU64, Ordering}; + static NEXT_ID: AtomicU64 = AtomicU64::new(1); + NEXT_ID.fetch_add(1, Ordering::Relaxed) + } +} + +#[tonic::async_trait] +impl World for EcsService { + type SubscribeContractsStream = ReceiverStream>; + type SubscribeEntitiesStream = ReceiverStream>; + type SubscribeEventMessagesStream = ReceiverStream>; + type SubscribeEventsStream = ReceiverStream>; + + async fn subscribe_contracts( + &self, + request: Request, + ) -> Result, Status> { + let query = request + .into_inner() + .query + .ok_or_else(|| Status::invalid_argument("Missing query argument"))?; + let (sender, receiver) = mpsc::channel(256); + for contract in self.load_contracts(&query).await.map_err(internal_status)? { + let _ = sender + .send(Ok(SubscribeContractsResponse { + contract: Some(contract), + })) + .await; + } + self.state.contract_subscriptions.lock().await.insert( + Self::next_subscription_id().await, + ContractSubscription { query, sender }, + ); + Ok(Response::new(ReceiverStream::new(receiver))) + } + + async fn worlds( + &self, + request: Request, + ) -> Result, Status> { + let start = Instant::now(); + let world_addresses = request + .into_inner() + .world_addresses + .iter() + .map(|bytes| felt_from_bytes(bytes)) + .collect::>>() + .map_err(internal_status)?; + let worlds = self.load_worlds(&world_addresses).await; + ::metrics::histogram!("torii_ecs_worlds_latency_seconds") + .record(start.elapsed().as_secs_f64()); + let worlds = worlds.map_err(internal_status)?; + Ok(Response::new(WorldsResponse { worlds })) + } + + async fn subscribe_entities( + &self, + request: Request, + ) -> Result, Status> { + let request = request.into_inner(); + let subscription_id = Self::next_subscription_id().await; + let (sender, receiver) = mpsc::channel(256); + sender + .send(Ok(SubscribeEntityResponse { + entity: None, + subscription_id, + })) + .await + .map_err(|_| Status::internal("subscription setup failed"))?; + self.state.entity_subscriptions.lock().await.insert( + subscription_id, + EntitySubscription { + clause: request.clause, + world_addresses: request.world_addresses.into_iter().collect(), + sender, + }, + ); + Ok(Response::new(ReceiverStream::new(receiver))) + } + + async fn update_entities_subscription( + &self, + request: Request, + ) -> Result, Status> { + let request = request.into_inner(); + if let Some(subscription) = self + .state + .entity_subscriptions + .lock() + .await + .get_mut(&request.subscription_id) + { + subscription.clause = request.clause; + subscription.world_addresses = request.world_addresses.into_iter().collect(); + } + Ok(Response::new(())) + } + + async fn retrieve_entities( + &self, + request: Request, + ) -> Result, Status> { + let start = Instant::now(); + let query = request + .into_inner() + .query + .ok_or_else(|| Status::invalid_argument("Missing query argument"))?; + let result = self.load_entity_page(TableKind::Entity, &query).await; + ::metrics::histogram!("torii_ecs_retrieve_entities_latency_seconds") + .record(start.elapsed().as_secs_f64()); + let (entities, next_cursor) = result.map_err(internal_status)?; + ::metrics::counter!("torii_ecs_retrieve_entities_returned_total") + .increment(entities.len() as u64); + Ok(Response::new(RetrieveEntitiesResponse { + next_cursor, + entities, + })) + } + + async fn subscribe_event_messages( + &self, + request: Request, + ) -> Result, Status> { + let request = request.into_inner(); + let subscription_id = Self::next_subscription_id().await; + let (sender, receiver) = mpsc::channel(256); + sender + .send(Ok(SubscribeEntityResponse { + entity: None, + subscription_id, + })) + .await + .map_err(|_| Status::internal("subscription setup failed"))?; + self.state.event_message_subscriptions.lock().await.insert( + subscription_id, + EntitySubscription { + clause: request.clause, + world_addresses: request.world_addresses.into_iter().collect(), + sender, + }, + ); + Ok(Response::new(ReceiverStream::new(receiver))) + } + + async fn update_event_messages_subscription( + &self, + request: Request, + ) -> Result, Status> { + let request = request.into_inner(); + if let Some(subscription) = self + .state + .event_message_subscriptions + .lock() + .await + .get_mut(&request.subscription_id) + { + subscription.clause = request.clause; + subscription.world_addresses = request.world_addresses.into_iter().collect(); + } + Ok(Response::new(())) + } + + async fn retrieve_event_messages( + &self, + request: Request, + ) -> Result, Status> { + let start = Instant::now(); + let query = request + .into_inner() + .query + .ok_or_else(|| Status::invalid_argument("Missing query argument"))?; + let result = self.load_entity_page(TableKind::EventMessage, &query).await; + ::metrics::histogram!("torii_ecs_retrieve_event_messages_latency_seconds") + .record(start.elapsed().as_secs_f64()); + let (entities, next_cursor) = result.map_err(internal_status)?; + ::metrics::counter!("torii_ecs_retrieve_event_messages_returned_total") + .increment(entities.len() as u64); + Ok(Response::new(RetrieveEntitiesResponse { + next_cursor, + entities, + })) + } + + async fn retrieve_events( + &self, + request: Request, + ) -> Result, Status> { + let start = Instant::now(); + let query = request + .into_inner() + .query + .ok_or_else(|| Status::invalid_argument("Missing query argument"))?; + let result = self.load_events_page(&query).await; + ::metrics::histogram!("torii_ecs_retrieve_events_latency_seconds") + .record(start.elapsed().as_secs_f64()); + let (events, next_cursor) = result.map_err(internal_status)?; + ::metrics::counter!("torii_ecs_retrieve_events_returned_total") + .increment(events.len() as u64); + Ok(Response::new(RetrieveEventsResponse { + next_cursor, + events, + })) + } + + async fn subscribe_events( + &self, + request: Request, + ) -> Result, Status> { + let request = request.into_inner(); + let (sender, receiver) = mpsc::channel(256); + sender + .send(Ok(SubscribeEventsResponse { event: None })) + .await + .map_err(|_| Status::internal("subscription setup failed"))?; + self.state.event_subscriptions.lock().await.insert( + Self::next_subscription_id().await, + EventSubscription { + keys: request.keys, + sender, + }, + ); + Ok(Response::new(ReceiverStream::new(receiver))) + } +} + +#[derive(Clone, Copy)] +struct EntityMetaRow { + created_at: u64, + updated_at: u64, + executed_at: u64, + deleted: bool, +} + +struct EntityModelRow { + world_address: String, + table_id: String, + entity_id: String, + row_json: String, +} + +struct SnapshotJsonSerializer; + +fn internal_status(error: anyhow::Error) -> Status { + Status::internal(error.to_string()) +} + +fn sqlite_url(path: &str) -> Result { + if path == ":memory:" || path == "sqlite::memory:" { + return Ok("sqlite::memory:".to_string()); + } + if path.starts_with("sqlite:") { + return Ok(path.to_string()); + } + let options = SqliteConnectOptions::from_str(&format!("sqlite://{path}")) + .or_else(|_| Ok::<_, sqlx::Error>(SqliteConnectOptions::new().filename(path)))?; + if let Some(parent) = options + .get_filename() + .parent() + .filter(|path| !path.as_os_str().is_empty()) + { + std::fs::create_dir_all(parent)?; + } + Ok(options.to_url_lossy().to_string()) +} + +impl CairoTypeSerialization for SnapshotJsonSerializer { + fn serialize_byte_array( + &self, + serializer: S, + value: &[u8], + ) -> Result { + serializer.serialize_str(&format!("0x{}", hex::encode(value))) + } + + fn serialize_felt( + &self, + serializer: S, + value: &[u8; 32], + ) -> Result { + self.serialize_byte_array(serializer, value) + } + + fn serialize_eth_address( + &self, + serializer: S, + value: &[u8; 20], + ) -> Result { + self.serialize_byte_array(serializer, value) + } + + fn serialize_tuple<'a, S: Serializer>( + &'a self, + data: &mut impl CairoDeserializer, + serializer: S, + tuple: &'a TupleDef, + ) -> Result { + let mut seq = serializer.serialize_map(Some(tuple.elements.len()))?; + for (index, element) in tuple.elements.iter().enumerate() { + seq.serialize_entry(&format!("_{index}"), &element.to_de_se(data, self))?; + } + seq.end() + } + + fn serialize_variant<'a, S: Serializer>( + &'a self, + data: &mut impl CairoDeserializer, + serializer: S, + name: &str, + type_def: &'a TypeDef, + ) -> Result { + if type_def == &TypeDef::None { + let mut map = serializer.serialize_map(Some(1))?; + map.serialize_entry("variant", name)?; + map + } else { + let mut map = serializer.serialize_map(Some(2))?; + map.serialize_entry("variant", name)?; + map.serialize_entry(&format!("_{name}"), &type_def.to_de_se(data, self))?; + map + } + .end() + } + + fn serialize_result<'a, S: Serializer>( + &'a self, + data: &mut impl CairoDeserializer, + serializer: S, + result: &'a ResultDef, + is_ok: bool, + ) -> Result { + let mut map = serializer.serialize_map(Some(1))?; + map.serialize_entry("is_ok", &is_ok)?; + if is_ok { + map.serialize_entry("Ok", &result.ok.to_de_se(data, self))?; + } else { + map.serialize_entry("Err", &result.err.to_de_se(data, self))?; + } + map.end() + } +} + +fn model_from_table(table: &ManagedTable) -> types::Model { + let (namespace, name) = split_table_name(&table.table.name); + types::Model { + selector: table.table.id.to_bytes_be().to_vec(), + namespace, + name, + packed_size: table.table.columns.len() as u32, + unpacked_size: table.table.columns.len() as u32, + class_hash: Vec::new(), + layout: Vec::new(), + schema: serde_json::to_vec(&table.table).unwrap_or_default(), + contract_address: Vec::new(), + use_legacy_store: table.table.legacy, + world_address: table.world_address.to_bytes_be().to_vec(), + } +} + +fn split_table_name(name: &str) -> (String, String) { + match name.split_once('-') { + Some((namespace, model)) => (namespace.to_string(), model.to_string()), + None => (String::new(), name.to_string()), + } +} + +fn row_to_model_struct(table: &DojoTable, row: &Map) -> Result { + let children = table + .key_fields + .iter() + .chain(table.value_fields.iter()) + .filter_map(|column_id| table.columns.get(column_id)) + .map(|column| { + let value = row.get(&column.name).cloned().unwrap_or(Value::Null); + Ok(types::Member { + name: column.name.clone(), + ty: Some(type_to_proto_value(&column.type_def, value)?), + key: column.attributes.has_attribute("key"), + }) + }) + .collect::>>()?; + + Ok(types::Struct { + name: table.name.clone(), + children, + }) +} + +fn record_to_json_map( + table: &DojoTable, + columns: &[Felt], + record: &Record, +) -> Result> { + let schema_table = table.to_schema(); + let columns_by_id = schema_table + .columns + .iter() + .cloned() + .map(|column| (column.id, column)) + .collect::>(); + let schema_columns = columns + .iter() + .map(|column_id| { + columns_by_id + .get(column_id) + .ok_or_else(|| anyhow!("column {column_id:#x} not found in table {}", table.name)) + }) + .collect::>>()? + .into_iter() + .cloned() + .map(|column| { + let (_, info): (Felt, ColumnInfo) = column.into(); + info + }) + .collect::>(); + let schema = torii_introspect::tables::RecordSchema::new( + &schema_table.primary, + schema_columns.iter().collect(), + ); + + let mut bytes = Vec::new(); + let mut serializer = JsonSerializer::new(&mut bytes); + schema.parse_records_with_metadata( + std::slice::from_ref(record), + &(), + &mut serializer, + &SnapshotJsonSerializer, + )?; + + let value = serde_json::from_slice::>(&bytes)? + .into_iter() + .next() + .ok_or_else(|| anyhow!("record serializer returned no rows"))?; + value + .as_object() + .cloned() + .ok_or_else(|| anyhow!("record serializer must return an object")) +} + +fn build_entity_from_snapshot( + key: &(String, String), + meta: &HashMap, + rows: Option<&Vec<(String, String)>>, + models: &HashMap, +) -> Result> { + let Some(rows) = rows else { + return Ok(None); + }; + + let mut aggregate = EntityAggregate { + world_address: felt_from_hex(&key.0)?.to_bytes_be().to_vec(), + hashed_keys: felt_from_hex(&key.1)?.to_bytes_be().to_vec(), + ..EntityAggregate::default() + }; + + for (table_id, row_json) in rows { + let Some(table_meta) = meta.get(table_id) else { + continue; + }; + if table_meta.deleted { + continue; + } + let Some(table) = models.get(table_id) else { + continue; + }; + + let value: Value = serde_json::from_str(row_json)?; + let object = value + .as_object() + .cloned() + .ok_or_else(|| anyhow!("entity snapshot row must be an object"))?; + aggregate + .models + .push(row_to_model_struct(&table.table, &object)?); + if aggregate.created_at == 0 || table_meta.created_at < aggregate.created_at { + aggregate.created_at = table_meta.created_at; + } + aggregate.updated_at = aggregate.updated_at.max(table_meta.updated_at); + aggregate.executed_at = aggregate.executed_at.max(table_meta.executed_at); + } + + if aggregate.models.is_empty() { + return Ok(None); + } + Ok(Some(aggregate)) +} + +#[allow(dead_code)] +fn any_row_to_json_map(row: &sqlx::any::AnyRow, table: &DojoTable) -> Result> { + let mut map = Map::new(); + let primary_name = &table.primary.name; + map.insert( + primary_name.clone(), + read_row_value(row, primary_name, &TypeDef::from(&table.primary.type_def))?, + ); + for column in table.columns.values() { + map.insert( + column.name.clone(), + read_row_value(row, &column.name, &column.type_def)?, + ); + } + Ok(map) +} + +#[allow(dead_code)] +fn read_row_value(row: &sqlx::any::AnyRow, name: &str, type_def: &TypeDef) -> Result { + if is_integer_type(type_def) { + let value = row.try_get::, _>(name)?; + return Ok(value.map_or(Value::Null, |value| Value::Number(value.into()))); + } + + let value = row.try_get::, _>(name)?; + let Some(value) = value else { + return Ok(Value::Null); + }; + if is_json_text_type(type_def) { + Ok(serde_json::from_str(&value).unwrap_or(Value::String(value))) + } else { + Ok(Value::String(value)) + } +} + +#[allow(dead_code)] +fn is_integer_type(type_def: &TypeDef) -> bool { + matches!( + type_def, + TypeDef::Bool + | TypeDef::I8 + | TypeDef::I16 + | TypeDef::I32 + | TypeDef::I64 + | TypeDef::U8 + | TypeDef::U16 + | TypeDef::U32 + ) +} + +#[allow(dead_code)] +fn is_json_text_type(type_def: &TypeDef) -> bool { + matches!( + type_def, + TypeDef::Tuple(_) + | TypeDef::Array(_) + | TypeDef::FixedArray(_) + | TypeDef::Struct(_) + | TypeDef::Enum(_) + | TypeDef::Option(_) + | TypeDef::Nullable(_) + | TypeDef::Result(_) + | TypeDef::Felt252Dict(_) + ) +} + +fn type_to_proto_value(type_def: &TypeDef, value: Value) -> Result { + use types::ty::TyType; + + let ty_type = match type_def { + TypeDef::Bool => TyType::Primitive(types::Primitive { + primitive_type: Some(types::primitive::PrimitiveType::Bool( + value.as_bool().unwrap_or_default(), + )), + }), + TypeDef::I8 | TypeDef::I16 | TypeDef::I32 => TyType::Primitive(types::Primitive { + primitive_type: Some(types::primitive::PrimitiveType::I32( + value.as_i64().unwrap_or_default() as i32, + )), + }), + TypeDef::I64 => TyType::Primitive(types::Primitive { + primitive_type: Some(types::primitive::PrimitiveType::I64( + value.as_i64().unwrap_or_default(), + )), + }), + TypeDef::U8 | TypeDef::U16 | TypeDef::U32 => TyType::Primitive(types::Primitive { + primitive_type: Some(types::primitive::PrimitiveType::U32( + value.as_u64().unwrap_or_default() as u32, + )), + }), + TypeDef::U64 => TyType::Primitive(types::Primitive { + primitive_type: Some(types::primitive::PrimitiveType::U64(value_as_u64(&value))), + }), + TypeDef::Felt252 + | TypeDef::ClassHash + | TypeDef::ContractAddress + | TypeDef::StorageAddress + | TypeDef::StorageBaseAddress + | TypeDef::EthAddress => { + let bytes = value_as_felt_bytes(&value)?; + let primitive = match type_def { + TypeDef::ClassHash => types::primitive::PrimitiveType::ClassHash(bytes), + TypeDef::ContractAddress + | TypeDef::StorageAddress + | TypeDef::StorageBaseAddress => { + types::primitive::PrimitiveType::ContractAddress(bytes) + } + TypeDef::EthAddress => types::primitive::PrimitiveType::EthAddress(bytes), + _ => types::primitive::PrimitiveType::Felt252(bytes), + }; + TyType::Primitive(types::Primitive { + primitive_type: Some(primitive), + }) + } + TypeDef::U128 | TypeDef::I128 | TypeDef::U256 | TypeDef::U512 => { + TyType::Primitive(types::Primitive { + primitive_type: Some(types::primitive::PrimitiveType::U256(value_as_bytes( + &value, + ))), + }) + } + TypeDef::Utf8String + | TypeDef::ShortUtf8 + | TypeDef::ByteArray + | TypeDef::ByteArrayEncoded(_) + | TypeDef::Bytes31 + | TypeDef::Bytes31Encoded(_) => TyType::Bytearray(value_as_string(&value)), + TypeDef::Struct(def) => { + let object = value.as_object().cloned().unwrap_or_default(); + TyType::Struct(types::Struct { + name: def.name.clone(), + children: def + .members + .iter() + .map(|member| { + Ok(types::Member { + name: member.name.clone(), + ty: Some(type_to_proto_value( + &member.type_def, + object.get(&member.name).cloned().unwrap_or(Value::Null), + )?), + key: member.attributes.has_attribute("key"), + }) + }) + .collect::>>()?, + }) + } + TypeDef::Tuple(def) => TyType::Tuple(types::Array { + children: value + .as_array() + .cloned() + .unwrap_or_default() + .into_iter() + .zip(def.elements.iter()) + .map(|(item, inner)| type_to_proto_value(inner, item)) + .collect::>>()?, + }), + TypeDef::Array(def) => TyType::Array(types::Array { + children: value + .as_array() + .cloned() + .unwrap_or_default() + .into_iter() + .map(|item| type_to_proto_value(&def.type_def, item)) + .collect::>>()?, + }), + TypeDef::FixedArray(def) => TyType::FixedSizeArray(types::FixedSizeArray { + children: value + .as_array() + .cloned() + .unwrap_or_default() + .into_iter() + .map(|item| type_to_proto_value(&def.type_def, item)) + .collect::>>()?, + size: def.size, + }), + TypeDef::Enum(def) => { + let object = value.as_object().cloned().unwrap_or_default(); + let variant_name = object + .get("option") + .and_then(Value::as_str) + .unwrap_or_default() + .to_string(); + let (option, options) = def + .order + .iter() + .enumerate() + .filter_map(|(index, selector)| { + def.variants.get(selector).map(|variant| (index, variant)) + }) + .map(|(index, variant)| { + let ty = object + .get(&variant.name) + .cloned() + .map(|value| type_to_proto_value(&variant.type_def, value)) + .transpose()? + .unwrap_or_default(); + Ok::<_, anyhow::Error>(( + if variant.name == variant_name { + index as u32 + } else { + u32::MAX + }, + types::EnumOption { + name: variant.name.clone(), + ty: Some(ty), + }, + )) + }) + .collect::>>()? + .into_iter() + .fold( + (0, Vec::new()), + |(selected, mut options), (index, option)| { + let selected = if index == u32::MAX { selected } else { index }; + options.push(option); + (selected, options) + }, + ); + TyType::Enum(types::Enum { + name: def.name.clone(), + option, + options, + }) + } + TypeDef::Option(def) => { + if value.is_null() { + TyType::Bytearray(String::new()) + } else { + return type_to_proto_value(&def.type_def, value); + } + } + TypeDef::Nullable(def) => { + if value.is_null() { + TyType::Bytearray(String::new()) + } else { + return type_to_proto_value(&def.type_def, value); + } + } + TypeDef::Result(def) => { + let object = value.as_object().cloned().unwrap_or_default(); + if let Some(ok) = object.get("Ok").cloned() { + return type_to_proto_value(&def.ok, ok); + } + if let Some(err) = object.get("Err").cloned() { + return type_to_proto_value(&def.err, err); + } + TyType::Bytearray(value_as_string(&value)) + } + TypeDef::None | TypeDef::Felt252Dict(_) | TypeDef::Ref(_) | TypeDef::Custom(_) => { + TyType::Bytearray(value_as_string(&value)) + } + }; + + Ok(types::Ty { + ty_type: Some(ty_type), + }) +} + +fn value_as_u64(value: &Value) -> u64 { + value + .as_u64() + .or_else(|| value.as_i64().map(|value| value as u64)) + .or_else(|| value.as_str().and_then(|value| value.parse::().ok())) + .unwrap_or_default() +} + +fn value_as_string(value: &Value) -> String { + match value { + Value::String(value) => value.clone(), + Value::Null => String::new(), + other => other.to_string(), + } +} + +fn value_as_bytes(value: &Value) -> Vec { + value.as_str().map_or_else( + || value_as_string(value).into_bytes(), + |value| value.as_bytes().to_vec(), + ) +} + +fn value_as_felt_bytes(value: &Value) -> Result> { + if value.is_null() { + return Ok(Felt::ZERO.to_bytes_be().to_vec()); + } + + let string = value_as_string(value); + let trimmed = string.trim(); + if trimmed.is_empty() { + return Ok(Felt::ZERO.to_bytes_be().to_vec()); + } + + Ok(felt_from_hex(trimmed)?.to_bytes_be().to_vec()) +} + +#[allow(dead_code)] +fn value_to_primary_felt(value: &Value, type_def: &PrimaryTypeDef) -> Result { + match type_def { + PrimaryTypeDef::Bool => Ok(Felt::from(value.as_bool().unwrap_or_default() as u64)), + PrimaryTypeDef::U8 + | PrimaryTypeDef::U16 + | PrimaryTypeDef::U32 + | PrimaryTypeDef::U64 + | PrimaryTypeDef::U128 + | PrimaryTypeDef::I8 + | PrimaryTypeDef::I16 + | PrimaryTypeDef::I32 + | PrimaryTypeDef::I64 + | PrimaryTypeDef::I128 => Ok(Felt::from(value_as_u64(value))), + _ => felt_from_hex(&value_as_string(value)), + } +} + +fn member_pushdown_from_clause( + clause: Option<&types::Clause>, + models: &HashMap, +) -> Option { + let ClauseType::Member(member) = clause?.clause_type.as_ref()? else { + return None; + }; + if member.member.contains('.') { + return None; + } + if !member + .member + .chars() + .all(|ch| ch.is_ascii_alphanumeric() || ch == '_') + { + return None; + } + + let expected = scalar_from_member_value(member.value.as_ref()?)?; + if matches!(expected, ScalarValue::Bytes(_)) { + return None; + } + + let table = models + .values() + .find(|table| table.table.name == member.model)?; + if !table + .table + .columns + .values() + .any(|column| column.name == member.member) + { + return None; + } + + Some(MemberPushdown { + table_id: felt_hex(table.table.id), + member: member.member.clone(), + operator: member.operator, + expected, + }) +} + +fn sql_operator(operator: i32) -> Option<&'static str> { + match operator { + value if value == ComparisonOperator::Eq as i32 => Some("="), + value if value == ComparisonOperator::Neq as i32 => Some("!="), + value if value == ComparisonOperator::Gt as i32 => Some(">"), + value if value == ComparisonOperator::Gte as i32 => Some(">="), + value if value == ComparisonOperator::Lt as i32 => Some("<"), + value if value == ComparisonOperator::Lte as i32 => Some("<="), + _ => None, + } +} + +fn append_member_pushdown_sql( + builder: &mut QueryBuilder<'_, Any>, + backend: DbBackend, + pushdown: &MemberPushdown, +) { + let Some(operator) = sql_operator(pushdown.operator) else { + return; + }; + + builder.push(" AND EXISTS (SELECT 1 FROM torii_ecs_entity_models em WHERE em.kind = m.kind"); + builder.push( + " AND em.world_address = m.world_address AND em.table_id = m.table_id AND em.entity_id = m.entity_id", + ); + builder.push(" AND em.table_id = "); + builder.push_bind(pushdown.table_id.clone()); + + match (&pushdown.expected, backend) { + (ScalarValue::Bool(value), DbBackend::Sqlite) => { + builder.push(" AND CAST(json_extract(em.row_json, '$."); + builder.push(pushdown.member.as_str()); + builder.push("') AS INTEGER) "); + builder.push(operator); + builder.push(" "); + builder.push_bind(i64::from(*value)); + } + (ScalarValue::Bool(value), DbBackend::Postgres) => { + builder.push(" AND CASE WHEN lower((em.row_json::jsonb ->> '"); + builder.push(pushdown.member.as_str()); + builder.push("')) = 'true' THEN 1 ELSE 0 END "); + builder.push(operator); + builder.push(" "); + builder.push_bind(i64::from(*value)); + } + (ScalarValue::Signed(value), DbBackend::Sqlite) => { + builder.push(" AND CAST(json_extract(em.row_json, '$."); + builder.push(pushdown.member.as_str()); + builder.push("') AS INTEGER) "); + builder.push(operator); + builder.push(" "); + builder.push_bind(*value); + } + (ScalarValue::Signed(value), DbBackend::Postgres) => { + builder.push(" AND CAST((em.row_json::jsonb ->> '"); + builder.push(pushdown.member.as_str()); + builder.push("') AS BIGINT) "); + builder.push(operator); + builder.push(" "); + builder.push_bind(*value); + } + (ScalarValue::Unsigned(value), DbBackend::Sqlite) => { + builder.push(" AND CAST(json_extract(em.row_json, '$."); + builder.push(pushdown.member.as_str()); + builder.push("') AS INTEGER) "); + builder.push(operator); + builder.push(" "); + builder.push_bind(*value as i64); + } + (ScalarValue::Unsigned(value), DbBackend::Postgres) => { + builder.push(" AND CAST((em.row_json::jsonb ->> '"); + builder.push(pushdown.member.as_str()); + builder.push("') AS BIGINT) "); + builder.push(operator); + builder.push(" "); + builder.push_bind(*value as i64); + } + (ScalarValue::Text(value), DbBackend::Sqlite) => { + builder.push(" AND CAST(json_extract(em.row_json, '$."); + builder.push(pushdown.member.as_str()); + builder.push("') AS TEXT) "); + builder.push(operator); + builder.push(" "); + builder.push_bind(value.clone()); + } + (ScalarValue::Text(value), DbBackend::Postgres) => { + builder.push(" AND (em.row_json::jsonb ->> '"); + builder.push(pushdown.member.as_str()); + builder.push("') "); + builder.push(operator); + builder.push(" "); + builder.push_bind(value.clone()); + } + (ScalarValue::Bytes(_), _) => {} + } + + builder.push(")"); +} + +fn entity_matches_clause(entity: &types::Entity, clause: &types::Clause) -> bool { + match &clause.clause_type { + Some(ClauseType::HashedKeys(hashed)) => { + hashed.hashed_keys.is_empty() || hashed.hashed_keys.contains(&entity.hashed_keys) + } + Some(ClauseType::Keys(keys)) => { + if !keys.models.is_empty() + && !entity + .models + .iter() + .any(|model| keys.models.contains(&model.name)) + { + return false; + } + let entity_keys = entity_key_values(entity); + if keys.pattern_matching == PatternMatching::FixedLen as i32 + && entity_keys.len() != keys.keys.len() + { + return false; + } + entity_keys.iter().enumerate().all(|(index, key)| { + keys.keys + .get(index) + .is_none_or(|expected| expected.is_empty() || expected == key) + }) + } + Some(ClauseType::Member(member)) => entity.models.iter().any(|model| { + model.name == member.model + && model_member_matches( + model, + &member.member, + member.operator, + member.value.as_ref(), + ) + }), + Some(ClauseType::Composite(composite)) => match composite.operator { + value if value == LogicalOperator::And as i32 => composite + .clauses + .iter() + .all(|clause| entity_matches_clause(entity, clause)), + _ => composite + .clauses + .iter() + .any(|clause| entity_matches_clause(entity, clause)), + }, + None => true, + } +} + +fn match_keys(keys: &[Vec], clauses: &[types::KeysClause]) -> bool { + clauses.is_empty() + || clauses.iter().any(|clause| { + if clause.pattern_matching == PatternMatching::FixedLen as i32 + && keys.len() != clause.keys.len() + { + return false; + } + keys.iter().enumerate().all(|(index, key)| { + clause + .keys + .get(index) + .is_none_or(|expected| expected.is_empty() || expected == key) + }) + }) +} + +fn model_member_matches( + model: &types::Struct, + path: &str, + operator: i32, + value: Option<&types::MemberValue>, +) -> bool { + let mut current = find_member_value(model, path); + let Some(current) = current.take() else { + return false; + }; + let Some(value) = value else { + return false; + }; + compare_member_value(current, operator, value) +} + +fn find_member_value<'a>(model: &'a types::Struct, path: &str) -> Option<&'a types::Ty> { + let mut members = &model.children; + let parts = path.split('.').collect::>(); + let mut current = None; + for (index, part) in parts.iter().enumerate() { + let member = members.iter().find(|member| member.name == *part)?; + current = member.ty.as_ref(); + if index + 1 == parts.len() { + return current; + } + let ty = current?.ty_type.as_ref()?; + match ty { + types::ty::TyType::Struct(value) => { + members = &value.children; + } + _ => return None, + } + } + current +} + +fn compare_member_value(current: &types::Ty, operator: i32, expected: &types::MemberValue) -> bool { + let current_value = scalar_from_ty(current); + let expected_value = scalar_from_member_value(expected); + match (current_value, expected_value) { + (Some(current), Some(expected)) => compare_scalars(¤t, operator, &expected), + _ => false, + } +} + +fn scalar_from_ty(value: &types::Ty) -> Option { + match value.ty_type.as_ref()? { + types::ty::TyType::Primitive(primitive) => match primitive.primitive_type.as_ref()? { + types::primitive::PrimitiveType::Bool(value) => Some(ScalarValue::Bool(*value)), + types::primitive::PrimitiveType::I32(value) => Some(ScalarValue::Signed(*value as i64)), + types::primitive::PrimitiveType::I64(value) => Some(ScalarValue::Signed(*value)), + types::primitive::PrimitiveType::U32(value) => { + Some(ScalarValue::Unsigned(*value as u64)) + } + types::primitive::PrimitiveType::U64(value) => Some(ScalarValue::Unsigned(*value)), + types::primitive::PrimitiveType::Felt252(value) + | types::primitive::PrimitiveType::ClassHash(value) + | types::primitive::PrimitiveType::ContractAddress(value) + | types::primitive::PrimitiveType::EthAddress(value) + | types::primitive::PrimitiveType::U256(value) + | types::primitive::PrimitiveType::U128(value) + | types::primitive::PrimitiveType::I128(value) => { + Some(ScalarValue::Bytes(value.clone())) + } + _ => None, + }, + types::ty::TyType::Bytearray(value) => Some(ScalarValue::Text(value.clone())), + _ => None, + } +} + +fn scalar_from_member_value(value: &types::MemberValue) -> Option { + match value.value_type.as_ref()? { + ValueType::Primitive(primitive) => scalar_from_ty(&types::Ty { + ty_type: Some(types::ty::TyType::Primitive(primitive.clone())), + }), + ValueType::String(value) => Some(ScalarValue::Text(value.clone())), + ValueType::List(_) => None, + } +} + +#[derive(Clone)] +enum ScalarValue { + Bool(bool), + Signed(i64), + Unsigned(u64), + Text(String), + Bytes(Vec), +} + +struct MemberPushdown { + table_id: String, + member: String, + operator: i32, + expected: ScalarValue, +} + +fn compare_scalars(current: &ScalarValue, operator: i32, expected: &ScalarValue) -> bool { + match (current, expected) { + (ScalarValue::Bool(current), ScalarValue::Bool(expected)) => match operator { + value if value == ComparisonOperator::Eq as i32 => current == expected, + value if value == ComparisonOperator::Neq as i32 => current != expected, + _ => false, + }, + (ScalarValue::Signed(current), ScalarValue::Signed(expected)) => { + compare_ord(current, operator, expected) + } + (ScalarValue::Unsigned(current), ScalarValue::Unsigned(expected)) => { + compare_ord(current, operator, expected) + } + (ScalarValue::Text(current), ScalarValue::Text(expected)) => { + compare_ord(current, operator, expected) + } + (ScalarValue::Bytes(current), ScalarValue::Bytes(expected)) => { + compare_ord(current, operator, expected) + } + _ => false, + } +} + +fn compare_ord(current: &T, operator: i32, expected: &T) -> bool { + match operator { + value if value == ComparisonOperator::Eq as i32 => current == expected, + value if value == ComparisonOperator::Neq as i32 => current != expected, + value if value == ComparisonOperator::Gt as i32 => current > expected, + value if value == ComparisonOperator::Gte as i32 => current >= expected, + value if value == ComparisonOperator::Lt as i32 => current < expected, + value if value == ComparisonOperator::Lte as i32 => current <= expected, + _ => false, + } +} + +fn entity_key_values(entity: &types::Entity) -> Vec> { + entity + .models + .iter() + .find_map(|model| { + let keys = model + .children + .iter() + .filter(|member| member.key) + .filter_map(|member| { + scalar_from_ty(member.ty.as_ref()?).map(|value| match value { + ScalarValue::Bool(value) => vec![u8::from(value)], + ScalarValue::Signed(value) => value.to_be_bytes().to_vec(), + ScalarValue::Unsigned(value) => value.to_be_bytes().to_vec(), + ScalarValue::Text(value) => value.into_bytes(), + ScalarValue::Bytes(value) => value, + }) + }) + .collect::>(); + (!keys.is_empty()).then_some(keys) + }) + .unwrap_or_default() +} + +fn contract_matches_query(contract: &types::Contract, query: &types::ContractQuery) -> bool { + (query.contract_addresses.is_empty() + || query + .contract_addresses + .contains(&contract.contract_address)) + && (query.contract_types.is_empty() + || query.contract_types.contains(&contract.contract_type)) +} + +fn felt_hex(value: Felt) -> String { + format!("{value:#x}") +} + +fn felt_from_hex(value: &str) -> Result { + Felt::from_str(value).map_err(|err| anyhow!("invalid felt {value}: {err}")) +} + +fn felt_from_bytes(value: &[u8]) -> Result { + Ok(Felt::from_bytes_be_slice(value)) +} + +#[cfg(test)] +mod tests { + use super::*; + use introspect_types::{ColumnDef, PrimaryDef, PrimaryTypeDef, TypeDef}; + use tokio::time::{timeout, Duration}; + use tokio_stream::StreamExt; + + fn test_db_path(name: &str) -> String { + let nonce = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .expect("clock before epoch") + .as_nanos(); + format!("sqlite:file:torii-ecs-sink-{name}-{nonce}?mode=memory&cache=shared") + } + + fn member_bool_clause(model: &str, member: &str, value: bool) -> types::Clause { + types::Clause { + clause_type: Some(ClauseType::Member(types::MemberClause { + model: model.to_string(), + member: member.to_string(), + operator: ComparisonOperator::Eq as i32, + value: Some(types::MemberValue { + value_type: Some(ValueType::Primitive(types::Primitive { + primitive_type: Some(types::primitive::PrimitiveType::Bool(value)), + })), + }), + })), + } + } + + async fn seed_entity( + service: &EcsService, + kind: TableKind, + world_address: Felt, + table_id: Felt, + table_name: &str, + entity_id: Felt, + open: bool, + ) { + sqlx::query( + "CREATE TABLE IF NOT EXISTS dojo_tables ( + owner BLOB NOT NULL, + id BLOB NOT NULL, + name TEXT NOT NULL, + attributes TEXT NOT NULL, + keys_json TEXT NOT NULL, + values_json TEXT NOT NULL, + legacy INTEGER NOT NULL, + created_at INTEGER, + updated_at INTEGER, + created_block INTEGER, + updated_block INTEGER, + created_tx BLOB, + updated_tx BLOB, + PRIMARY KEY(owner, id) + )", + ) + .execute(&service.state.pool) + .await + .expect("create dojo_tables"); + sqlx::query( + "CREATE TABLE IF NOT EXISTS dojo_columns ( + owner BLOB NOT NULL, + table_id BLOB NOT NULL, + id BLOB NOT NULL, + payload TEXT NOT NULL, + PRIMARY KEY(owner, table_id, id) + )", + ) + .execute(&service.state.pool) + .await + .expect("create dojo_columns"); + + let table = CreateTable { + id: table_id, + name: table_name.to_string(), + attributes: vec![], + primary: PrimaryDef { + name: "entity_id".to_string(), + attributes: vec![], + type_def: PrimaryTypeDef::Felt252, + }, + columns: vec![ColumnDef { + id: Felt::from(1_u64), + name: "open".to_string(), + attributes: vec![], + type_def: TypeDef::Bool, + }], + }; + + service.cache_created_table(world_address, &table).await; + service + .record_table_kind(world_address, table_id, kind) + .await + .expect("record table kind"); + service + .upsert_entity_meta(kind, world_address, table_id, entity_id, 1, false) + .await + .expect("upsert entity meta"); + + let row_json = serde_json::json!({ "open": open }).to_string(); + sqlx::query( + "INSERT INTO torii_ecs_entity_models ( + kind, world_address, table_id, entity_id, row_json, updated_at + ) VALUES (?1, ?2, ?3, ?4, ?5, ?6) + ON CONFLICT(kind, world_address, table_id, entity_id) DO UPDATE SET + row_json = excluded.row_json, + updated_at = excluded.updated_at", + ) + .bind(kind.as_str()) + .bind(felt_hex(world_address)) + .bind(felt_hex(table_id)) + .bind(felt_hex(entity_id)) + .bind(row_json) + .bind(1_i64) + .execute(&service.state.pool) + .await + .expect("insert entity model"); + } + + #[tokio::test] + async fn subscribe_entities_update_flow() { + let db_path = test_db_path("entities-sub"); + let service = EcsService::new(&db_path, Some(1)) + .await + .expect("service init"); + let world = Felt::from(10_u64); + let table = Felt::from(20_u64); + let entity = Felt::from(30_u64); + seed_entity( + &service, + TableKind::Entity, + world, + table, + "test-Lobby", + entity, + true, + ) + .await; + + let response = service + .subscribe_entities(Request::new(SubscribeEntitiesRequest { + clause: Some(member_bool_clause("test-Lobby", "open", true)), + world_addresses: vec![world.to_bytes_be().to_vec()], + })) + .await + .expect("subscribe entities"); + let mut stream = response.into_inner(); + + let setup = timeout(Duration::from_secs(1), stream.next()) + .await + .expect("setup timeout") + .expect("setup frame") + .expect("setup ok"); + let subscription_id = setup.subscription_id; + assert!(setup.entity.is_none()); + + service + .publish_entity_update(TableKind::Entity, world, entity) + .await + .expect("publish entity update"); + let matched = timeout(Duration::from_secs(1), stream.next()) + .await + .expect("entity timeout") + .expect("entity frame") + .expect("entity ok"); + assert_eq!(matched.subscription_id, subscription_id); + let matched_entity = matched.entity.expect("entity payload"); + assert_eq!(matched_entity.world_address, world.to_bytes_be().to_vec()); + + service + .update_entities_subscription(Request::new(UpdateEntitiesSubscriptionRequest { + subscription_id, + clause: Some(member_bool_clause("test-Lobby", "open", false)), + world_addresses: vec![world.to_bytes_be().to_vec()], + })) + .await + .expect("update entities subscription"); + + service + .publish_entity_update(TableKind::Entity, world, entity) + .await + .expect("publish filtered entity update"); + let filtered = timeout(Duration::from_millis(150), stream.next()).await; + assert!(filtered.is_err(), "entity should be filtered out"); + } + + #[tokio::test] + async fn subscribe_event_messages_update_flow() { + let db_path = test_db_path("event-messages-sub"); + let service = EcsService::new(&db_path, Some(1)) + .await + .expect("service init"); + let world = Felt::from(101_u64); + let table = Felt::from(201_u64); + let entity = Felt::from(301_u64); + seed_entity( + &service, + TableKind::EventMessage, + world, + table, + "test-EventMessage", + entity, + true, + ) + .await; + + let response = service + .subscribe_event_messages(Request::new(SubscribeEntitiesRequest { + clause: Some(member_bool_clause("test-EventMessage", "open", true)), + world_addresses: vec![world.to_bytes_be().to_vec()], + })) + .await + .expect("subscribe event messages"); + let mut stream = response.into_inner(); + + let setup = timeout(Duration::from_secs(1), stream.next()) + .await + .expect("setup timeout") + .expect("setup frame") + .expect("setup ok"); + let subscription_id = setup.subscription_id; + assert!(setup.entity.is_none()); + + service + .publish_entity_update(TableKind::EventMessage, world, entity) + .await + .expect("publish event message update"); + let matched = timeout(Duration::from_secs(1), stream.next()) + .await + .expect("event message timeout") + .expect("event message frame") + .expect("event message ok"); + assert_eq!(matched.subscription_id, subscription_id); + assert!(matched.entity.is_some()); + + service + .update_event_messages_subscription(Request::new(UpdateEntitiesSubscriptionRequest { + subscription_id, + clause: Some(member_bool_clause("test-EventMessage", "open", false)), + world_addresses: vec![world.to_bytes_be().to_vec()], + })) + .await + .expect("update event messages subscription"); + + service + .publish_entity_update(TableKind::EventMessage, world, entity) + .await + .expect("publish filtered event message update"); + let filtered = timeout(Duration::from_millis(150), stream.next()).await; + assert!(filtered.is_err(), "event message should be filtered out"); + } + + #[tokio::test] + async fn subscribe_events_keys_filter() { + let db_path = test_db_path("events-sub"); + let service = EcsService::new(&db_path, Some(1)) + .await + .expect("service init"); + let world = Felt::from(1_u64); + let tx1 = Felt::from(2_u64); + let tx2 = Felt::from(3_u64); + let key_match = Felt::from(444_u64); + + let response = service + .subscribe_events(Request::new(SubscribeEventsRequest { + keys: vec![types::KeysClause { + keys: vec![key_match.to_bytes_be().to_vec()], + pattern_matching: PatternMatching::VariableLen as i32, + models: vec![], + }], + })) + .await + .expect("subscribe events"); + let mut stream = response.into_inner(); + + let setup = timeout(Duration::from_secs(1), stream.next()) + .await + .expect("setup timeout") + .expect("setup frame") + .expect("setup ok"); + assert!(setup.event.is_none()); + + service + .store_event( + world, + tx1, + 1, + 1, + &[Felt::from(999_u64)], + &[Felt::from(7_u64)], + 0, + ) + .await + .expect("store non matching event"); + let no_match = timeout(Duration::from_millis(150), stream.next()).await; + assert!( + no_match.is_err(), + "non matching event should be filtered out" + ); + + service + .store_event(world, tx2, 1, 1, &[key_match, Felt::from(123_u64)], &[], 1) + .await + .expect("store matching event"); + let matched = timeout(Duration::from_secs(1), stream.next()) + .await + .expect("matched timeout") + .expect("matched frame") + .expect("matched ok"); + let event = matched.event.expect("event payload"); + assert_eq!(event.transaction_hash, tx2.to_bytes_be().to_vec()); + assert_eq!( + event.keys.first().cloned(), + Some(key_match.to_bytes_be().to_vec()) + ); + } + + #[tokio::test] + async fn subscribe_contracts_query_filter() { + let db_path = test_db_path("contracts-sub"); + let service = EcsService::new(&db_path, Some(1)) + .await + .expect("service init"); + let world_contract = Felt::from(1111_u64); + let other_contract = Felt::from(2222_u64); + + let response = service + .subscribe_contracts(Request::new(SubscribeContractsRequest { + query: Some(types::ContractQuery { + contract_addresses: vec![world_contract.to_bytes_be().to_vec()], + contract_types: vec![ContractType::World as i32], + }), + })) + .await + .expect("subscribe contracts"); + let mut stream = response.into_inner(); + + service + .record_contract_progress(other_contract, ContractType::World, 7, 77, None) + .await + .expect("record other contract progress"); + let no_match = timeout(Duration::from_millis(150), stream.next()).await; + assert!( + no_match.is_err(), + "non matching contract should be filtered out" + ); + + service + .record_contract_progress(world_contract, ContractType::World, 9, 99, None) + .await + .expect("record world contract progress"); + let matched = timeout(Duration::from_secs(1), stream.next()) + .await + .expect("matched timeout") + .expect("matched frame") + .expect("matched ok"); + let contract = matched.contract.expect("contract payload"); + assert_eq!( + contract.contract_address, + world_contract.to_bytes_be().to_vec() + ); + assert_eq!(contract.contract_type, ContractType::World as i32); + } + + #[tokio::test] + async fn retrieve_entities_without_filters_uses_bounded_query() { + let db_path = test_db_path("retrieve-entities"); + let service = EcsService::new(&db_path, Some(1)) + .await + .expect("service init"); + let world = Felt::from(900_u64); + + seed_entity( + &service, + TableKind::Entity, + world, + Felt::from(901_u64), + "test-ModelA", + Felt::from(1001_u64), + true, + ) + .await; + seed_entity( + &service, + TableKind::Entity, + world, + Felt::from(902_u64), + "test-ModelB", + Felt::from(1001_u64), + true, + ) + .await; + + let response = service + .retrieve_entities(Request::new(RetrieveEntitiesRequest { + query: Some(types::Query { + pagination: Some(types::Pagination { + cursor: String::new(), + limit: 10, + direction: PaginationDirection::Forward as i32, + order_by: vec![], + }), + world_addresses: vec![], + models: vec![], + clause: None, + no_hashed_keys: false, + historical: false, + }), + })) + .await + .expect("retrieve entities") + .into_inner(); + + assert_eq!(response.entities.len(), 1); + assert!(!response.next_cursor.is_empty() || response.entities.len() < 10); + } +} diff --git a/crates/torii-ecs-sink/src/lib.rs b/crates/torii-ecs-sink/src/lib.rs new file mode 100644 index 00000000..0a3d28c9 --- /dev/null +++ b/crates/torii-ecs-sink/src/lib.rs @@ -0,0 +1,19 @@ +pub mod grpc_service; +pub mod sink; + +pub mod proto { + pub mod world { + tonic::include_proto!("world"); + + pub const FILE_DESCRIPTOR_SET: &[u8] = + tonic::include_file_descriptor_set!("world_descriptor"); + } + + pub mod types { + tonic::include_proto!("types"); + } +} + +pub use grpc_service::EcsService; +pub use proto::world::FILE_DESCRIPTOR_SET; +pub use sink::EcsSink; diff --git a/crates/torii-ecs-sink/src/sink.rs b/crates/torii-ecs-sink/src/sink.rs new file mode 100644 index 00000000..314a46af --- /dev/null +++ b/crates/torii-ecs-sink/src/sink.rs @@ -0,0 +1,227 @@ +use std::sync::Arc; + +use anyhow::Result; +use async_trait::async_trait; +use dojo_introspect::events::{ + EventEmitted, StoreDelRecord, StoreSetRecord, StoreUpdateMember, StoreUpdateRecord, +}; +use introspect_types::event::CairoEventInfo; +use starknet::core::types::Felt; +use torii::axum::Router; +use torii::etl::{ + envelope::{Envelope, TypeId}, + extractor::ExtractionBatch, + sink::{EventBus, Sink, SinkContext, TopicInfo}, +}; +use torii_introspect::events::{IntrospectBody, IntrospectMsg}; + +use crate::grpc_service::{EcsService, TableKind}; + +pub struct EcsSink { + service: Arc, +} + +impl EcsSink { + pub async fn new(database_url: &str, max_connections: Option) -> Result { + Ok(Self { + service: Arc::new(EcsService::new(database_url, max_connections).await?), + }) + } + + pub fn get_grpc_service_impl(&self) -> Arc { + self.service.clone() + } +} + +#[async_trait] +impl Sink for EcsSink { + fn name(&self) -> &'static str { + "ecs" + } + + fn interested_types(&self) -> Vec { + vec![TypeId::new("introspect")] + } + + async fn process(&self, envelopes: &[Envelope], batch: &ExtractionBatch) -> Result<()> { + for (ordinal, event) in batch.events.iter().enumerate() { + let context = batch + .get_event_context(&event.transaction_hash, event.from_address) + .unwrap_or_default(); + self.service + .store_event( + event.from_address, + event.transaction_hash, + context.transaction.block_number, + context.block.timestamp, + &event.keys, + &event.data, + ordinal, + ) + .await?; + + self.service + .record_contract_progress( + event.from_address, + crate::proto::types::ContractType::World, + context.transaction.block_number, + context.block.timestamp, + None, + ) + .await?; + + let Some(selector) = event.keys.first() else { + continue; + }; + let selector_raw = selector.to_raw(); + if matches!( + selector_raw, + StoreSetRecord::SELECTOR_RAW + | StoreUpdateRecord::SELECTOR_RAW + | StoreUpdateMember::SELECTOR_RAW + | StoreDelRecord::SELECTOR_RAW + ) { + if event.keys.len() >= 3 { + self.service + .record_table_kind(event.from_address, event.keys[1], TableKind::Entity) + .await?; + self.service + .upsert_entity_meta( + TableKind::Entity, + event.from_address, + event.keys[1], + event.keys[2], + context.block.timestamp, + selector_raw == StoreDelRecord::SELECTOR_RAW, + ) + .await?; + } + } else if selector_raw == EventEmitted::SELECTOR_RAW && event.keys.len() >= 2 { + self.service + .record_table_kind(event.from_address, event.keys[1], TableKind::EventMessage) + .await?; + } + } + + for envelope in envelopes { + if envelope.type_id != TypeId::new("introspect") { + continue; + } + + let Some(body) = envelope.downcast_ref::() else { + continue; + }; + let context = batch + .get_event_context(&body.metadata.transaction_hash, body.metadata.from_address) + .unwrap_or_default(); + + match &body.msg { + IntrospectMsg::CreateTable(table) => { + self.service + .cache_created_table(body.metadata.from_address, table) + .await; + self.service + .record_table_kind(body.metadata.from_address, table.id, TableKind::Entity) + .await + .ok(); + } + IntrospectMsg::UpdateTable(table) => { + self.service + .cache_updated_table(body.metadata.from_address, table) + .await; + self.service + .record_table_kind(body.metadata.from_address, table.id, TableKind::Entity) + .await + .ok(); + } + IntrospectMsg::InsertsFields(insert) => { + let kind = if batch.events.iter().any(|event| { + event.keys.first().is_some_and(|selector| { + selector.to_raw() == EventEmitted::SELECTOR_RAW + && event.keys.get(1).copied() == Some(insert.table) + }) + }) { + TableKind::EventMessage + } else { + TableKind::Entity + }; + self.service + .record_table_kind(body.metadata.from_address, insert.table, kind) + .await?; + for record in &insert.records { + let entity_id = Felt::from_bytes_be(&record.id); + self.service + .upsert_entity_meta( + kind, + body.metadata.from_address, + insert.table, + entity_id, + context.block.timestamp, + false, + ) + .await?; + self.service + .upsert_entity_model( + kind, + body.metadata.from_address, + insert.table, + &insert.columns, + record, + context.block.timestamp, + ) + .await?; + self.service + .publish_entity_update(kind, body.metadata.from_address, entity_id) + .await?; + } + } + IntrospectMsg::DeleteRecords(delete) => { + let kind = self.service.table_kind(delete.table).await?; + for row in &delete.rows { + let entity_id = row.to_felt(); + self.service + .upsert_entity_meta( + kind, + body.metadata.from_address, + delete.table, + entity_id, + context.block.timestamp, + true, + ) + .await?; + self.service + .delete_entity_model( + kind, + body.metadata.from_address, + delete.table, + entity_id, + ) + .await?; + self.service + .publish_entity_update(kind, body.metadata.from_address, entity_id) + .await?; + } + } + _ => {} + } + } + + Ok(()) + } + + fn topics(&self) -> Vec { + Vec::new() + } + + fn build_routes(&self) -> Router { + Router::new() + } + + async fn initialize( + &mut self, + _event_bus: Arc, + _context: &SinkContext, + ) -> Result<()> { + Ok(()) + } +} diff --git a/scripts/test-ecs-entity-model-filters.sh b/scripts/test-ecs-entity-model-filters.sh new file mode 100755 index 00000000..61045ae7 --- /dev/null +++ b/scripts/test-ecs-entity-model-filters.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +set -euo pipefail + +GRPC_ADDR="${GRPC_ADDR:-localhost:3000}" +DB_PATH="${DB_PATH:-torii-data/introspect.db}" +WORLD_ADDRESS="${WORLD_ADDRESS:-}" +LIMIT="${LIMIT:-20}" + +fail(){ printf '[ecs-model-filter] error: %s\n' "$*" >&2; exit 1; } +log(){ printf '[ecs-model-filter] %s\n' "$*"; } + +for cmd in grpcurl sqlite3 rg xxd base64 mktemp; do command -v "$cmd" >/dev/null 2>&1 || fail "missing $cmd"; done +[[ -f "$DB_PATH" ]] || fail "missing db at $DB_PATH" + +if [[ -z "$WORLD_ADDRESS" ]]; then + WORLD_ADDRESS="$(sqlite3 -batch -noheader "$DB_PATH" "SELECT world_address FROM torii_ecs_entity_models WHERE kind='entity' ORDER BY world_address LIMIT 1;")" +fi +[[ -n "$WORLD_ADDRESS" ]] || fail 'cannot discover world' + +mapfile -t MODELS < <(sqlite3 -batch -noheader "$DB_PATH" " +SELECT DISTINCT d.name +FROM torii_ecs_table_kinds k +JOIN dojo_tables d + ON ltrim(lower(hex(d.owner)), '0') = ltrim(substr(k.world_address, 3), '0') + AND ltrim(lower(hex(d.id)), '0') = ltrim(substr(k.table_id, 3), '0') +WHERE k.world_address='${WORLD_ADDRESS}' AND k.kind='entity' +ORDER BY d.name LIMIT 2;") +[[ ${#MODELS[@]} -ge 2 ]] || fail 'not enough models' + +felt_to_b64(){ local h="${1#0x}"; h="$(printf '%064s' "$h" | tr ' ' '0')"; printf '%s' "$h" | xxd -r -p | base64 | tr -d '\n'; } +WORLD_B64="$(felt_to_b64 "$WORLD_ADDRESS")" + +tmp="$(mktemp -d)"; trap 'rm -rf "$tmp"' EXIT +one="$tmp/one.json"; two="$tmp/two.json"; both="$tmp/both.json"; bad="$tmp/bad.json" + +grpcurl -plaintext -d "{\"query\":{\"world_addresses\":[\"$WORLD_B64\"],\"models\":[\"${MODELS[0]}\"],\"pagination\":{\"limit\":$LIMIT,\"direction\":\"FORWARD\"}}}" "$GRPC_ADDR" world.World/RetrieveEntities >"$one" +grpcurl -plaintext -d "{\"query\":{\"world_addresses\":[\"$WORLD_B64\"],\"models\":[\"${MODELS[1]}\"],\"pagination\":{\"limit\":$LIMIT,\"direction\":\"FORWARD\"}}}" "$GRPC_ADDR" world.World/RetrieveEntities >"$two" +grpcurl -plaintext -d "{\"query\":{\"world_addresses\":[\"$WORLD_B64\"],\"models\":[\"${MODELS[0]}\",\"${MODELS[1]}\"],\"pagination\":{\"limit\":$LIMIT,\"direction\":\"FORWARD\"}}}" "$GRPC_ADDR" world.World/RetrieveEntities >"$both" +grpcurl -plaintext -d "{\"query\":{\"world_addresses\":[\"$WORLD_B64\"],\"models\":[\"definitely-not-a-real-model\"],\"pagination\":{\"limit\":$LIMIT,\"direction\":\"FORWARD\"}}}" "$GRPC_ADDR" world.World/RetrieveEntities >"$bad" + +rg -q 'hashedKeys' "$one" || fail 'single model A returned no entities' +rg -q 'hashedKeys' "$two" || fail 'single model B returned no entities' +rg -q 'hashedKeys' "$both" || fail 'union model query returned no entities' +! rg -q 'hashedKeys' "$bad" || fail 'unknown model should return no entities' + +log "world=${WORLD_ADDRESS}" +log "model_a=${MODELS[0]}" +log "model_b=${MODELS[1]}" +log 'entity model filter checks passed' diff --git a/scripts/test-ecs-grpc-e2e.sh b/scripts/test-ecs-grpc-e2e.sh new file mode 100755 index 00000000..e8d1e950 --- /dev/null +++ b/scripts/test-ecs-grpc-e2e.sh @@ -0,0 +1,247 @@ +#!/usr/bin/env bash + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" + +GRPC_ADDR="${GRPC_ADDR:-localhost:3000}" +DB_PATH="${DB_PATH:-${REPO_ROOT}/torii-data/introspect.db}" +DB_URL="${DB_URL:-${DATABASE_URL:-}}" +LIMIT="${LIMIT:-1}" +RUN_SUBSCRIPTIONS_SMOKE="${RUN_SUBSCRIPTIONS_SMOKE:-1}" + +log() { + printf '[ecs-e2e] %s\n' "$*" +} + +fail() { + printf '[ecs-e2e] error: %s\n' "$*" >&2 + exit 1 +} + +require_cmd() { + command -v "$1" >/dev/null 2>&1 || fail "missing required command: $1" +} + +sql_one() { + local query="$1" + case "$DB_BACKEND" in + sqlite) + sqlite3 -batch -noheader "$DB_PATH" "$query" | head -n 1 + ;; + postgres) + psql "$DB_URL" -Atqc "$query" | head -n 1 + ;; + *) + fail "unsupported DB_BACKEND: $DB_BACKEND" + ;; + esac +} + +model_name_query() { + local kind="$1" + case "$DB_BACKEND" in + sqlite) + cat <&1 | rg -q '\-d'; then + base64 -d + else + base64 -D + fi +} + +base64_to_felt() { + local value="$1" + local hex + hex="$(printf '%s' "$value" | base64_decode | xxd -p -c 256 | tr -d '\n')" + hex="$(printf '%s' "$hex" | sed 's/^0*//')" + if [[ -z "$hex" ]]; then + hex="0" + fi + printf '0x%s' "$hex" +} + +grpc_call() { + local method="$1" + local payload="$2" + local output="$3" + grpcurl -plaintext -d "$payload" "$GRPC_ADDR" "$method" >"$output" +} + +assert_contains() { + local needle="$1" + local file="$2" + rg -F -q "$needle" "$file" || fail "expected to find '$needle' in $file" +} + +assert_non_empty_entities() { + local file="$1" + assert_contains '"entities": [' "$file" + if rg -q '"entities": \[\]' "$file"; then + fail "expected non-empty entities response in $file" + fi +} + +assert_non_empty_events() { + local file="$1" + assert_contains '"events": [' "$file" + if rg -q '"events": \[\]' "$file"; then + fail "expected non-empty events response in $file" + fi +} + +require_cmd grpcurl +require_cmd rg +require_cmd xxd +require_cmd base64 +require_cmd mktemp + +if [[ -n "$DB_URL" && "$DB_URL" =~ ^postgres(ql)?:// ]]; then + DB_BACKEND="postgres" +else + DB_BACKEND="sqlite" +fi + +if [[ -z "${WORLD_ADDRESS:-}" || -z "${ENTITY_MODEL:-}" || -z "${EVENT_MESSAGE_MODEL:-}" ]]; then + if [[ "$DB_BACKEND" == "sqlite" ]]; then + require_cmd sqlite3 + [[ -f "$DB_PATH" ]] || fail "sqlite database not found at $DB_PATH" + else + require_cmd psql + [[ -n "$DB_URL" ]] || fail "postgres DB_URL is required for metadata autodiscovery" + fi +fi + +discovery_dir="$(mktemp -d)" +trap 'rm -rf "$discovery_dir"' EXIT +discovered_worlds_output="$discovery_dir/worlds_discovery.json" +grpc_call "world.World/Worlds" "{}" "$discovered_worlds_output" + +if [[ -z "${WORLD_ADDRESS:-}" ]]; then + world_address_b64="$( + rg -o '"worldAddress": "[^"]+"' "$discovered_worlds_output" \ + | rg -v '"worldAddress": "[0-9]+"' \ + | sed -E 's/"worldAddress": "([^"]+)"/\1/' \ + | while IFS= read -r candidate; do + candidate_hex="$(base64_to_felt "$candidate")" + if [[ "$candidate_hex" != "0x0" ]]; then + printf '%s\n' "$candidate" + break + fi + done + )" + [[ -n "$world_address_b64" ]] || fail "could not discover a world from live Worlds response" + WORLD_ADDRESS="$(base64_to_felt "$world_address_b64")" +fi + +[[ -n "$WORLD_ADDRESS" ]] || fail "could not discover a world address" + +ENTITY_MODEL="${ENTITY_MODEL:-$(sql_one "$(model_name_query entity)")}" +EVENT_MESSAGE_MODEL="${EVENT_MESSAGE_MODEL:-$(sql_one "$(model_name_query event_message)")}" + +[[ -n "$ENTITY_MODEL" ]] || fail "could not discover an entity model for world $WORLD_ADDRESS" +[[ -n "$EVENT_MESSAGE_MODEL" ]] || fail "could not discover an event_message model for world $WORLD_ADDRESS" + +WORLD_ADDRESS_B64="$(felt_to_base64 "$WORLD_ADDRESS")" + +tmp_dir="$(mktemp -d)" +if [[ "${KEEP_TMP:-0}" != "1" ]]; then + trap 'rm -rf "$tmp_dir" "$discovery_dir"' EXIT +else + trap 'rm -rf "$discovery_dir"' EXIT +fi + +list_output="$tmp_dir/list.txt" +describe_output="$tmp_dir/describe.txt" +worlds_output="$tmp_dir/worlds.json" +entities_output="$tmp_dir/entities.json" +event_messages_output="$tmp_dir/event_messages.json" +events_output="$tmp_dir/events.json" + +log "gRPC target: $GRPC_ADDR" +log "db backend: $DB_BACKEND" +log "world: $WORLD_ADDRESS" +log "entity model: $ENTITY_MODEL" +log "event message model: $EVENT_MESSAGE_MODEL" + +grpcurl -plaintext "$GRPC_ADDR" list >"$list_output" +assert_contains 'world.World' "$list_output" +assert_contains 'torii.Torii' "$list_output" + +grpcurl -plaintext "$GRPC_ADDR" describe world.World >"$describe_output" +assert_contains 'rpc RetrieveEntities' "$describe_output" +assert_contains 'rpc RetrieveEventMessages' "$describe_output" +assert_contains 'rpc RetrieveEvents' "$describe_output" + +grpc_call "world.World/Worlds" "{\"world_addresses\":[\"$WORLD_ADDRESS_B64\"]}" "$worlds_output" +assert_contains '"worlds": [' "$worlds_output" +assert_contains "\"worldAddress\": \"$WORLD_ADDRESS_B64\"" "$worlds_output" + +grpc_call \ + "world.World/RetrieveEntities" \ + "{\"query\":{\"world_addresses\":[\"$WORLD_ADDRESS_B64\"],\"models\":[\"$ENTITY_MODEL\"],\"pagination\":{\"limit\":$LIMIT,\"direction\":\"FORWARD\"}}}" \ + "$entities_output" +assert_non_empty_entities "$entities_output" +assert_contains "\"name\": \"$ENTITY_MODEL\"" "$entities_output" + +grpc_call \ + "world.World/RetrieveEventMessages" \ + "{\"query\":{\"world_addresses\":[\"$WORLD_ADDRESS_B64\"],\"models\":[\"$EVENT_MESSAGE_MODEL\"],\"pagination\":{\"limit\":$LIMIT,\"direction\":\"FORWARD\"}}}" \ + "$event_messages_output" +assert_non_empty_entities "$event_messages_output" +assert_contains "\"name\": \"$EVENT_MESSAGE_MODEL\"" "$event_messages_output" + +grpc_call \ + "world.World/RetrieveEvents" \ + "{\"query\":{\"pagination\":{\"limit\":$LIMIT,\"direction\":\"FORWARD\"}}}" \ + "$events_output" +assert_non_empty_events "$events_output" + +log "verification passed" + +if [[ "$RUN_SUBSCRIPTIONS_SMOKE" == "1" ]]; then + log "running subscriptions smoke checks" + GRPC_ADDR="$GRPC_ADDR" "$REPO_ROOT/scripts/test-ecs-subscriptions-smoke.sh" +fi + +if [[ "${KEEP_TMP:-0}" == "1" ]]; then + log "artifacts kept in $tmp_dir" +fi diff --git a/scripts/test-ecs-subscriptions-smoke.sh b/scripts/test-ecs-subscriptions-smoke.sh new file mode 100755 index 00000000..3ceed1c5 --- /dev/null +++ b/scripts/test-ecs-subscriptions-smoke.sh @@ -0,0 +1,92 @@ +#!/usr/bin/env bash + +set -euo pipefail + +GRPC_ADDR="${GRPC_ADDR:-localhost:3000}" +STREAM_TIMEOUT_SEC="${STREAM_TIMEOUT_SEC:-2.5}" +REQUIRE_LIVE_UPDATES="${REQUIRE_LIVE_UPDATES:-0}" +WORLD_ADDRESS_B64="${WORLD_ADDRESS_B64:-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=}" + +log() { + printf '[ecs-sub-smoke] %s\n' "$*" +} + +fail() { + printf '[ecs-sub-smoke] error: %s\n' "$*" >&2 + exit 1 +} + +require_cmd() { + command -v "$1" >/dev/null 2>&1 || fail "missing required command: $1" +} + +stream_call() { + local method="$1" + local payload="$2" + local out_file="$3" + local err_file="$4" + + set +e + grpcurl -plaintext -max-time "$STREAM_TIMEOUT_SEC" -d "$payload" "$GRPC_ADDR" "$method" >"$out_file" 2>"$err_file" + local status=$? + set -e + + if [[ $status -ne 0 ]]; then + if rg -q 'DeadlineExceeded|max-time reached' "$err_file"; then + return 0 + fi + fail "stream call failed for $method: $(<"$err_file")" + fi +} + +require_cmd grpcurl +require_cmd rg +require_cmd mktemp + +list_out="$(mktemp)" +trap 'rm -f "$list_out" "$entities_out" "$entities_err" "$messages_out" "$messages_err" "$events_out" "$events_err" "$contracts_out" "$contracts_err"' EXIT + +grpcurl -plaintext "$GRPC_ADDR" list >"$list_out" +rg -F -q 'world.World' "$list_out" || fail 'world.World service not exposed' + +entities_out="$(mktemp)"; entities_err="$(mktemp)" +messages_out="$(mktemp)"; messages_err="$(mktemp)" +events_out="$(mktemp)"; events_err="$(mktemp)" +contracts_out="$(mktemp)"; contracts_err="$(mktemp)" + +log "gRPC target: $GRPC_ADDR" +log "world filter: $WORLD_ADDRESS_B64" + +stream_call "world.World/SubscribeEntities" "{\"world_addresses\":[\"$WORLD_ADDRESS_B64\"]}" "$entities_out" "$entities_err" +rg -q '"subscriptionId":' "$entities_out" || fail 'SubscribeEntities missing subscriptionId frame' +if [[ "$REQUIRE_LIVE_UPDATES" == "1" ]]; then + rg -q '"entity": \{' "$entities_out" || fail 'SubscribeEntities did not receive live entity update' +fi +log 'SubscribeEntities setup frame: OK' + +stream_call "world.World/SubscribeEventMessages" "{\"world_addresses\":[\"$WORLD_ADDRESS_B64\"]}" "$messages_out" "$messages_err" +rg -q '"subscriptionId":' "$messages_out" || fail 'SubscribeEventMessages missing subscriptionId frame' +if [[ "$REQUIRE_LIVE_UPDATES" == "1" ]]; then + rg -q '"entity": \{' "$messages_out" || fail 'SubscribeEventMessages did not receive live update' +fi +log 'SubscribeEventMessages setup frame: OK' + +stream_call "world.World/SubscribeEvents" '{"keys":[]}' "$events_out" "$events_err" +if ! rg -q '^\{\}' "$events_out" && ! rg -q '"event": \{' "$events_out"; then + fail 'SubscribeEvents did not emit setup or live frame' +fi +if [[ "$REQUIRE_LIVE_UPDATES" == "1" ]]; then + rg -q '"event": \{' "$events_out" || fail 'SubscribeEvents did not receive live event' +fi +log 'SubscribeEvents setup frame: OK' + +stream_call "world.World/SubscribeContracts" '{"query":{"contract_addresses":["AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="]}}' "$contracts_out" "$contracts_err" +if rg -q '"contract": \{' "$contracts_out"; then + log 'SubscribeContracts initial/live frame: OK' +elif [[ "$REQUIRE_LIVE_UPDATES" == "1" ]]; then + fail 'SubscribeContracts did not receive contract frame' +else + log 'SubscribeContracts: no contract frame during window (inconclusive)' +fi + +log 'subscription smoke checks passed' diff --git a/src/etl/decoder/context.rs b/src/etl/decoder/context.rs index 5732db39..a8068a32 100644 --- a/src/etl/decoder/context.rs +++ b/src/etl/decoder/context.rs @@ -12,7 +12,6 @@ //! - Deterministic ordering: decoders are always called in sorted DecoderId order use async_trait::async_trait; -use futures::stream::{self, StreamExt}; use starknet::core::types::{EmittedEvent, Felt}; use std::collections::HashMap; use std::sync::Arc; @@ -22,12 +21,21 @@ use super::{ContractFilter, Decoder, DecoderId}; use crate::etl::engine_db::EngineDb; use crate::etl::envelope::Envelope; +fn event_preview(event: &EmittedEvent) -> String { + format!( + "contract={:#x} tx={:#x}", + event.from_address, event.transaction_hash + ) +} + /// DecoderContext manages multiple decoders with contract filtering. /// /// Routes events to decoders based on: /// 1. Explicit mappings (highest priority, from ContractFilter) /// 2. Registry mappings (from ContractRegistry auto-identification) /// 3. All decoders (fallback when no registry is configured) +/// +#[allow(dead_code)] pub struct DecoderContext { /// Decoders indexed by their ID (hash of name) decoders: HashMap>, @@ -237,9 +245,10 @@ impl DecoderContext { Err(e) => { tracing::warn!( target: "torii::etl::decoder_context", - "Decoder '{}' failed: {}", + "Decoder '{}' failed: {} | {}", decoder.decoder_name(), - e + e, + event_preview(event) ); } } @@ -280,9 +289,10 @@ impl DecoderContext { Err(e) => { tracing::warn!( target: "torii::etl::decoder_context", - "Decoder '{}' failed: {}", + "Decoder '{}' failed: {} | {}", decoder.decoder_name(), - e + e, + event_preview(event) ); } } @@ -361,43 +371,14 @@ impl Decoder for DecoderContext { } async fn decode(&self, events: &[EmittedEvent]) -> anyhow::Result> { - const DECODE_CHUNK_SIZE: usize = 256; - - let chunk_count = events.len().div_ceil(DECODE_CHUNK_SIZE); - ::metrics::gauge!("torii_decode_parallelism").set(self.decode_parallelism as f64); - - let decode_start = std::time::Instant::now(); - let decode_tasks = events - .chunks(DECODE_CHUNK_SIZE) - .enumerate() - .map(|(chunk_index, chunk)| async move { - let mut chunk_envelopes = Vec::new(); - for event in chunk { - let envelopes = self.decode_event(event).await?; - chunk_envelopes.extend(envelopes); - } - - Ok::<_, anyhow::Error>((chunk_index, chunk_envelopes)) - }) - .collect::>(); - - let mut decoded_chunks = stream::iter(decode_tasks) - .buffer_unordered(self.decode_parallelism) - .collect::>() - .await - .into_iter() - .collect::>>()?; + let mut all_envelopes = Vec::new(); - decoded_chunks.sort_by_key(|(chunk_index, _)| *chunk_index); + for event in events { + let envelopes = self.decode_event(event).await?; - let mut all_envelopes = Vec::new(); - for (_, envelopes) in decoded_chunks { all_envelopes.extend(envelopes); } - ::metrics::histogram!("torii_decode_chunk_duration_seconds") - .record(decode_start.elapsed().as_secs_f64() / chunk_count.max(1) as f64); - tracing::debug!( target: "torii::etl::decoder_context", "Decoded {} events into {} envelopes across {} decoders",