From dfdf9b99af4096662bac6c35a47a29dbcee4c766 Mon Sep 17 00:00:00 2001 From: MartianGreed Date: Tue, 10 Mar 2026 17:14:34 +0100 Subject: [PATCH 01/10] fix(sqlite): torii-introspect --- Cargo.lock | 4 + bins/torii-introspect-bin/src/config.rs | 27 +- bins/torii-introspect-bin/src/main.rs | 35 + crates/dojo/src/manager.rs | 1131 +++++++++++++++++ crates/introspect-postgres-sink/Cargo.toml | 1 + crates/introspect-postgres-sink/src/lib.rs | 2 + crates/introspect-postgres-sink/src/sink.rs | 54 +- crates/introspect-postgres-sink/src/sqlite.rs | 748 +++++++++++ 8 files changed, 1976 insertions(+), 26 deletions(-) create mode 100644 crates/dojo/src/manager.rs create mode 100644 crates/introspect-postgres-sink/src/sqlite.rs diff --git a/Cargo.lock b/Cargo.lock index 70bb0b26..30cb16a1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -917,6 +917,7 @@ dependencies = [ [[package]] name = "dojo-introspect" version = "0.1.0" +source = "git+https://github.com/dojoengine/dojo-introspect?rev=aadc3c9#aadc3c980706596a4a083413813a0a3ab01fded7" dependencies = [ "async-trait", "introspect-rust-macros", @@ -1863,6 +1864,7 @@ dependencies = [ [[package]] name = "introspect-events" version = "0.1.2" +source = "git+https://github.com/cartridge-gg/introspect?rev=34e93c1#34e93c10c867c53c622cce03abb6431c9dae0ef5" dependencies = [ "cainome-cairo-serde", "introspect-types", @@ -1874,6 +1876,7 @@ dependencies = [ [[package]] name = "introspect-rust-macros" version = "0.1.0" +source = "git+https://github.com/cartridge-gg/introspect?rev=34e93c1#34e93c10c867c53c622cce03abb6431c9dae0ef5" dependencies = [ "paste", "proc-macro2", @@ -1885,6 +1888,7 @@ dependencies = [ [[package]] name = "introspect-types" version = "0.1.2" +source = "git+https://github.com/cartridge-gg/introspect?rev=34e93c1#34e93c10c867c53c622cce03abb6431c9dae0ef5" dependencies = [ "blake3", "convert_case", diff --git a/bins/torii-introspect-bin/src/config.rs b/bins/torii-introspect-bin/src/config.rs index 5b7bf658..784824b1 100644 --- a/bins/torii-introspect-bin/src/config.rs +++ b/bins/torii-introspect-bin/src/config.rs @@ -133,31 +133,18 @@ mod tests { #[test] fn observability_defaults_to_disabled() { - let cfg = Config::parse_from([ - "torii-introspect", - "--contracts", - "0x1", - "--storage-database-url", - "postgres://localhost/torii", - ]); + let cfg = Config::parse_from(["torii-introspect", "--contracts", "0x1"]); assert!(!cfg.observability); } #[test] fn observability_flag_enables_metrics() { - let cfg = Config::parse_from([ - "torii-introspect", - "--contracts", - "0x1", - "--storage-database-url", - "postgres://localhost/torii", - "--observability", - ]); + let cfg = Config::parse_from(["torii-introspect", "--contracts", "0x1", "--observability"]); assert!(cfg.observability); } #[test] - fn storage_database_url_must_be_postgres() { + fn storage_database_url_accepts_sqlite() { let cfg = Config::parse_from([ "torii-introspect", "--contracts", @@ -182,13 +169,7 @@ mod tests { #[test] fn contract_addresses_parse_from_hex() { - let cfg = Config::parse_from([ - "torii-introspect", - "--contracts", - "0x1,0x2", - "--storage-database-url", - "postgres://localhost/torii", - ]); + let cfg = Config::parse_from(["torii-introspect", "--contracts", "0x1,0x2"]); let contracts = cfg.contract_addresses().unwrap(); assert_eq!(contracts.len(), 2); diff --git a/bins/torii-introspect-bin/src/main.rs b/bins/torii-introspect-bin/src/main.rs index 95c07861..168ec5c7 100644 --- a/bins/torii-introspect-bin/src/main.rs +++ b/bins/torii-introspect-bin/src/main.rs @@ -23,6 +23,40 @@ use torii_introspect_postgres_sink::processor::IntrospectPgDb; use torii_introspect_sqlite_sink::processor::IntrospectSqliteDb; use torii_sqlite::{is_sqlite_memory_path, sqlite_connect_options}; +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum DbBackend { + Postgres, + Sqlite, +} + +fn detect_database_backend(database_url: &str) -> DbBackend { + if database_url.starts_with("postgres://") || database_url.starts_with("postgresql://") { + DbBackend::Postgres + } else { + DbBackend::Sqlite + } +} + +fn sqlite_connect_options(path: &str) -> Result { + if path == ":memory:" || path == "sqlite::memory:" { + return SqliteConnectOptions::from_str("sqlite::memory:") + .map_err(|err| anyhow::anyhow!("Failed to parse sqlite URL: {err}")); + } + + let options = if path.starts_with("sqlite:") { + SqliteConnectOptions::from_str(path) + .map_err(|err| anyhow::anyhow!("Failed to parse sqlite URL {path}: {err}"))? + } else { + SqliteConnectOptions::new().filename(path) + }; + + if path.starts_with("sqlite:") && path.contains("mode=") { + Ok(options) + } else { + Ok(options.create_if_missing(true)) + } +} + #[tokio::main] async fn main() -> Result<()> { let env_filter = tracing_subscriber::EnvFilter::try_from_default_env() @@ -59,6 +93,7 @@ async fn run_indexer(config: Config) -> Result<()> { tracing::info!("Storage backend: {:?}", backend); tracing::info!("Engine database URL: {}", engine_database_url); tracing::info!("Storage database URL: {}", storage_database_url); + tracing::info!("Database backend: {:?}", backend); tracing::info!( "Saved state handling: {}", if config.ignore_saved_state { diff --git a/crates/dojo/src/manager.rs b/crates/dojo/src/manager.rs new file mode 100644 index 00000000..ece7ade0 --- /dev/null +++ b/crates/dojo/src/manager.rs @@ -0,0 +1,1131 @@ +pub use crate::store::json::JsonStore; +use crate::store::postgres::initialize_dojo_schema; +use crate::store::DojoStoreTrait; +use crate::{DojoTable, DojoToriiError, DojoToriiResult}; +use async_trait::async_trait; +use itertools::Itertools; +use sqlx::postgres::PgPoolOptions; +use sqlx::sqlite::{SqliteConnectOptions, SqlitePoolOptions}; +use sqlx::{PgPool, Postgres, QueryBuilder, Row, Sqlite, SqlitePool}; +use starknet_types_core::felt::Felt; +use std::collections::HashMap; +use std::path::PathBuf; +use std::str::FromStr; +use thiserror::Error; +use torii_introspect::events::CreateTable; + +pub struct DojoTableStore { + pub tables: HashMap, + pub store: Store, +} + +impl DojoTableStore +where + Store: DojoStoreTrait + Send + Sync, + Store::Error: ToString, +{ + pub async fn with_store(store: Store) -> DojoToriiResult { + let tables = store + .load_tables(&[]) + .await + .map_err(DojoToriiError::store_error)?; + Ok(Self::from_loaded_tables(store, tables)) + } + + pub fn from_loaded_tables(store: Store, tables: Vec) -> Self { + Self { + tables: tables.into_iter().map(|table| (table.id, table)).collect(), + store, + } + } + + pub fn create_table_messages(&self) -> DojoToriiResult> { + Ok(self + .tables + .values() + .map(|table| CreateTable::from(table.clone().to_schema())) + .collect_vec()) + } +} + +#[async_trait] +impl DojoStoreTrait for DojoTableStore +where + Store: DojoStoreTrait + Send + Sync, + Store::Error: ToString, +{ + type Error = DojoToriiError; + + async fn save_table_at_block( + &self, + owner: &Felt, + table: &DojoTable, + block_number: Option, + ) -> DojoToriiResult<()> { + self.store + .save_table_at_block(owner, table, block_number) + .await + .map_err(DojoToriiError::store_error) + } + + async fn load_tables(&self, owners: &[Felt]) -> DojoToriiResult> { + self.store + .load_tables(owners) + .await + .map_err(DojoToriiError::store_error) + } + + async fn load_tables_at_blocks( + &self, + owner_blocks: &[(Felt, u64)], + ) -> DojoToriiResult> { + self.store + .load_tables_at_blocks(owner_blocks) + .await + .map_err(DojoToriiError::store_error) + } +} + +pub struct PostgresStore { + pool: PgPool, +} + +pub struct SqliteStore { + pool: SqlitePool, +} + +#[derive(Debug, Clone)] +pub struct SchemaBootstrapPoint { + pub owner: Felt, + pub block_number: u64, + pub had_saved_state: bool, +} + +#[derive(Debug, Clone)] +pub struct HistoricalBootstrapDiagnostics { + pub tables: Vec, + pub latest_table_count: usize, + pub historical_table_count: usize, + pub unsafe_owners: Vec, +} + +impl PostgresStore { + pub async fn new(database_url: &str) -> DojoToriiResult { + let pool = PgPoolOptions::new() + .max_connections(5) + .connect(database_url) + .await + .map_err(DojoToriiError::store_error)?; + + let store = Self { pool }; + store + .initialize() + .await + .map_err(DojoToriiError::store_error)?; + + Ok(store) + } + + async fn initialize(&self) -> Result<(), sqlx::Error> { + sqlx::query( + r" + CREATE TABLE IF NOT EXISTS torii_dojo_manager_state ( + owner BYTEA NOT NULL, + table_id BYTEA NOT NULL, + table_json TEXT NOT NULL, + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + PRIMARY KEY (owner, table_id) + ) + ", + ) + .execute(&self.pool) + .await?; + + sqlx::query( + r" + CREATE TABLE IF NOT EXISTS torii_dojo_manager_state_history ( + owner BYTEA NOT NULL, + table_id BYTEA NOT NULL, + applied_at_block BIGINT NOT NULL, + table_json TEXT NOT NULL, + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + PRIMARY KEY (owner, table_id, applied_at_block) + ) + ", + ) + .execute(&self.pool) + .await?; + + Ok(()) + } + + pub async fn load_historical_bootstrap( + &self, + bootstrap_points: &[SchemaBootstrapPoint], + ) -> Result { + let latest_tables = self + .load_tables( + &bootstrap_points + .iter() + .map(|point| point.owner) + .collect::>(), + ) + .await?; + let historical_tables = self + .load_tables_at_blocks( + &bootstrap_points + .iter() + .map(|point| (point.owner, point.block_number)) + .collect::>(), + ) + .await?; + + let latest_counts = self + .latest_table_counts( + &bootstrap_points + .iter() + .map(|point| point.owner) + .collect::>(), + ) + .await?; + let history_presence = self + .history_presence( + &bootstrap_points + .iter() + .map(|point| point.owner) + .collect::>(), + ) + .await?; + + let unsafe_owners = bootstrap_points + .iter() + .filter(|point| point.block_number > 0) + .filter(|point| latest_counts.get(&point.owner).copied().unwrap_or_default() > 0) + .filter(|point| !history_presence.get(&point.owner).copied().unwrap_or(false)) + .map(|point| point.owner) + .collect_vec(); + + Ok(HistoricalBootstrapDiagnostics { + latest_table_count: latest_tables.len(), + historical_table_count: historical_tables.len(), + tables: historical_tables, + unsafe_owners, + }) + } + + async fn latest_table_counts( + &self, + owners: &[Felt], + ) -> Result, sqlx::Error> { + if owners.is_empty() { + return Ok(HashMap::new()); + } + + let mut query = QueryBuilder::::new( + "SELECT owner, COUNT(*) AS table_count FROM torii_dojo_manager_state WHERE owner IN (", + ); + { + let mut separated = query.separated(", "); + for owner in owners { + separated.push_bind(owner.to_bytes_be().to_vec()); + } + } + query.push(") GROUP BY owner"); + + let rows = query.build().fetch_all(&self.pool).await?; + Ok(rows + .into_iter() + .map(|row| { + let owner: Vec = row.get("owner"); + let table_count: i64 = row.get("table_count"); + (Felt::from_bytes_be_slice(&owner), table_count as usize) + }) + .collect()) + } + + async fn history_presence(&self, owners: &[Felt]) -> Result, sqlx::Error> { + if owners.is_empty() { + return Ok(HashMap::new()); + } + + let mut query = QueryBuilder::::new( + "SELECT owner, TRUE AS has_history FROM torii_dojo_manager_state_history WHERE owner IN (", + ); + { + let mut separated = query.separated(", "); + for owner in owners { + separated.push_bind(owner.to_bytes_be().to_vec()); + } + } + query.push(") GROUP BY owner"); + + let rows = query.build().fetch_all(&self.pool).await?; + Ok(rows + .into_iter() + .map(|row| { + let owner: Vec = row.get("owner"); + (Felt::from_bytes_be_slice(&owner), true) + }) + .collect()) + } +} + +impl SqliteStore { + pub async fn new(database_url: &str) -> DojoToriiResult { + let options = sqlite_connect_options(database_url).map_err(DojoToriiError::store_error)?; + if let Some(parent) = sqlite_parent_dir(&options) { + std::fs::create_dir_all(parent).map_err(DojoToriiError::store_error)?; + } + + let pool = SqlitePoolOptions::new() + .max_connections(1) + .connect_with(options) + .await + .map_err(DojoToriiError::store_error)?; + + let store = Self { pool }; + store + .initialize() + .await + .map_err(DojoToriiError::store_error)?; + + Ok(store) + } + + async fn initialize(&self) -> Result<(), sqlx::Error> { + sqlx::query( + r#" + CREATE TABLE IF NOT EXISTS torii_dojo_manager_state ( + owner BLOB NOT NULL, + table_id BLOB NOT NULL, + table_json TEXT NOT NULL, + updated_at INTEGER NOT NULL DEFAULT (unixepoch()), + PRIMARY KEY (owner, table_id) + ) + "#, + ) + .execute(&self.pool) + .await?; + + sqlx::query( + r#" + CREATE TABLE IF NOT EXISTS torii_dojo_manager_state_history ( + owner BLOB NOT NULL, + table_id BLOB NOT NULL, + applied_at_block INTEGER NOT NULL, + table_json TEXT NOT NULL, + updated_at INTEGER NOT NULL DEFAULT (unixepoch()), + PRIMARY KEY (owner, table_id, applied_at_block) + ) + "#, + ) + .execute(&self.pool) + .await?; + + Ok(()) + } + + pub async fn load_historical_bootstrap( + &self, + bootstrap_points: &[SchemaBootstrapPoint], + ) -> Result { + let owners = bootstrap_points + .iter() + .map(|point| point.owner) + .collect::>(); + + let latest_tables = self.load_tables(&owners).await?; + let historical_tables = self + .load_tables_at_blocks( + &bootstrap_points + .iter() + .map(|point| (point.owner, point.block_number)) + .collect::>(), + ) + .await?; + + let latest_counts = self.latest_table_counts(&owners).await?; + let history_presence = self.history_presence(&owners).await?; + + let unsafe_owners = bootstrap_points + .iter() + .filter(|point| point.block_number > 0) + .filter(|point| latest_counts.get(&point.owner).copied().unwrap_or_default() > 0) + .filter(|point| !history_presence.get(&point.owner).copied().unwrap_or(false)) + .map(|point| point.owner) + .collect_vec(); + + Ok(HistoricalBootstrapDiagnostics { + latest_table_count: latest_tables.len(), + historical_table_count: historical_tables.len(), + tables: historical_tables, + unsafe_owners, + }) + } + + async fn latest_table_counts( + &self, + owners: &[Felt], + ) -> Result, sqlx::Error> { + if owners.is_empty() { + return Ok(HashMap::new()); + } + + let mut query = QueryBuilder::::new( + "SELECT owner, COUNT(*) AS table_count FROM torii_dojo_manager_state WHERE owner IN (", + ); + { + let mut separated = query.separated(", "); + for owner in owners { + separated.push_bind(owner.to_bytes_be().to_vec()); + } + } + query.push(") GROUP BY owner"); + + let rows = query.build().fetch_all(&self.pool).await?; + Ok(rows + .into_iter() + .map(|row| { + let owner: Vec = row.get("owner"); + let table_count: i64 = row.get("table_count"); + (Felt::from_bytes_be_slice(&owner), table_count as usize) + }) + .collect()) + } + + async fn history_presence(&self, owners: &[Felt]) -> Result, sqlx::Error> { + if owners.is_empty() { + return Ok(HashMap::new()); + } + + let mut query = QueryBuilder::::new( + "SELECT owner, 1 AS has_history FROM torii_dojo_manager_state_history WHERE owner IN (", + ); + { + let mut separated = query.separated(", "); + for owner in owners { + separated.push_bind(owner.to_bytes_be().to_vec()); + } + } + query.push(") GROUP BY owner"); + + let rows = query.build().fetch_all(&self.pool).await?; + Ok(rows + .into_iter() + .map(|row| { + let owner: Vec = row.get("owner"); + (Felt::from_bytes_be_slice(&owner), true) + }) + .collect()) + } +} + +#[async_trait] +impl DojoStoreTrait for PostgresStore { + type Error = sqlx::Error; + + async fn save_table_at_block( + &self, + owner: &Felt, + table: &DojoTable, + block_number: Option, + ) -> Result<(), Self::Error> { + let json = + serde_json::to_string(table).map_err(|e| sqlx::Error::Protocol(e.to_string()))?; + let mut tx = self.pool.begin().await?; + sqlx::query( + r" + INSERT INTO torii_dojo_manager_state (owner, table_id, table_json, updated_at) + VALUES ($1, $2, $3, NOW()) + ON CONFLICT (owner, table_id) + DO UPDATE SET + table_json = EXCLUDED.table_json, + updated_at = NOW() + ", + ) + .bind(owner.to_bytes_be().to_vec()) + .bind(table.id.to_bytes_be().to_vec()) + .bind(json) + .execute(&mut *tx) + .await?; + + if let Some(block_number) = block_number { + let history_json = + serde_json::to_string(table).map_err(|e| sqlx::Error::Protocol(e.to_string()))?; + sqlx::query( + r" + INSERT INTO torii_dojo_manager_state_history + (owner, table_id, applied_at_block, table_json, updated_at) + VALUES ($1, $2, $3, $4, NOW()) + ON CONFLICT (owner, table_id, applied_at_block) + DO UPDATE SET + table_json = EXCLUDED.table_json, + updated_at = NOW() + ", + ) + .bind(owner.to_bytes_be().to_vec()) + .bind(table.id.to_bytes_be().to_vec()) + .bind(block_number as i64) + .bind(history_json) + .execute(&mut *tx) + .await?; + } + + tx.commit().await?; + Ok(()) + } + + async fn load_tables(&self, owners: &[Felt]) -> Result, Self::Error> { + let rows = if owners.is_empty() { + sqlx::query( + r" + SELECT table_json + FROM torii_dojo_manager_state + ORDER BY updated_at ASC + ", + ) + .fetch_all(&self.pool) + .await? + } else { + let placeholders = (1..=owners.len()) + .map(|index| format!("${index}")) + .join(", "); + let query = format!( + " + SELECT table_json + FROM torii_dojo_manager_state + WHERE owner IN ({placeholders}) + ORDER BY updated_at ASC + " + ); + let mut query = sqlx::query(&query); + for owner in owners { + query = query.bind(owner.to_bytes_be().to_vec()); + } + query.fetch_all(&self.pool).await? + }; + + rows.into_iter() + .map(|row| { + let json: String = row.try_get("table_json")?; + serde_json::from_str(&json).map_err(|e| sqlx::Error::Protocol(e.to_string())) + }) + .collect() + } + + async fn load_tables_at_blocks( + &self, + owner_blocks: &[(Felt, u64)], + ) -> Result, Self::Error> { + if owner_blocks.is_empty() { + return self.load_tables(&[]).await; + } + + let mut query = QueryBuilder::::new("WITH requested(owner, resume_block) AS ("); + query.push_values(owner_blocks, |mut builder, (owner, block_number)| { + builder + .push_bind(owner.to_bytes_be().to_vec()) + .push_bind(*block_number as i64); + }); + query.push( + ") SELECT DISTINCT ON (requested.owner, history.table_id) history.table_json \ + FROM requested \ + JOIN torii_dojo_manager_state_history AS history \ + ON history.owner = requested.owner \ + AND history.applied_at_block <= requested.resume_block \ + ORDER BY requested.owner, history.table_id, history.applied_at_block DESC", + ); + + let rows = query.build().fetch_all(&self.pool).await?; + rows.into_iter() + .map(|row| { + let json: String = row.try_get("table_json")?; + serde_json::from_str(&json).map_err(|e| sqlx::Error::Protocol(e.to_string())) + }) + .collect() + } +} + +#[async_trait] +impl DojoStoreTrait for SqliteStore { + type Error = sqlx::Error; + + async fn save_table_at_block( + &self, + owner: &Felt, + table: &DojoTable, + block_number: Option, + ) -> Result<(), Self::Error> { + let json = + serde_json::to_string(table).map_err(|e| sqlx::Error::Protocol(e.to_string()))?; + let mut tx = self.pool.begin().await?; + sqlx::query( + r#" + INSERT INTO torii_dojo_manager_state (owner, table_id, table_json, updated_at) + VALUES (?1, ?2, ?3, unixepoch()) + ON CONFLICT (owner, table_id) + DO UPDATE SET + table_json = excluded.table_json, + updated_at = unixepoch() + "#, + ) + .bind(owner.to_bytes_be().to_vec()) + .bind(table.id.to_bytes_be().to_vec()) + .bind(json) + .execute(&mut *tx) + .await?; + + if let Some(block_number) = block_number { + let history_json = + serde_json::to_string(table).map_err(|e| sqlx::Error::Protocol(e.to_string()))?; + sqlx::query( + r#" + INSERT INTO torii_dojo_manager_state_history + (owner, table_id, applied_at_block, table_json, updated_at) + VALUES (?1, ?2, ?3, ?4, unixepoch()) + ON CONFLICT (owner, table_id, applied_at_block) + DO UPDATE SET + table_json = excluded.table_json, + updated_at = unixepoch() + "#, + ) + .bind(owner.to_bytes_be().to_vec()) + .bind(table.id.to_bytes_be().to_vec()) + .bind(block_number as i64) + .bind(history_json) + .execute(&mut *tx) + .await?; + } + + tx.commit().await?; + Ok(()) + } + + async fn load_tables(&self, owners: &[Felt]) -> Result, Self::Error> { + let rows = if owners.is_empty() { + sqlx::query( + r#" + SELECT table_json + FROM torii_dojo_manager_state + ORDER BY updated_at ASC + "#, + ) + .fetch_all(&self.pool) + .await? + } else { + let placeholders = std::iter::repeat_n("?", owners.len()).join(", "); + let query = format!( + " + SELECT table_json + FROM torii_dojo_manager_state + WHERE owner IN ({placeholders}) + ORDER BY updated_at ASC + " + ); + let mut query = sqlx::query(&query); + for owner in owners { + query = query.bind(owner.to_bytes_be().to_vec()); + } + query.fetch_all(&self.pool).await? + }; + + rows.into_iter() + .map(|row| { + let json: String = row.try_get("table_json")?; + serde_json::from_str(&json).map_err(|e| sqlx::Error::Protocol(e.to_string())) + }) + .collect() + } + + async fn load_tables_at_blocks( + &self, + owner_blocks: &[(Felt, u64)], + ) -> Result, Self::Error> { + if owner_blocks.is_empty() { + return self.load_tables(&[]).await; + } + + let mut query = QueryBuilder::::new("WITH requested(owner, resume_block) AS ("); + query.push_values(owner_blocks, |mut builder, (owner, block_number)| { + builder + .push_bind(owner.to_bytes_be().to_vec()) + .push_bind(*block_number as i64); + }); + query.push( + ") SELECT history.table_json \ + FROM torii_dojo_manager_state_history AS history \ + JOIN requested \ + ON history.owner = requested.owner \ + AND history.applied_at_block = ( \ + SELECT MAX(h2.applied_at_block) \ + FROM torii_dojo_manager_state_history AS h2 \ + WHERE h2.owner = requested.owner \ + AND h2.table_id = history.table_id \ + AND h2.applied_at_block <= requested.resume_block \ + ) \ + ORDER BY history.updated_at ASC", + ); + + let rows = query.build().fetch_all(&self.pool).await?; + rows.into_iter() + .map(|row| { + let json: String = row.try_get("table_json")?; + serde_json::from_str(&json).map_err(|e| sqlx::Error::Protocol(e.to_string())) + }) + .collect() + } +} + +impl DojoTableStore { + pub async fn new>(path: P) -> DojoToriiResult { + Self::with_store(JsonStore::from(path.into())).await + } +} + +impl DojoTableStore { + pub async fn new_postgres(database_url: &str) -> DojoToriiResult { + Self::with_store(PostgresStore::new(database_url).await?).await + } +} + +impl DojoTableStore { + pub async fn new_sqlite(database_url: &str) -> DojoToriiResult { + Self::with_store(SqliteStore::new(database_url).await?).await + } +} + +fn is_sqlite_memory_path(path: &str) -> bool { + path == ":memory:" + || path == "sqlite::memory:" + || path == "sqlite://:memory:" + || path.contains("mode=memory") +} + +fn sqlite_connect_options(path: &str) -> Result { + if path == ":memory:" || path == "sqlite::memory:" { + return SqliteConnectOptions::from_str("sqlite::memory:") + .map_err(|err| sqlx::Error::Configuration(Box::new(err))); + } + + let options = if path.starts_with("sqlite:") { + SqliteConnectOptions::from_str(path) + .map_err(|err| sqlx::Error::Configuration(Box::new(err)))? + } else { + SqliteConnectOptions::new().filename(path) + }; + + if path.starts_with("sqlite:") && path.contains("mode=") { + Ok(options) + } else { + Ok(options.create_if_missing(true)) + } +} + +fn sqlite_parent_dir(options: &SqliteConnectOptions) -> Option { + let filename = options.get_filename(); + if is_sqlite_memory_path(filename.to_string_lossy().as_ref()) { + return None; + } + filename + .parent() + .map(PathBuf::from) + .filter(|path| !path.as_os_str().is_empty()) +} + +#[derive(Debug, Error)] +pub enum MergedStoreError { + #[error("failed to load primary metadata source: {0}")] + PrimaryLoad(String), + #[error("failed to load secondary metadata source: {0}")] + SecondaryLoad(String), + #[error("failed to load primary metadata source: {primary}; failed to load secondary metadata source: {secondary}")] + BothLoad { primary: String, secondary: String }, + #[error("failed to save primary metadata source: {0}")] + PrimarySave(String), + #[error("failed to save secondary metadata source: {0}")] + SecondarySave(String), + #[error("failed to save primary metadata source: {primary}; failed to save secondary metadata source: {secondary}")] + BothSave { primary: String, secondary: String }, +} + +pub struct MergedStore { + primary: Primary, + secondary: Secondary, +} + +pub struct MergedTableDrift { + pub table_id: Felt, + pub primary_name: String, + pub secondary_name: String, +} + +pub struct MergedLoadResult { + pub tables: Vec, + pub primary_count: usize, + pub secondary_count: usize, + pub merged_count: usize, + pub primary_only: usize, + pub secondary_only: usize, + pub overlapping: usize, + pub drifted: Vec, +} + +impl MergedStore { + pub fn new(primary: Primary, secondary: Secondary) -> Self { + Self { primary, secondary } + } +} + +impl MergedStore +where + Primary: DojoStoreTrait + Send + Sync, + Secondary: DojoStoreTrait + Send + Sync, + Primary::Error: ToString, + Secondary::Error: ToString, +{ + async fn load_tables_diagnostics( + &self, + owners: &[Felt], + ) -> Result { + let primary = self + .primary + .load_tables(owners) + .await + .map_err(|err| err.to_string()); + let secondary = self + .secondary + .load_tables(owners) + .await + .map_err(|err| err.to_string()); + + match (primary, secondary) { + (Ok(primary_tables), Ok(secondary_tables)) => { + Ok(merge_tables(primary_tables, secondary_tables)) + } + (Ok(primary_tables), Err(err)) => { + tracing::warn!( + target: "torii::dojo::metadata", + error = %err, + "Secondary Dojo metadata source unavailable; using primary source only" + ); + Ok(merge_tables(primary_tables, Vec::new())) + } + (Err(err), Ok(secondary_tables)) => { + tracing::warn!( + target: "torii::dojo::metadata", + error = %err, + "Primary Dojo metadata source unavailable; using secondary source only" + ); + Ok(merge_tables(Vec::new(), secondary_tables)) + } + (Err(primary), Err(secondary)) => { + Err(MergedStoreError::BothLoad { primary, secondary }) + } + } + } +} + +fn tables_differ(primary: &DojoTable, secondary: &DojoTable) -> bool { + match ( + serde_json::to_string(primary), + serde_json::to_string(secondary), + ) { + (Ok(primary), Ok(secondary)) => primary != secondary, + _ => primary.name != secondary.name || primary.columns.len() != secondary.columns.len(), + } +} + +fn merge_tables( + primary_tables: Vec, + secondary_tables: Vec, +) -> MergedLoadResult { + let primary_count = primary_tables.len(); + let secondary_count = secondary_tables.len(); + let mut secondary_by_id = secondary_tables + .into_iter() + .map(|table| (table.id, table)) + .collect::>(); + let mut merged_by_id = HashMap::new(); + let mut primary_only = 0usize; + let mut overlapping = 0usize; + let mut drifted = Vec::new(); + + for table in primary_tables { + if let Some(secondary) = secondary_by_id.remove(&table.id) { + overlapping += 1; + if tables_differ(&table, &secondary) { + drifted.push(MergedTableDrift { + table_id: table.id, + primary_name: table.name.clone(), + secondary_name: secondary.name, + }); + } + } else { + primary_only += 1; + } + merged_by_id.insert(table.id, table); + } + + let secondary_only = secondary_by_id.len(); + merged_by_id.extend(secondary_by_id); + + MergedLoadResult { + merged_count: merged_by_id.len(), + tables: merged_by_id.into_values().collect(), + primary_count, + secondary_count, + primary_only, + secondary_only, + overlapping, + drifted, + } +} + +#[async_trait] +impl DojoStoreTrait for MergedStore +where + Primary: DojoStoreTrait + Send + Sync, + Secondary: DojoStoreTrait + Send + Sync, + Primary::Error: ToString, + Secondary::Error: ToString, +{ + type Error = MergedStoreError; + + async fn save_table_at_block( + &self, + owner: &Felt, + table: &DojoTable, + block_number: Option, + ) -> Result<(), Self::Error> { + let primary = self + .primary + .save_table_at_block(owner, table, block_number) + .await + .map_err(|err| err.to_string()); + let secondary = self + .secondary + .save_table_at_block(owner, table, block_number) + .await + .map_err(|err| err.to_string()); + + match (primary, secondary) { + (Ok(()), Ok(())) => Ok(()), + (Err(err), Ok(())) => Err(MergedStoreError::PrimarySave(err)), + (Ok(()), Err(err)) => Err(MergedStoreError::SecondarySave(err)), + (Err(primary), Err(secondary)) => { + Err(MergedStoreError::BothSave { primary, secondary }) + } + } + } + + async fn load_tables(&self, owners: &[Felt]) -> Result, Self::Error> { + self.load_tables_diagnostics(owners) + .await + .map(|result| result.tables) + } + + async fn load_tables_at_blocks( + &self, + owner_blocks: &[(Felt, u64)], + ) -> Result, Self::Error> { + self.primary + .load_tables_at_blocks(owner_blocks) + .await + .map_err(|err| MergedStoreError::PrimaryLoad(err.to_string())) + } +} + +impl DojoTableStore> { + pub async fn new_merged_postgres(database_url: &str) -> DojoToriiResult { + let secondary = PgPoolOptions::new() + .max_connections(5) + .connect(database_url) + .await + .map_err(DojoToriiError::store_error)?; + initialize_dojo_schema(&secondary) + .await + .map_err(DojoToriiError::store_error)?; + let primary = PostgresStore::new(database_url).await?; + let store = MergedStore::new(primary, secondary); + let merged = store + .load_tables_diagnostics(&[]) + .await + .map_err(DojoToriiError::store_error)?; + + tracing::info!( + target: "torii::dojo::metadata", + manager_state_tables = merged.primary_count, + dojo_schema_tables = merged.secondary_count, + merged_tables = merged.merged_count, + manager_only = merged.primary_only, + dojo_only = merged.secondary_only, + overlapping = merged.overlapping, + drifted = merged.drifted.len(), + "Merged Dojo metadata sources for decoder bootstrap" + ); + for drift in &merged.drifted { + tracing::warn!( + target: "torii::dojo::metadata", + table_id = %drift.table_id, + manager_name = %drift.primary_name, + dojo_name = %drift.secondary_name, + "Dojo metadata sources disagree; preferring persisted manager state" + ); + } + + Ok(Self::from_loaded_tables(store, merged.tables)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::store::DojoStoreTrait; + use crate::table::sort_columns; + use introspect_types::{Attribute, ColumnDef, PrimaryDef, PrimaryTypeDef, TypeDef}; + use std::sync::Mutex; + + #[derive(Debug, Error)] + #[error("{0}")] + struct FakeStoreError(String); + + #[derive(Default)] + struct FakeStore { + tables: Vec, + saved: Mutex>, + fail_load: Option, + fail_save: Option, + } + + impl FakeStore { + fn with_tables(tables: Vec) -> Self { + Self { + tables, + ..Self::default() + } + } + } + + #[async_trait] + impl DojoStoreTrait for FakeStore { + type Error = FakeStoreError; + + async fn save_table_at_block( + &self, + _owner: &Felt, + table: &DojoTable, + _block_number: Option, + ) -> Result<(), Self::Error> { + if let Some(err) = &self.fail_save { + return Err(FakeStoreError(err.clone())); + } + self.saved.lock().unwrap().push(table.id); + Ok(()) + } + + async fn load_tables(&self, _owners: &[Felt]) -> Result, Self::Error> { + if let Some(err) = &self.fail_load { + return Err(FakeStoreError(err.clone())); + } + Ok(self.tables.clone()) + } + } + + fn table(id: u64, name: &str, columns: &[u64]) -> DojoTable { + let columns = columns + .iter() + .enumerate() + .map(|(idx, column_id)| ColumnDef { + id: Felt::from(*column_id), + name: format!("field_{idx}"), + attributes: if idx == 0 { + vec![Attribute::new_empty("key".to_string())] + } else { + vec![] + }, + type_def: TypeDef::U32, + }) + .collect::>(); + let (columns, key_fields, value_fields) = sort_columns(columns); + DojoTable { + id: Felt::from(id), + name: name.to_string(), + attributes: vec![], + primary: PrimaryDef { + name: "entity_id".to_string(), + attributes: vec![], + type_def: PrimaryTypeDef::Felt252, + }, + columns, + key_fields, + value_fields, + legacy: false, + } + } + + #[tokio::test] + async fn merged_store_prefers_primary_tables_and_keeps_secondary_only_ids() { + let primary = FakeStore::with_tables(vec![table(1, "dojo-position", &[10, 11])]); + let secondary = FakeStore::with_tables(vec![ + table(1, "manager-position", &[10, 12]), + table(2, "manager-score", &[20, 21]), + ]); + let store = MergedStore::new(primary, secondary); + + let merged = store.load_tables_diagnostics(&[]).await.unwrap(); + let table_names = merged + .tables + .iter() + .map(|table| (table.id, table.name.clone())) + .collect::>(); + + assert_eq!(merged.primary_count, 1); + assert_eq!(merged.secondary_count, 2); + assert_eq!(merged.primary_only, 0); + assert_eq!(merged.secondary_only, 1); + assert_eq!(merged.overlapping, 1); + assert_eq!(merged.merged_count, 2); + assert_eq!(merged.drifted.len(), 1); + assert_eq!( + table_names.get(&Felt::from(1_u64)).unwrap(), + "dojo-position" + ); + assert_eq!( + table_names.get(&Felt::from(2_u64)).unwrap(), + "manager-score" + ); + } + + #[tokio::test] + async fn merged_store_save_writes_to_both_sources() { + let primary = FakeStore::default(); + let secondary = FakeStore::default(); + let store = MergedStore::new(primary, secondary); + let table = table(3, "dojo-health", &[30, 31]); + + store.save_table(&Felt::ONE, &table).await.unwrap(); + + assert_eq!(store.primary.saved.lock().unwrap().as_slice(), &[table.id]); + assert_eq!( + store.secondary.saved.lock().unwrap().as_slice(), + &[table.id] + ); + } + + #[tokio::test] + async fn merged_store_falls_back_to_secondary_when_primary_load_fails() { + let primary = FakeStore { + fail_load: Some("primary unavailable".to_string()), + ..FakeStore::default() + }; + let secondary = FakeStore::with_tables(vec![table(4, "manager-fallback", &[40, 41])]); + let store = MergedStore::new(primary, secondary); + + let merged = store.load_tables_diagnostics(&[]).await.unwrap(); + + assert_eq!(merged.primary_count, 0); + assert_eq!(merged.secondary_count, 1); + assert_eq!(merged.merged_count, 1); + assert_eq!(merged.tables[0].name, "manager-fallback"); + } +} diff --git a/crates/introspect-postgres-sink/Cargo.toml b/crates/introspect-postgres-sink/Cargo.toml index 357b4029..1702540e 100644 --- a/crates/introspect-postgres-sink/Cargo.toml +++ b/crates/introspect-postgres-sink/Cargo.toml @@ -9,6 +9,7 @@ license = "Apache-2.0" [dependencies] sqlx = { workspace = true, features = [ "postgres", + "sqlite", "runtime-tokio-rustls", "macros", "migrate", diff --git a/crates/introspect-postgres-sink/src/lib.rs b/crates/introspect-postgres-sink/src/lib.rs index 83d42122..2d397d05 100644 --- a/crates/introspect-postgres-sink/src/lib.rs +++ b/crates/introspect-postgres-sink/src/lib.rs @@ -4,6 +4,8 @@ pub mod json; pub mod processor; pub mod query; pub mod sink; +pub mod sql; +pub mod sqlite; pub mod table; pub mod types; pub mod upgrade; diff --git a/crates/introspect-postgres-sink/src/sink.rs b/crates/introspect-postgres-sink/src/sink.rs index d33b260b..ac841806 100644 --- a/crates/introspect-postgres-sink/src/sink.rs +++ b/crates/introspect-postgres-sink/src/sink.rs @@ -11,13 +11,58 @@ use torii::etl::{ use torii_introspect::events::{IntrospectBody, IntrospectMsg}; use torii_postgres::PostgresConnection; -pub const LOGGING_TARGET: &str = "torii::sinks::introspect::postgres"; +pub const LOGGING_TARGET: &str = "torii::sinks::introspect"; const INTROSPECT_TYPE: TypeId = TypeId::new("introspect"); +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum DbBackend { + Postgres, + Sqlite, +} + +enum IntrospectDb { + Postgres(PostgresSimpleDb), + Sqlite(SqliteSimpleDb), +} + +impl IntrospectDb { + async fn initialize(&mut self) -> Result<()> { + match self { + Self::Postgres(db) => db.initialize().await.map_err(Into::into), + Self::Sqlite(db) => db.initialize().await.map_err(Into::into), + } + } + + fn has_tables(&self) -> bool { + match self { + Self::Postgres(db) => db.has_tables(), + Self::Sqlite(db) => db.has_tables(), + } + } + + async fn bootstrap_tables(&mut self, tables: &[CreateTable]) -> Result<()> { + match self { + Self::Postgres(db) => db.bootstrap_tables(tables).await.map_err(Into::into), + Self::Sqlite(db) => db.bootstrap_tables(tables).await.map_err(Into::into), + } + } + + async fn process_message( + &mut self, + msg: &IntrospectMsg, + context: &torii::etl::EventContext, + ) -> Result<()> { + match self { + Self::Postgres(db) => db.process_message(msg, context).await.map_err(Into::into), + Self::Sqlite(db) => db.process_message(msg, context).await.map_err(Into::into), + } + } +} + #[async_trait] impl Sink for IntrospectPgDb { fn name(&self) -> &'static str { - "introspect-postgres" + "introspect-sql" } fn interested_types(&self) -> Vec { @@ -95,8 +140,11 @@ impl Sink for IntrospectPgDb { self.initialize_introspect_pg_sink().await?; tracing::info!( target: LOGGING_TARGET, - "Initialized introspect Postgres sink" + backend = ?backend, + "Initialized introspect sink" ); Ok(()) } } + +pub type IntrospectSqlSink = IntrospectPostgresSink; diff --git a/crates/introspect-postgres-sink/src/sqlite.rs b/crates/introspect-postgres-sink/src/sqlite.rs new file mode 100644 index 00000000..d6fb2c31 --- /dev/null +++ b/crates/introspect-postgres-sink/src/sqlite.rs @@ -0,0 +1,748 @@ +use anyhow::Context; +use introspect_types::serialize::ToCairoDeSeFrom; +use introspect_types::serialize_def::CairoTypeSerialization; +use introspect_types::{ + CairoDeserializer, ColumnDef, PrimaryDef, PrimaryTypeDef, ResultDef, TupleDef, TypeDef, +}; +use serde::ser::SerializeMap; +use serde::Serializer; +use serde_json::{Serializer as JsonSerializer, Value}; +use sqlx::{ + sqlite::{SqliteConnectOptions, SqlitePoolOptions}, + Row, SqlitePool, +}; +use starknet_types_core::felt::Felt; +use std::collections::{HashMap, HashSet}; +use std::path::PathBuf; +use std::str::FromStr; +use thiserror::Error; +use torii::etl::EventContext; +use torii_introspect::events::{InsertsFields, IntrospectMsg}; +use torii_introspect::{CreateTable, UpdateTable}; + +#[derive(Debug, Error)] +pub enum SqliteDbError { + #[error(transparent)] + Database(#[from] sqlx::Error), + #[error(transparent)] + Json(#[from] serde_json::Error), + #[error("table with id {0:#x} not found")] + TableNotFound(Felt), + #[error("table with id {0:#x} is no longer live")] + TableNotAlive(Felt), + #[error("column with id {column_id:#x} not found in table {table_name}")] + ColumnNotFound { table_name: String, column_id: Felt }, + #[error("table with id {0:#x} already exists as {2}, incoming name {1}")] + TableAlreadyExists(Felt, String, String), + #[error("record missing column {0}")] + MissingColumn(String), +} + +type SqliteResult = Result; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum SqliteAffinity { + Integer, + Text, +} + +impl SqliteAffinity { + fn sql_type(self) -> &'static str { + match self { + Self::Integer => "INTEGER", + Self::Text => "TEXT", + } + } +} + +#[derive(Debug)] +struct SqliteTable { + name: String, + primary: PrimaryDef, + columns: HashMap, + alive: bool, +} + +impl SqliteTable { + fn new(event: CreateTable) -> (Felt, Self) { + let columns = event + .columns + .into_iter() + .map(|column| (column.id, column)) + .collect(); + + ( + event.id, + Self { + name: event.name, + primary: event.primary, + columns, + alive: true, + }, + ) + } + + fn get_living_column(&self, id: &Felt) -> SqliteResult<&ColumnDef> { + self.columns + .get(id) + .ok_or_else(|| SqliteDbError::ColumnNotFound { + table_name: self.name.clone(), + column_id: *id, + }) + } +} + +#[derive(Debug, Default)] +struct SqliteSchema { + tables: HashMap, +} + +impl SqliteSchema { + fn get_living_table(&self, id: &Felt) -> SqliteResult<&SqliteTable> { + let table = self + .tables + .get(id) + .ok_or(SqliteDbError::TableNotFound(*id))?; + if table.alive { + Ok(table) + } else { + Err(SqliteDbError::TableNotAlive(*id)) + } + } + + fn create_table_sql(&mut self, event: CreateTable) -> SqliteResult<(String, String)> { + if let Some(existing) = self.tables.get(&event.id) { + if existing.name == event.name { + return Ok((existing.name.clone(), String::new())); + } + return Err(SqliteDbError::TableAlreadyExists( + event.id, + event.name, + existing.name.clone(), + )); + } + + let table_name = event.name.clone(); + let create_sql = build_create_table_sql(&event); + let (id, table) = SqliteTable::new(event); + self.tables.insert(id, table); + Ok((table_name, create_sql)) + } + + fn update_table_sql(&mut self, event: UpdateTable) -> SqliteResult<(String, Vec)> { + let snapshot = CreateTable { + id: event.id, + name: event.name, + attributes: event.attributes, + primary: event.primary, + columns: event.columns, + }; + + let Some(existing) = self.tables.get(&snapshot.id) else { + let (table_name, create_sql) = self.create_table_sql(snapshot)?; + return Ok((table_name, vec![create_sql])); + }; + + if existing.name != snapshot.name { + return Err(SqliteDbError::TableAlreadyExists( + snapshot.id, + snapshot.name, + existing.name.clone(), + )); + } + + let mut alterations = Vec::new(); + for column in &snapshot.columns { + if !existing.columns.contains_key(&column.id) { + alterations.push(build_add_column_sql(&existing.name, column)); + } + } + + let table_name = existing.name.clone(); + let (_, replacement) = SqliteTable::new(snapshot); + self.tables.insert(event.id, replacement); + + Ok((table_name, alterations)) + } + + fn set_table_dead(&mut self, id: &Felt) -> SqliteResult<()> { + let table = self + .tables + .get_mut(id) + .ok_or(SqliteDbError::TableNotFound(*id))?; + table.alive = false; + Ok(()) + } + + fn insert_rows(&self, event: &InsertsFields) -> SqliteResult<(String, Vec)> { + let table = self.get_living_table(&event.table)?; + let records = sqlite_records_for_event(table, event)?; + Ok((table.name.clone(), records)) + } +} + +#[derive(Debug)] +struct SqliteRow { + columns: Vec, + primary: String, + values: Vec<(SqliteAffinity, Value)>, +} + +pub struct SqliteSimpleDb { + schema: SqliteSchema, + pool: SqlitePool, +} + +impl SqliteSimpleDb { + pub async fn new(database_url: &str, max_connections: Option) -> SqliteResult { + let options = sqlite_connect_options(database_url)?; + if let Some(parent) = sqlite_parent_dir(&options) { + std::fs::create_dir_all(parent).map_err(sqlx::Error::Io)?; + } + + let pool = SqlitePoolOptions::new() + .max_connections(max_connections.unwrap_or(1)) + .connect_with(options) + .await?; + + Ok(Self { + schema: SqliteSchema::default(), + pool, + }) + } + + pub async fn initialize(&mut self) -> SqliteResult<()> { + sqlx::query( + r#" + CREATE TABLE IF NOT EXISTS torii_introspect_schema_state ( + table_id TEXT PRIMARY KEY, + create_table_json TEXT NOT NULL, + updated_at INTEGER NOT NULL DEFAULT (unixepoch()) + ) + "#, + ) + .execute(&self.pool) + .await?; + + let tables = self.load_stored_tables().await?; + if !tables.is_empty() { + self.bootstrap_tables(&tables).await?; + tracing::info!( + target: "torii::sinks::introspect::sqlite", + tables = tables.len(), + "Loaded sink schema state from SQLite storage" + ); + } + + Ok(()) + } + + pub fn has_tables(&self) -> bool { + !self.schema.tables.is_empty() + } + + pub async fn bootstrap_tables(&mut self, tables: &[CreateTable]) -> SqliteResult<()> { + let context = EventContext::default(); + for table in tables { + self.process_message(&IntrospectMsg::CreateTable(table.clone()), &context) + .await?; + self.reconcile_table_columns(&table.name, &table.columns) + .await?; + } + Ok(()) + } + + pub async fn process_message( + &mut self, + msg: &IntrospectMsg, + _context: &EventContext, + ) -> SqliteResult<()> { + match msg { + IntrospectMsg::CreateTable(table) => { + let (table_name, create_sql) = self.schema.create_table_sql(table.clone())?; + if !create_sql.is_empty() { + sqlx::query(&create_sql).execute(&self.pool).await?; + } + self.reconcile_table_columns(&table_name, &table.columns) + .await?; + self.persist_table_state(table).await?; + } + IntrospectMsg::UpdateTable(table) => { + let (table_name, alter_sql) = self.schema.update_table_sql(table.clone())?; + for sql in alter_sql { + sqlx::query(&sql).execute(&self.pool).await?; + } + self.reconcile_table_columns(&table_name, &table.columns) + .await?; + self.persist_table_state(&CreateTable { + id: table.id, + name: table.name.clone(), + attributes: table.attributes.clone(), + primary: table.primary.clone(), + columns: table.columns.clone(), + }) + .await?; + } + IntrospectMsg::AddColumns(event) => { + self.schema.set_table_dead(&event.table)?; + } + IntrospectMsg::DropColumns(event) => { + self.schema.set_table_dead(&event.table)?; + } + IntrospectMsg::RetypeColumns(event) => { + self.schema.set_table_dead(&event.table)?; + } + IntrospectMsg::RetypePrimary(event) => { + self.schema.set_table_dead(&event.table)?; + } + IntrospectMsg::InsertsFields(event) => { + let (table_name, rows) = self.schema.insert_rows(event)?; + for row in rows { + self.insert_row(&table_name, row).await?; + } + } + IntrospectMsg::RenameTable(_) + | IntrospectMsg::DropTable(_) + | IntrospectMsg::RenameColumns(_) + | IntrospectMsg::RenamePrimary(_) + | IntrospectMsg::DeleteRecords(_) + | IntrospectMsg::DeletesFields(_) => {} + } + + Ok(()) + } + + async fn reconcile_table_columns( + &self, + table_name: &str, + columns: &[ColumnDef], + ) -> SqliteResult<()> { + let existing = self.table_columns(table_name).await?; + for column in columns { + if existing.contains(&column.name) { + continue; + } + + let sql = build_add_column_sql(table_name, column); + sqlx::query(&sql).execute(&self.pool).await?; + } + Ok(()) + } + + async fn table_columns(&self, table_name: &str) -> SqliteResult> { + let pragma = format!("PRAGMA table_info({})", quote_ident(table_name)); + let rows = sqlx::query(&pragma).fetch_all(&self.pool).await?; + Ok(rows + .into_iter() + .filter_map(|row| row.try_get::("name").ok()) + .collect()) + } + + async fn load_stored_tables(&self) -> SqliteResult> { + let rows = sqlx::query( + r#" + SELECT create_table_json + FROM torii_introspect_schema_state + ORDER BY updated_at ASC + "#, + ) + .fetch_all(&self.pool) + .await?; + + rows.into_iter() + .map(|row| { + let json: String = row.try_get("create_table_json")?; + Ok(serde_json::from_str(&json)?) + }) + .collect() + } + + async fn persist_table_state(&self, table: &CreateTable) -> SqliteResult<()> { + let json = serde_json::to_string(table)?; + sqlx::query( + r#" + INSERT INTO torii_introspect_schema_state (table_id, create_table_json, updated_at) + VALUES (?1, ?2, unixepoch()) + ON CONFLICT (table_id) + DO UPDATE SET + create_table_json = excluded.create_table_json, + updated_at = unixepoch() + "#, + ) + .bind(felt_key(&table.id)) + .bind(json) + .execute(&self.pool) + .await?; + Ok(()) + } + + async fn insert_row(&self, table_name: &str, row: SqliteRow) -> SqliteResult<()> { + let column_names = row + .columns + .iter() + .map(|column| quote_ident(column)) + .collect::>() + .join(", "); + let placeholders = vec!["?"; row.columns.len()].join(", "); + let conflict_sql = row + .columns + .iter() + .filter(|column| *column != &row.primary) + .map(|column| { + format!( + "{column_ident} = COALESCE(excluded.{column_ident}, {table_ident}.{column_ident})", + column_ident = quote_ident(column), + table_ident = quote_ident(table_name), + ) + }) + .collect::>(); + + let sql = if conflict_sql.is_empty() { + format!( + "INSERT INTO {table_ident} ({column_names}) VALUES ({placeholders}) \ + ON CONFLICT ({primary}) DO NOTHING", + table_ident = quote_ident(table_name), + primary = quote_ident(&row.primary), + ) + } else { + format!( + "INSERT INTO {table_ident} ({column_names}) VALUES ({placeholders}) \ + ON CONFLICT ({primary}) DO UPDATE SET {updates}", + table_ident = quote_ident(table_name), + primary = quote_ident(&row.primary), + updates = conflict_sql.join(", "), + ) + }; + + let mut query = sqlx::query(&sql); + for (affinity, value) in row.values { + query = bind_value(query, affinity, value); + } + query.execute(&self.pool).await?; + Ok(()) + } +} + +fn sqlite_records_for_event( + table: &SqliteTable, + event: &InsertsFields, +) -> SqliteResult> { + let schema = torii_introspect::tables::RecordSchema::new( + &table.primary, + event + .columns + .iter() + .map(|column_id| table.get_living_column(column_id)) + .collect::>>()?, + ); + + let mut bytes = Vec::new(); + let mut serializer = JsonSerializer::new(&mut bytes); + schema.parse_records_with_metadata( + &event.records, + &(), + &mut serializer, + &SqliteJsonSerializer, + )?; + + let rows = serde_json::from_slice::>(&bytes)?; + rows.into_iter() + .map(|row| sqlite_row_from_value(&schema, row)) + .collect() +} + +fn sqlite_row_from_value( + schema: &torii_introspect::tables::RecordSchema<'_>, + row: Value, +) -> SqliteResult { + let object = row + .as_object() + .cloned() + .context("serialized record must be an object") + .map_err(|err| SqliteDbError::Json(serde_json::Error::io(std::io::Error::other(err))))?; + + let mut columns = Vec::with_capacity(1 + schema.columns().len()); + let mut values = Vec::with_capacity(1 + schema.columns().len()); + + columns.push(schema.primary().name.clone()); + values.push(( + affinity_for_primary(&schema.primary().type_def), + object + .get(&schema.primary().name) + .cloned() + .ok_or_else(|| SqliteDbError::MissingColumn(schema.primary().name.clone()))?, + )); + + for column in schema.columns() { + columns.push(column.name.clone()); + values.push(( + affinity_for_type(&column.type_def), + object + .get(&column.name) + .cloned() + .ok_or_else(|| SqliteDbError::MissingColumn(column.name.clone()))?, + )); + } + + Ok(SqliteRow { + columns, + primary: schema.primary().name.clone(), + values, + }) +} + +fn bind_value<'q>( + query: sqlx::query::Query<'q, sqlx::Sqlite, sqlx::sqlite::SqliteArguments<'q>>, + affinity: SqliteAffinity, + value: Value, +) -> sqlx::query::Query<'q, sqlx::Sqlite, sqlx::sqlite::SqliteArguments<'q>> { + match affinity { + SqliteAffinity::Integer => match value { + Value::Bool(value) => query.bind(i64::from(value)), + Value::Number(number) => { + if let Some(value) = number.as_i64() { + query.bind(value) + } else if let Some(value) = number.as_u64() { + match i64::try_from(value) { + Ok(value) => query.bind(value), + Err(_) => query.bind(number.to_string()), + } + } else { + query.bind(number.to_string()) + } + } + Value::String(value) => query.bind(value), + Value::Null => query.bind(Option::::None), + other => query.bind(other.to_string()), + }, + SqliteAffinity::Text => match value { + Value::Null => query.bind(Option::::None), + Value::String(value) => query.bind(value), + other => query.bind(other.to_string()), + }, + } +} + +fn build_create_table_sql(event: &CreateTable) -> String { + let mut columns = Vec::with_capacity(1 + event.columns.len()); + columns.push(format!( + "{} {} PRIMARY KEY", + quote_ident(&event.primary.name), + affinity_for_primary(&event.primary.type_def).sql_type() + )); + columns.extend(event.columns.iter().map(|column| { + format!( + "{} {}", + quote_ident(&column.name), + affinity_for_type(&column.type_def).sql_type() + ) + })); + + format!( + "CREATE TABLE IF NOT EXISTS {} ({})", + quote_ident(&event.name), + columns.join(", ") + ) +} + +fn build_add_column_sql(table_name: &str, column: &ColumnDef) -> String { + format!( + "ALTER TABLE {} ADD COLUMN {} {}", + quote_ident(table_name), + quote_ident(&column.name), + affinity_for_type(&column.type_def).sql_type() + ) +} + +fn affinity_for_primary(type_def: &PrimaryTypeDef) -> SqliteAffinity { + match type_def { + PrimaryTypeDef::Bool + | PrimaryTypeDef::I8 + | PrimaryTypeDef::I16 + | PrimaryTypeDef::I32 + | PrimaryTypeDef::I64 + | PrimaryTypeDef::U8 + | PrimaryTypeDef::U16 + | PrimaryTypeDef::U32 => SqliteAffinity::Integer, + PrimaryTypeDef::U64 + | PrimaryTypeDef::U128 + | PrimaryTypeDef::I128 + | PrimaryTypeDef::Felt252 + | PrimaryTypeDef::ContractAddress + | PrimaryTypeDef::ClassHash + | PrimaryTypeDef::StorageAddress + | PrimaryTypeDef::StorageBaseAddress + | PrimaryTypeDef::EthAddress + | PrimaryTypeDef::ShortUtf8 + | PrimaryTypeDef::Bytes31 + | PrimaryTypeDef::Bytes31Encoded(_) => SqliteAffinity::Text, + } +} + +fn affinity_for_type(type_def: &TypeDef) -> SqliteAffinity { + match type_def { + TypeDef::Bool + | TypeDef::I8 + | TypeDef::I16 + | TypeDef::I32 + | TypeDef::I64 + | TypeDef::U8 + | TypeDef::U16 + | TypeDef::U32 => SqliteAffinity::Integer, + TypeDef::None + | TypeDef::U64 + | TypeDef::U128 + | TypeDef::I128 + | TypeDef::U256 + | TypeDef::U512 + | TypeDef::Felt252 + | TypeDef::ContractAddress + | TypeDef::ClassHash + | TypeDef::StorageAddress + | TypeDef::StorageBaseAddress + | TypeDef::EthAddress + | TypeDef::Utf8String + | TypeDef::ShortUtf8 + | TypeDef::ByteArray + | TypeDef::ByteArrayEncoded(_) + | TypeDef::Bytes31 + | TypeDef::Bytes31Encoded(_) + | TypeDef::Tuple(_) + | TypeDef::Enum(_) + | TypeDef::Array(_) + | TypeDef::FixedArray(_) + | TypeDef::Struct(_) + | TypeDef::Option(_) + | TypeDef::Nullable(_) + | TypeDef::Felt252Dict(_) + | TypeDef::Result(_) + | TypeDef::Ref(_) + | TypeDef::Custom(_) => SqliteAffinity::Text, + } +} + +fn quote_ident(name: &str) -> String { + format!("\"{}\"", name.replace('"', "\"\"")) +} + +fn felt_key(value: &Felt) -> String { + format!("{value:#x}") +} + +struct SqliteJsonSerializer; + +fn is_sqlite_memory_path(path: &str) -> bool { + path == ":memory:" + || path == "sqlite::memory:" + || path == "sqlite://:memory:" + || path.contains("mode=memory") +} + +fn sqlite_connect_options(path: &str) -> Result { + if path == ":memory:" || path == "sqlite::memory:" { + return SqliteConnectOptions::from_str("sqlite::memory:") + .map_err(|err| sqlx::Error::Configuration(Box::new(err))); + } + + let options = if path.starts_with("sqlite:") { + SqliteConnectOptions::from_str(path) + .map_err(|err| sqlx::Error::Configuration(Box::new(err)))? + } else { + SqliteConnectOptions::new().filename(path) + }; + + if path.starts_with("sqlite:") && path.contains("mode=") { + Ok(options) + } else { + Ok(options.create_if_missing(true)) + } +} + +fn sqlite_parent_dir(options: &SqliteConnectOptions) -> Option { + let filename = options.get_filename(); + if is_sqlite_memory_path(filename.to_string_lossy().as_ref()) { + return None; + } + filename + .parent() + .map(PathBuf::from) + .filter(|path| !path.as_os_str().is_empty()) +} + +impl CairoTypeSerialization for SqliteJsonSerializer { + fn serialize_byte_array( + &self, + serializer: S, + value: &[u8], + ) -> Result { + serializer.serialize_str(&format!("0x{}", hex::encode(value))) + } + + fn serialize_felt( + &self, + serializer: S, + value: &[u8; 32], + ) -> Result { + self.serialize_byte_array(serializer, value) + } + + fn serialize_eth_address( + &self, + serializer: S, + value: &[u8; 20], + ) -> Result { + self.serialize_byte_array(serializer, value) + } + + fn serialize_tuple<'a, S: Serializer>( + &'a self, + data: &mut impl CairoDeserializer, + serializer: S, + tuple: &'a TupleDef, + ) -> Result { + let mut seq = serializer.serialize_map(Some(tuple.elements.len()))?; + for (index, element) in tuple.elements.iter().enumerate() { + seq.serialize_entry(&format!("_{index}"), &element.to_de_se(data, self))?; + } + seq.end() + } + + fn serialize_variant<'a, S: Serializer>( + &'a self, + data: &mut impl CairoDeserializer, + serializer: S, + name: &str, + type_def: &'a TypeDef, + ) -> Result { + match type_def { + TypeDef::None => { + let mut map = serializer.serialize_map(Some(1))?; + map.serialize_entry("variant", name)?; + map + } + _ => { + let mut map = serializer.serialize_map(Some(2))?; + map.serialize_entry("variant", name)?; + map.serialize_entry(&format!("_{name}"), &type_def.to_de_se(data, self))?; + map + } + } + .end() + } + + fn serialize_result<'a, S: Serializer>( + &'a self, + data: &mut impl CairoDeserializer, + serializer: S, + result: &'a ResultDef, + is_ok: bool, + ) -> Result { + let mut map = serializer.serialize_map(Some(2))?; + map.serialize_entry("is_ok", &is_ok)?; + match is_ok { + true => map.serialize_entry("Ok", &result.ok.to_de_se(data, self))?, + false => map.serialize_entry("Err", &result.err.to_de_se(data, self))?, + } + map.end() + } +} From d9539853980a846549b9ac86fa6d9b4bfa382fe5 Mon Sep 17 00:00:00 2001 From: MartianGreed Date: Tue, 10 Mar 2026 18:44:38 +0100 Subject: [PATCH 02/10] feat(ecs): port ecs server --- Cargo.lock | 32 + Cargo.toml | 2 + bins/torii-introspect-bin/Cargo.toml | 1 + bins/torii-introspect-bin/src/main.rs | 19 + crates/dojo/src/manager.rs | 20 +- .../introspect-postgres-sink/src/processor.rs | 39 + crates/introspect-postgres-sink/src/sqlite.rs | 86 +- crates/torii-ecs-sink/Cargo.toml | 34 + crates/torii-ecs-sink/README.md | 88 + crates/torii-ecs-sink/build.rs | 15 + crates/torii-ecs-sink/ecs-grpc.http | 159 ++ crates/torii-ecs-sink/proto/schema.proto | 66 + crates/torii-ecs-sink/proto/types.proto | 169 ++ crates/torii-ecs-sink/proto/world.proto | 76 + crates/torii-ecs-sink/src/grpc_service.rs | 2542 +++++++++++++++++ crates/torii-ecs-sink/src/lib.rs | 19 + crates/torii-ecs-sink/src/sink.rs | 222 ++ scripts/test-ecs-grpc-e2e.sh | 240 ++ 18 files changed, 3780 insertions(+), 49 deletions(-) create mode 100644 crates/torii-ecs-sink/Cargo.toml create mode 100644 crates/torii-ecs-sink/README.md create mode 100644 crates/torii-ecs-sink/build.rs create mode 100644 crates/torii-ecs-sink/ecs-grpc.http create mode 100644 crates/torii-ecs-sink/proto/schema.proto create mode 100644 crates/torii-ecs-sink/proto/types.proto create mode 100644 crates/torii-ecs-sink/proto/world.proto create mode 100644 crates/torii-ecs-sink/src/grpc_service.rs create mode 100644 crates/torii-ecs-sink/src/lib.rs create mode 100644 crates/torii-ecs-sink/src/sink.rs create mode 100755 scripts/test-ecs-grpc-e2e.sh diff --git a/Cargo.lock b/Cargo.lock index 30cb16a1..3bb64755 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4682,6 +4682,34 @@ dependencies = [ "tracing", ] +[[package]] +name = "torii-ecs-sink" +version = "0.1.0" +dependencies = [ + "anyhow", + "async-trait", + "chrono", + "dojo-introspect", + "futures", + "futures-util", + "hex", + "introspect-types", + "prost 0.13.5", + "prost-types 0.13.5", + "serde", + "serde_json", + "sqlx", + "starknet", + "tokio", + "tokio-stream", + "tonic", + "tonic-build", + "torii", + "torii-dojo", + "torii-introspect", + "tracing", +] + [[package]] name = "torii-erc1155" version = "0.1.0" @@ -4830,9 +4858,13 @@ dependencies = [ "sqlx", "starknet", "tokio", + "tonic", + "tonic-reflection", + "tonic-web", "torii", "torii-config-common", "torii-dojo", + "torii-ecs-sink", "torii-introspect-postgres-sink", "torii-introspect-sqlite-sink", "torii-sqlite", diff --git a/Cargo.toml b/Cargo.toml index 47989e6c..2fc08306 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,6 +6,7 @@ members = [ "crates/torii-runtime-common", "crates/torii-sql-sink", "crates/torii-log-sink", + "crates/torii-ecs-sink", "crates/torii-erc20", "crates/torii-erc721", "crates/torii-erc1155", @@ -126,6 +127,7 @@ torii-runtime-common = { path = "crates/torii-runtime-common" } # Internal crates torii-sql-sink.path = "crates/torii-sql-sink" torii-log-sink.path = "crates/torii-log-sink" +torii-ecs-sink.path = "crates/torii-ecs-sink" torii-erc20.path = "crates/torii-erc20" torii-erc721.path = "crates/torii-erc721" torii-erc1155.path = "crates/torii-erc1155" diff --git a/bins/torii-introspect-bin/Cargo.toml b/bins/torii-introspect-bin/Cargo.toml index e255c6c1..de096c99 100644 --- a/bins/torii-introspect-bin/Cargo.toml +++ b/bins/torii-introspect-bin/Cargo.toml @@ -12,6 +12,7 @@ path = "src/main.rs" torii = { path = "../../" } torii-config-common.workspace = true torii-dojo = { path = "../../crates/dojo" } +torii-ecs-sink = { path = "../../crates/torii-ecs-sink" } torii-introspect-postgres-sink = { path = "../../crates/introspect-postgres-sink" } torii-introspect-sqlite-sink = { path = "../../crates/introspect-sqlite-sink" } torii-sqlite = { path = "../../crates/sqlite" } diff --git a/bins/torii-introspect-bin/src/main.rs b/bins/torii-introspect-bin/src/main.rs index 168ec5c7..4771b8df 100644 --- a/bins/torii-introspect-bin/src/main.rs +++ b/bins/torii-introspect-bin/src/main.rs @@ -200,8 +200,26 @@ async fn run_with_postgres( let decoder: Arc = Arc::new(decoder); + let ecs_sink = EcsSink::new(&storage_database_url, config.max_db_connections).await?; + let ecs_grpc_service = ecs_sink.get_grpc_service_impl(); + + let reflection = tonic_reflection::server::Builder::configure() + .register_encoded_file_descriptor_set(torii::TORII_DESCRIPTOR_SET) + .register_encoded_file_descriptor_set(ECS_DESCRIPTOR_SET) + .build_v1() + .expect("failed to build ECS reflection service"); + + let grpc_router = tonic::transport::Server::builder() + .accept_http1(true) + .add_service(tonic_web::enable(WorldServer::new( + (*ecs_grpc_service).clone(), + ))) + .add_service(tonic_web::enable(reflection)); + let mut torii_config = torii::ToriiConfig::builder() .port(config.port) + .with_grpc_router(grpc_router) + .with_custom_reflection(true) .engine_database_url(engine_database_url) .with_extractor(extractor) .add_decoder(decoder) @@ -216,6 +234,7 @@ async fn run_with_postgres( tracing::info!("Torii configured, starting ETL pipeline..."); tracing::info!("gRPC service available on port {}", config.port); tracing::info!(" - torii.Torii (core subscriptions and metrics endpoint)"); + tracing::info!(" - world.World (legacy ECS gRPC service)"); torii::run(torii_config.build()) .await diff --git a/crates/dojo/src/manager.rs b/crates/dojo/src/manager.rs index ece7ade0..09005bea 100644 --- a/crates/dojo/src/manager.rs +++ b/crates/dojo/src/manager.rs @@ -294,7 +294,7 @@ impl SqliteStore { async fn initialize(&self) -> Result<(), sqlx::Error> { sqlx::query( - r#" + r" CREATE TABLE IF NOT EXISTS torii_dojo_manager_state ( owner BLOB NOT NULL, table_id BLOB NOT NULL, @@ -302,13 +302,13 @@ impl SqliteStore { updated_at INTEGER NOT NULL DEFAULT (unixepoch()), PRIMARY KEY (owner, table_id) ) - "#, + ", ) .execute(&self.pool) .await?; sqlx::query( - r#" + r" CREATE TABLE IF NOT EXISTS torii_dojo_manager_state_history ( owner BLOB NOT NULL, table_id BLOB NOT NULL, @@ -317,7 +317,7 @@ impl SqliteStore { updated_at INTEGER NOT NULL DEFAULT (unixepoch()), PRIMARY KEY (owner, table_id, applied_at_block) ) - "#, + ", ) .execute(&self.pool) .await?; @@ -560,14 +560,14 @@ impl DojoStoreTrait for SqliteStore { serde_json::to_string(table).map_err(|e| sqlx::Error::Protocol(e.to_string()))?; let mut tx = self.pool.begin().await?; sqlx::query( - r#" + r" INSERT INTO torii_dojo_manager_state (owner, table_id, table_json, updated_at) VALUES (?1, ?2, ?3, unixepoch()) ON CONFLICT (owner, table_id) DO UPDATE SET table_json = excluded.table_json, updated_at = unixepoch() - "#, + ", ) .bind(owner.to_bytes_be().to_vec()) .bind(table.id.to_bytes_be().to_vec()) @@ -579,7 +579,7 @@ impl DojoStoreTrait for SqliteStore { let history_json = serde_json::to_string(table).map_err(|e| sqlx::Error::Protocol(e.to_string()))?; sqlx::query( - r#" + r" INSERT INTO torii_dojo_manager_state_history (owner, table_id, applied_at_block, table_json, updated_at) VALUES (?1, ?2, ?3, ?4, unixepoch()) @@ -587,7 +587,7 @@ impl DojoStoreTrait for SqliteStore { DO UPDATE SET table_json = excluded.table_json, updated_at = unixepoch() - "#, + ", ) .bind(owner.to_bytes_be().to_vec()) .bind(table.id.to_bytes_be().to_vec()) @@ -604,11 +604,11 @@ impl DojoStoreTrait for SqliteStore { async fn load_tables(&self, owners: &[Felt]) -> Result, Self::Error> { let rows = if owners.is_empty() { sqlx::query( - r#" + r" SELECT table_json FROM torii_dojo_manager_state ORDER BY updated_at ASC - "#, + ", ) .fetch_all(&self.pool) .await? diff --git a/crates/introspect-postgres-sink/src/processor.rs b/crates/introspect-postgres-sink/src/processor.rs index 053b2437..0f6589a5 100644 --- a/crates/introspect-postgres-sink/src/processor.rs +++ b/crates/introspect-postgres-sink/src/processor.rs @@ -182,6 +182,45 @@ impl PostgresTables { Ok(()) } + pub fn prepared_insert_fields( + &self, + event: &InsertsFields, + ) -> PGSinkResult> { + let table = self.get_living_table(&event.table)?; + let schema = table.get_schema(&event.columns)?; + let table_name = table.name(); + + let mut json_writer = Vec::new(); + schema.parse_records_with_metadata( + &event.records, + &(), + &mut JsonSerializer::new(&mut json_writer), + &PostgresJsonSerializer, + )?; + let payload = unsafe { String::from_utf8_unchecked(json_writer) }; + + let mut sql_writer = Vec::new(); + write!( + sql_writer, + r#"INSERT INTO "{table_name}" + SELECT * FROM jsonb_populate_recordset(NULL::"{table_name}", $1::jsonb) + ON CONFLICT ("{}") DO "#, + schema.primary().name + )?; + if let Some((coln, cols)) = schema.columns().split_last() { + write!(&mut sql_writer, "UPDATE SET ")?; + for column in cols { + write_conflict_res::(&mut sql_writer, table_name, &column.name)?; + } + write_conflict_res::(&mut sql_writer, table_name, &coln.name)?; + } else { + write!(&mut sql_writer, "NOTHING")?; + } + + let sql = unsafe { String::from_utf8_unchecked(sql_writer) }; + Ok(Some((sql, payload))) + } + pub fn handle_message( &self, schema: &Rc, diff --git a/crates/introspect-postgres-sink/src/sqlite.rs b/crates/introspect-postgres-sink/src/sqlite.rs index d6fb2c31..b8dd5cba 100644 --- a/crates/introspect-postgres-sink/src/sqlite.rs +++ b/crates/introspect-postgres-sink/src/sqlite.rs @@ -297,8 +297,11 @@ impl SqliteSimpleDb { } IntrospectMsg::InsertsFields(event) => { let (table_name, rows) = self.schema.insert_rows(event)?; + let insert_sql = rows + .first() + .map(|row| build_insert_row_sql(&table_name, row)); for row in rows { - self.insert_row(&table_name, row).await?; + self.insert_row(insert_sql.as_deref(), row).await?; } } IntrospectMsg::RenameTable(_) @@ -376,45 +379,11 @@ impl SqliteSimpleDb { Ok(()) } - async fn insert_row(&self, table_name: &str, row: SqliteRow) -> SqliteResult<()> { - let column_names = row - .columns - .iter() - .map(|column| quote_ident(column)) - .collect::>() - .join(", "); - let placeholders = vec!["?"; row.columns.len()].join(", "); - let conflict_sql = row - .columns - .iter() - .filter(|column| *column != &row.primary) - .map(|column| { - format!( - "{column_ident} = COALESCE(excluded.{column_ident}, {table_ident}.{column_ident})", - column_ident = quote_ident(column), - table_ident = quote_ident(table_name), - ) - }) - .collect::>(); - - let sql = if conflict_sql.is_empty() { - format!( - "INSERT INTO {table_ident} ({column_names}) VALUES ({placeholders}) \ - ON CONFLICT ({primary}) DO NOTHING", - table_ident = quote_ident(table_name), - primary = quote_ident(&row.primary), - ) - } else { - format!( - "INSERT INTO {table_ident} ({column_names}) VALUES ({placeholders}) \ - ON CONFLICT ({primary}) DO UPDATE SET {updates}", - table_ident = quote_ident(table_name), - primary = quote_ident(&row.primary), - updates = conflict_sql.join(", "), - ) + async fn insert_row(&self, sql: Option<&str>, row: SqliteRow) -> SqliteResult<()> { + let Some(sql) = sql else { + return Ok(()); }; - - let mut query = sqlx::query(&sql); + let mut query = sqlx::query(sql).persistent(true); for (affinity, value) in row.values { query = bind_value(query, affinity, value); } @@ -423,6 +392,45 @@ impl SqliteSimpleDb { } } +fn build_insert_row_sql(table_name: &str, row: &SqliteRow) -> String { + let column_names = row + .columns + .iter() + .map(|column| quote_ident(column)) + .collect::>() + .join(", "); + let placeholders = vec!["?"; row.columns.len()].join(", "); + let conflict_sql = row + .columns + .iter() + .filter(|column| *column != &row.primary) + .map(|column| { + format!( + "{column_ident} = COALESCE(excluded.{column_ident}, {table_ident}.{column_ident})", + column_ident = quote_ident(column), + table_ident = quote_ident(table_name), + ) + }) + .collect::>(); + + if conflict_sql.is_empty() { + format!( + "INSERT INTO {table_ident} ({column_names}) VALUES ({placeholders}) \ + ON CONFLICT ({primary}) DO NOTHING", + table_ident = quote_ident(table_name), + primary = quote_ident(&row.primary), + ) + } else { + format!( + "INSERT INTO {table_ident} ({column_names}) VALUES ({placeholders}) \ + ON CONFLICT ({primary}) DO UPDATE SET {updates}", + table_ident = quote_ident(table_name), + primary = quote_ident(&row.primary), + updates = conflict_sql.join(", "), + ) + } +} + fn sqlite_records_for_event( table: &SqliteTable, event: &InsertsFields, diff --git a/crates/torii-ecs-sink/Cargo.toml b/crates/torii-ecs-sink/Cargo.toml new file mode 100644 index 00000000..03760390 --- /dev/null +++ b/crates/torii-ecs-sink/Cargo.toml @@ -0,0 +1,34 @@ +[package] +name = "torii-ecs-sink" +version = "0.1.0" +edition = "2021" + +[dependencies] +torii = { path = "../.." } +torii-dojo = { path = "../dojo" } +torii-introspect = { path = "../introspect" } +dojo-introspect.workspace = true +introspect-types.workspace = true + +anyhow.workspace = true +async-trait.workspace = true +chrono.workspace = true +futures.workspace = true +futures-util.workspace = true +prost.workspace = true +prost-types.workspace = true +serde.workspace = true +serde_json.workspace = true +sqlx = { workspace = true, features = ["runtime-tokio", "sqlite", "postgres", "any"] } +starknet.workspace = true +tokio.workspace = true +tokio-stream.workspace = true +tonic.workspace = true +tracing.workspace = true +hex.workspace = true + +[build-dependencies] +tonic-build.workspace = true + +[lints] +workspace = true diff --git a/crates/torii-ecs-sink/README.md b/crates/torii-ecs-sink/README.md new file mode 100644 index 00000000..21405d63 --- /dev/null +++ b/crates/torii-ecs-sink/README.md @@ -0,0 +1,88 @@ +# Torii ECS Sink + +`torii-ecs-sink` exposes the legacy `world.World` gRPC API on top of the data already indexed by `torii-introspect`. + +## What It Serves + +The sink mounts these legacy RPCs into `torii-introspect-bin` on the same gRPC port: + +- `Worlds` +- `RetrieveEntities` +- `RetrieveEventMessages` +- `RetrieveEvents` +- `SubscribeContracts` +- `SubscribeEntities` +- `SubscribeEventMessages` +- `SubscribeEvents` +- `UpdateEntitiesSubscription` +- `UpdateEventMessagesSubscription` + +The read path is backed by: + +- `torii_dojo_manager_state` for Dojo table metadata +- `torii_ecs_entity_meta` and `torii_ecs_entity_models` for entity and event-message snapshots +- `torii_ecs_events` for raw events +- `torii_ecs_table_kinds` for entity vs event-message classification + +## Manual Validation + +With `torii-introspect-bin` running on the default port: + +```bash +grpcurl -plaintext localhost:3000 list +grpcurl -plaintext localhost:3000 describe world.World +grpcurl -plaintext -d '{}' localhost:3000 world.World/Worlds +grpcurl -max-time 10 -plaintext -d '{"query":{"pagination":{"limit":1,"direction":"FORWARD"}}}' localhost:3000 world.World/RetrieveEvents +``` + +Important: + +- `RetrieveEntities` only returns models classified as `entity` +- `RetrieveEventMessages` only returns models classified as `event_message` +- a model such as `ARCADE-Sale` can be valid for `RetrieveEventMessages` and intentionally return nothing from `RetrieveEntities` + +## End-to-End Test Script + +The repository ships a runnable script at `./scripts/test-ecs-grpc-e2e.sh`. + +From the repository root: + +```bash +./scripts/test-ecs-grpc-e2e.sh +``` + +By default the script: + +- targets `localhost:3000` +- reads sqlite state from `./torii-data/introspect.db` +- autodiscovers one world that has both `entity` and `event_message` models +- validates reflection, `Worlds`, `RetrieveEntities`, `RetrieveEventMessages`, and `RetrieveEvents` + +For Postgres-backed deployments, set `DB_URL` or `DATABASE_URL` to the same storage database used by `torii-introspect-bin`. The script will use `psql` for metadata discovery instead of `sqlite3`. + +### Environment Overrides + +You can override discovery when needed: + +```bash +GRPC_ADDR=localhost:3000 \ +DB_PATH=./torii-data/introspect.db \ +WORLD_ADDRESS=0x7a079295990e43441a7389fdc3b9ba063c6cd6aee16fb846f598c42a9f04ff7 \ +ENTITY_MODEL=ARCADE-Order \ +EVENT_MESSAGE_MODEL=ARCADE-Sale \ +./scripts/test-ecs-grpc-e2e.sh +``` + +If `WORLD_ADDRESS`, `ENTITY_MODEL`, and `EVENT_MESSAGE_MODEL` are provided, the script does not need sqlite introspection metadata to choose test cases. + +Postgres example: + +```bash +GRPC_ADDR=localhost:3000 \ +DB_URL=postgres://torii:torii@localhost:5432/torii \ +./scripts/test-ecs-grpc-e2e.sh +``` + +## Kulala + +For editor-driven checks, the crate also includes `./crates/torii-ecs-sink/ecs-grpc.http`. diff --git a/crates/torii-ecs-sink/build.rs b/crates/torii-ecs-sink/build.rs new file mode 100644 index 00000000..47298bad --- /dev/null +++ b/crates/torii-ecs-sink/build.rs @@ -0,0 +1,15 @@ +use std::path::PathBuf; + +fn main() -> Result<(), Box> { + let out_dir = + PathBuf::from(std::env::var("OUT_DIR").expect("OUT_DIR environment variable not set")); + + tonic_build::configure() + .build_server(true) + .build_client(false) + .file_descriptor_set_path(out_dir.join("world_descriptor.bin")) + .compile_protos(&["proto/world.proto"], &["proto"])?; + + println!("cargo:rerun-if-changed=proto"); + Ok(()) +} diff --git a/crates/torii-ecs-sink/ecs-grpc.http b/crates/torii-ecs-sink/ecs-grpc.http new file mode 100644 index 00000000..ed96bfb8 --- /dev/null +++ b/crates/torii-ecs-sink/ecs-grpc.http @@ -0,0 +1,159 @@ +@address=localhost:3000 + +@world_1_hex=0x2d26295d6c541d64740e1ae56abc079b82b22c35ab83985ef8bd15dc0f9edfb +@world_1_b64=LSYpXWxUHWR0DhrlarwHm4KyLDWrg5he+L0V3A+e3w== + +@world_2_hex=0x02ef591697f0fd9adc0ba9dbe0ca04dabad80cf95f08ba02e435d9cb6698a28a +@world_2_b64=Au9ZFpfw/ZrcC6nb4MoE2rrYDPlfCLoC5DXZy2aYooo= + +@world_3_hex=0x8b4838140a3cbd36ebe64d4b5aaf56a30cc3753c928a79338bf56c53f506c5 +@world_3_b64=i0g4FAo8vTbr5k1LWq9WowzDdTySinkzi/VsU/UGxQ== + +@world_4_hex=0x07a079295990e43441a7389fdc3b9ba063c6cd6aee16fb846f598c42a9f04ff7 +@world_4_b64=B6B5KVmQ5DRBpzif3DuboGPGzWruFvuEb1mMQqnwT/c= + +@subscription_id=1 + +### Shared +# @grpc-plaintext +# @grpc-import-path proto +# @grpc-proto world.proto + +### List gRPC Services +GRPC {{address}} list + +### Describe ECS Service +GRPC {{address}} describe world.World + +### Describe RetrieveEntities +GRPC {{address}} describe world.World/RetrieveEntities + +### Worlds For First Indexed World +GRPC {{address}} world.World/Worlds +Content-Type: application/json + +{ + "world_addresses": ["{{world_1_b64}}"] +} + +### Worlds For All Indexed Worlds +GRPC {{address}} world.World/Worlds +Content-Type: application/json + +{ + "world_addresses": [ + "{{world_1_b64}}", + "{{world_2_b64}}", + "{{world_3_b64}}", + "{{world_4_b64}}" + ] +} + +### Subscribe Contracts For First World +GRPC {{address}} world.World/SubscribeContracts +Content-Type: application/json + +{ + "query": { + "contract_addresses": ["{{world_1_b64}}"], + "contract_types": ["WORLD"] + } +} + +### Retrieve Entities For First World +GRPC {{address}} world.World/RetrieveEntities +Content-Type: application/json + +{ + "query": { + "world_addresses": ["{{world_1_b64}}"], + "pagination": { + "limit": 5, + "direction": "FORWARD" + } + } +} + +### Retrieve Entities For First World And Specific Model +GRPC {{address}} world.World/RetrieveEntities +Content-Type: application/json + +{ + "query": { + "world_addresses": ["{{world_1_b64}}"], + "models": ["blob_arena-CombatantState"], + "pagination": { + "limit": 5, + "direction": "FORWARD" + } + } +} + +### Subscribe Entities For First World +GRPC {{address}} world.World/SubscribeEntities +Content-Type: application/json + +{ + "world_addresses": ["{{world_1_b64}}"] +} + +### Update Entity Subscription +GRPC {{address}} world.World/UpdateEntitiesSubscription +Content-Type: application/json + +{ + "subscription_id": {{subscription_id}}, + "world_addresses": ["{{world_1_b64}}"] +} + +### Retrieve Event Messages For First World +GRPC {{address}} world.World/RetrieveEventMessages +Content-Type: application/json + +{ + "query": { + "world_addresses": ["{{world_1_b64}}"], + "pagination": { + "limit": 5, + "direction": "FORWARD" + } + } +} + +### Subscribe Event Messages For First World +GRPC {{address}} world.World/SubscribeEventMessages +Content-Type: application/json + +{ + "world_addresses": ["{{world_1_b64}}"] +} + +### Update Event Message Subscription +GRPC {{address}} world.World/UpdateEventMessagesSubscription +Content-Type: application/json + +{ + "subscription_id": {{subscription_id}}, + "world_addresses": ["{{world_1_b64}}"] +} + +### Retrieve Events +GRPC {{address}} world.World/RetrieveEvents +Content-Type: application/json + +{ + "query": { + "pagination": { + "limit": 5, + "direction": "FORWARD" + } + } +} + +### Subscribe Events +GRPC {{address}} world.World/SubscribeEvents +Content-Type: application/json + +{ + "keys": [] +} diff --git a/crates/torii-ecs-sink/proto/schema.proto b/crates/torii-ecs-sink/proto/schema.proto new file mode 100644 index 00000000..676e9cb0 --- /dev/null +++ b/crates/torii-ecs-sink/proto/schema.proto @@ -0,0 +1,66 @@ +syntax = "proto3"; +package types; + +message EnumOption { + string name = 1; + Ty ty = 2; +} + +message Enum { + string name = 1; + uint32 option = 2; + repeated EnumOption options = 3; +} + +message Primitive { + oneof primitive_type { + int32 i8 = 1; + int32 i16 = 2; + int32 i32 = 3; + int64 i64 = 4; + bytes i128 = 5; + uint32 u8 = 6; + uint32 u16 = 7; + uint32 u32 = 8; + uint64 u64 = 9; + bytes u128 = 10; + bytes u256 = 11; + bool bool = 12; + bytes felt252 = 13; + bytes class_hash = 14; + bytes contract_address = 15; + bytes eth_address = 16; + } +} + +message Struct { + string name = 1; + repeated Member children = 2; +} + +message Array { + repeated Ty children = 1; +} + +message FixedSizeArray { + repeated Ty children = 1; + uint32 size = 2; +} + +message Ty { + oneof ty_type { + Primitive primitive = 2; + Enum enum = 3; + Struct struct = 4; + Array tuple = 5; + Array array = 6; + string bytearray = 7; + FixedSizeArray fixed_size_array = 8; + } +} + +message Member { + string name = 1; + Ty ty = 2; + bool key = 3; +} diff --git a/crates/torii-ecs-sink/proto/types.proto b/crates/torii-ecs-sink/proto/types.proto new file mode 100644 index 00000000..b3e6d98a --- /dev/null +++ b/crates/torii-ecs-sink/proto/types.proto @@ -0,0 +1,169 @@ +syntax = "proto3"; +package types; + +import "schema.proto"; + +message World { + string world_address = 1; + repeated Model models = 2; +} + +message Model { + bytes selector = 1; + string namespace = 2; + string name = 3; + uint32 packed_size = 4; + uint32 unpacked_size = 5; + bytes class_hash = 6; + bytes layout = 7; + bytes schema = 8; + bytes contract_address = 9; + bool use_legacy_store = 10; + bytes world_address = 11; +} + +message Entity { + bytes hashed_keys = 1; + repeated Struct models = 2; + uint64 created_at = 3; + uint64 updated_at = 4; + uint64 executed_at = 5; + bytes world_address = 6; +} + +message Event { + repeated bytes keys = 1; + repeated bytes data = 2; + bytes transaction_hash = 3; +} + +message Query { + Clause clause = 1; + bool no_hashed_keys = 2; + repeated string models = 3; + Pagination pagination = 4; + bool historical = 5; + repeated bytes world_addresses = 6; +} + +message EventQuery { + KeysClause keys = 1; + Pagination pagination = 2; +} + +message Clause { + oneof clause_type { + HashedKeysClause hashed_keys = 1; + KeysClause keys = 2; + MemberClause member = 3; + CompositeClause composite = 4; + } +} + +message KeysClause { + repeated bytes keys = 2; + PatternMatching pattern_matching = 3; + repeated string models = 4; +} + +message HashedKeysClause { + repeated bytes hashed_keys = 1; +} + +message MemberValue { + oneof value_type { + Primitive primitive = 1; + string string = 2; + MemberValueList list = 3; + } +} + +message MemberValueList { + repeated MemberValue values = 1; +} + +message MemberClause { + string model = 2; + string member = 3; + ComparisonOperator operator = 4; + MemberValue value = 5; +} + +message CompositeClause { + LogicalOperator operator = 3; + repeated Clause clauses = 4; +} + +enum PatternMatching { + FixedLen = 0; + VariableLen = 1; +} + +enum LogicalOperator { + AND = 0; + OR = 1; +} + +enum ComparisonOperator { + EQ = 0; + NEQ = 1; + GT = 2; + GTE = 3; + LT = 4; + LTE = 5; + IN = 6; + NOT_IN = 7; + CONTAINS = 8; + CONTAINS_ALL = 9; + CONTAINS_ANY = 10; + ARRAY_LENGTH_EQ = 11; + ARRAY_LENGTH_GT = 12; + ARRAY_LENGTH_LT = 13; +} + +message OrderBy { + string field = 1; + OrderDirection direction = 2; +} + +enum OrderDirection { + ASC = 0; + DESC = 1; +} + +enum PaginationDirection { + FORWARD = 0; + BACKWARD = 1; +} + +message Pagination { + string cursor = 1; + uint32 limit = 2; + PaginationDirection direction = 3; + repeated OrderBy order_by = 4; +} + +enum ContractType { + WORLD = 0; + ERC20 = 1; + ERC721 = 2; + ERC1155 = 3; + UDC = 4; + OTHER = 5; +} + +message Contract { + bytes contract_address = 1; + ContractType contract_type = 2; + optional uint64 head = 3; + optional uint64 tps = 4; + optional uint64 last_block_timestamp = 5; + optional bytes last_pending_block_tx = 6; + uint64 updated_at = 7; + uint64 created_at = 8; +} + +message ContractQuery { + repeated bytes contract_addresses = 1; + repeated ContractType contract_types = 2; +} diff --git a/crates/torii-ecs-sink/proto/world.proto b/crates/torii-ecs-sink/proto/world.proto new file mode 100644 index 00000000..803c59ae --- /dev/null +++ b/crates/torii-ecs-sink/proto/world.proto @@ -0,0 +1,76 @@ +syntax = "proto3"; +package world; + +import "types.proto"; +import "google/protobuf/empty.proto"; + +service World { + rpc SubscribeContracts (SubscribeContractsRequest) returns (stream SubscribeContractsResponse); + rpc Worlds (WorldsRequest) returns (WorldsResponse); + rpc SubscribeEntities (SubscribeEntitiesRequest) returns (stream SubscribeEntityResponse); + rpc UpdateEntitiesSubscription (UpdateEntitiesSubscriptionRequest) returns (google.protobuf.Empty); + rpc RetrieveEntities (RetrieveEntitiesRequest) returns (RetrieveEntitiesResponse); + rpc SubscribeEventMessages (SubscribeEntitiesRequest) returns (stream SubscribeEntityResponse); + rpc UpdateEventMessagesSubscription (UpdateEntitiesSubscriptionRequest) returns (google.protobuf.Empty); + rpc RetrieveEventMessages (RetrieveEntitiesRequest) returns (RetrieveEntitiesResponse); + rpc RetrieveEvents (RetrieveEventsRequest) returns (RetrieveEventsResponse); + rpc SubscribeEvents (SubscribeEventsRequest) returns (stream SubscribeEventsResponse); +} + +message SubscribeContractsRequest { + types.ContractQuery query = 1; +} + +message SubscribeContractsResponse { + types.Contract contract = 1; +} + +message WorldsRequest { + repeated bytes world_addresses = 1; +} + +message WorldsResponse { + repeated types.World worlds = 1; +} + +message SubscribeEntitiesRequest { + types.Clause clause = 1; + repeated bytes world_addresses = 2; +} + +message UpdateEntitiesSubscriptionRequest { + uint64 subscription_id = 1; + types.Clause clause = 2; + repeated bytes world_addresses = 3; +} + +message SubscribeEntityResponse { + types.Entity entity = 1; + uint64 subscription_id = 2; +} + +message RetrieveEntitiesRequest { + types.Query query = 1; +} + +message RetrieveEntitiesResponse { + string next_cursor = 1; + repeated types.Entity entities = 2; +} + +message RetrieveEventsRequest { + types.EventQuery query = 1; +} + +message RetrieveEventsResponse { + string next_cursor = 1; + repeated types.Event events = 2; +} + +message SubscribeEventsRequest { + repeated types.KeysClause keys = 1; +} + +message SubscribeEventsResponse { + types.Event event = 1; +} diff --git a/crates/torii-ecs-sink/src/grpc_service.rs b/crates/torii-ecs-sink/src/grpc_service.rs new file mode 100644 index 00000000..f31165f7 --- /dev/null +++ b/crates/torii-ecs-sink/src/grpc_service.rs @@ -0,0 +1,2542 @@ +use std::collections::{HashMap, HashSet}; +use std::str::FromStr; +use std::sync::Arc; + +use anyhow::{anyhow, Result}; +use chrono::Utc; +use introspect_types::serialize::ToCairoDeSeFrom; +use introspect_types::serialize_def::CairoTypeSerialization; +use introspect_types::{ + Attributes, CairoDeserializer, ColumnDef, PrimaryTypeDef, ResultDef, TupleDef, TypeDef, +}; +use serde::ser::SerializeMap; +use serde::Serializer; +use serde_json::{Map, Serializer as JsonSerializer, Value}; +use sqlx::{ + any::AnyPoolOptions, sqlite::SqliteConnectOptions, Any, ConnectOptions, Pool, QueryBuilder, Row, +}; +use starknet::core::types::Felt; +use tokio::sync::{mpsc, Mutex}; +use tokio_stream::wrappers::ReceiverStream; +use tonic::{Request, Response, Status}; +use torii_dojo::manager::{PostgresStore, SqliteStore}; +use torii_dojo::store::DojoStoreTrait; +use torii_dojo::DojoTable; +use torii_introspect::events::{CreateTable, Record, UpdateTable}; +use torii_introspect::schema::TableSchema; + +use crate::proto::types::{ + self, clause::ClauseType, member_value::ValueType, ComparisonOperator, ContractType, + LogicalOperator, PaginationDirection, PatternMatching, +}; +use crate::proto::world::{ + world_server::World, RetrieveEntitiesRequest, RetrieveEntitiesResponse, RetrieveEventsRequest, + RetrieveEventsResponse, SubscribeContractsRequest, SubscribeContractsResponse, + SubscribeEntitiesRequest, SubscribeEntityResponse, SubscribeEventsRequest, + SubscribeEventsResponse, UpdateEntitiesSubscriptionRequest, WorldsRequest, WorldsResponse, +}; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum TableKind { + Entity, + EventMessage, +} + +impl TableKind { + fn as_str(self) -> &'static str { + match self { + Self::Entity => "entity", + Self::EventMessage => "event_message", + } + } + + fn from_str(value: &str) -> Option { + match value { + "entity" => Some(Self::Entity), + "event_message" => Some(Self::EventMessage), + _ => None, + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum DbBackend { + Sqlite, + Postgres, +} + +impl DbBackend { + fn detect(database_url: &str) -> Self { + if database_url.starts_with("postgres://") || database_url.starts_with("postgresql://") { + Self::Postgres + } else { + Self::Sqlite + } + } +} + +#[derive(Clone)] +pub struct EcsService { + state: Arc, +} + +struct EcsState { + pool: Pool, + backend: DbBackend, + database_url: String, + managed_tables: Mutex>>, + entity_subscriptions: Mutex>, + event_message_subscriptions: Mutex>, + event_subscriptions: Mutex>, + contract_subscriptions: Mutex>, +} + +struct EntitySubscription { + clause: Option, + world_addresses: Vec>, + sender: mpsc::Sender>, +} + +struct EventSubscription { + keys: Vec, + sender: mpsc::Sender>, +} + +struct ContractSubscription { + query: types::ContractQuery, + sender: mpsc::Sender>, +} + +#[derive(Clone)] +struct ManagedTable { + world_address: Felt, + table: DojoTable, + kind: TableKind, +} + +#[derive(Default)] +struct EntityAggregate { + world_address: Vec, + hashed_keys: Vec, + models: Vec, + created_at: u64, + updated_at: u64, + executed_at: u64, +} + +impl EntityAggregate { + fn into_proto(self) -> types::Entity { + types::Entity { + hashed_keys: self.hashed_keys, + models: self.models, + created_at: self.created_at, + updated_at: self.updated_at, + executed_at: self.executed_at, + world_address: self.world_address, + } + } +} + +impl EcsService { + pub async fn new(database_url: &str, max_connections: Option) -> Result { + sqlx::any::install_default_drivers(); + + let backend = DbBackend::detect(database_url); + let database_url = match backend { + DbBackend::Postgres => database_url.to_string(), + DbBackend::Sqlite => sqlite_url(database_url)?, + }; + + let pool = AnyPoolOptions::new() + .max_connections(max_connections.unwrap_or(if backend == DbBackend::Sqlite { + 1 + } else { + 5 + })) + .connect(&database_url) + .await?; + + let service = Self { + state: Arc::new(EcsState { + pool, + backend, + database_url, + managed_tables: Mutex::new(None), + entity_subscriptions: Mutex::new(HashMap::new()), + event_message_subscriptions: Mutex::new(HashMap::new()), + event_subscriptions: Mutex::new(HashMap::new()), + contract_subscriptions: Mutex::new(HashMap::new()), + }), + }; + service.initialize().await?; + Ok(service) + } + + async fn initialize(&self) -> Result<()> { + if self.state.backend == DbBackend::Sqlite { + sqlx::query("PRAGMA journal_mode=WAL") + .execute(&self.state.pool) + .await + .ok(); + } + + sqlx::query( + "CREATE TABLE IF NOT EXISTS torii_ecs_contracts ( + contract_address TEXT PRIMARY KEY, + contract_type INTEGER NOT NULL, + head BIGINT, + tps BIGINT, + last_block_timestamp BIGINT, + last_pending_block_tx TEXT, + updated_at BIGINT NOT NULL, + created_at BIGINT NOT NULL + )", + ) + .execute(&self.state.pool) + .await?; + + sqlx::query( + "CREATE TABLE IF NOT EXISTS torii_ecs_table_kinds ( + table_id TEXT PRIMARY KEY, + world_address TEXT NOT NULL, + kind TEXT NOT NULL, + updated_at BIGINT NOT NULL + )", + ) + .execute(&self.state.pool) + .await?; + + let entity_meta_sql = match self.state.backend { + DbBackend::Sqlite => { + "CREATE TABLE IF NOT EXISTS torii_ecs_entity_meta ( + kind TEXT NOT NULL, + world_address TEXT NOT NULL, + table_id TEXT NOT NULL, + entity_id TEXT NOT NULL, + created_at BIGINT NOT NULL, + updated_at BIGINT NOT NULL, + executed_at BIGINT NOT NULL, + deleted INTEGER NOT NULL DEFAULT 0, + PRIMARY KEY(kind, world_address, table_id, entity_id) + )" + } + DbBackend::Postgres => { + "CREATE TABLE IF NOT EXISTS torii_ecs_entity_meta ( + kind TEXT NOT NULL, + world_address TEXT NOT NULL, + table_id TEXT NOT NULL, + entity_id TEXT NOT NULL, + created_at BIGINT NOT NULL, + updated_at BIGINT NOT NULL, + executed_at BIGINT NOT NULL, + deleted BOOLEAN NOT NULL DEFAULT FALSE, + PRIMARY KEY(kind, world_address, table_id, entity_id) + )" + } + }; + sqlx::query(entity_meta_sql) + .execute(&self.state.pool) + .await?; + + sqlx::query( + "CREATE TABLE IF NOT EXISTS torii_ecs_events ( + event_id TEXT PRIMARY KEY, + world_address TEXT NOT NULL, + block_number BIGINT NOT NULL, + executed_at BIGINT NOT NULL, + transaction_hash TEXT NOT NULL, + keys_json TEXT NOT NULL, + data_json TEXT NOT NULL + )", + ) + .execute(&self.state.pool) + .await?; + + sqlx::query( + "CREATE TABLE IF NOT EXISTS torii_ecs_entity_models ( + kind TEXT NOT NULL, + world_address TEXT NOT NULL, + table_id TEXT NOT NULL, + entity_id TEXT NOT NULL, + row_json TEXT NOT NULL, + updated_at BIGINT NOT NULL, + PRIMARY KEY(kind, world_address, table_id, entity_id) + )", + ) + .execute(&self.state.pool) + .await?; + + sqlx::query( + "CREATE INDEX IF NOT EXISTS torii_ecs_entity_meta_lookup_idx + ON torii_ecs_entity_meta(kind, world_address, entity_id)", + ) + .execute(&self.state.pool) + .await?; + + sqlx::query( + "CREATE INDEX IF NOT EXISTS torii_ecs_entity_models_lookup_idx + ON torii_ecs_entity_models(kind, world_address, entity_id)", + ) + .execute(&self.state.pool) + .await?; + + Ok(()) + } + + pub async fn record_contract_progress( + &self, + contract_address: Felt, + contract_type: ContractType, + head: u64, + executed_at: u64, + last_pending_block_tx: Option, + ) -> Result<()> { + let now = Utc::now().timestamp() as u64; + let sql = match self.state.backend { + DbBackend::Sqlite => { + "INSERT INTO torii_ecs_contracts ( + contract_address, contract_type, head, tps, last_block_timestamp, + last_pending_block_tx, updated_at, created_at + ) VALUES (?1, ?2, ?3, NULL, ?4, ?5, ?6, ?6) + ON CONFLICT(contract_address) DO UPDATE SET + contract_type = excluded.contract_type, + head = excluded.head, + last_block_timestamp = excluded.last_block_timestamp, + last_pending_block_tx = excluded.last_pending_block_tx, + updated_at = excluded.updated_at" + } + DbBackend::Postgres => { + "INSERT INTO torii_ecs_contracts ( + contract_address, contract_type, head, tps, last_block_timestamp, + last_pending_block_tx, updated_at, created_at + ) VALUES ($1, $2, $3, NULL, $4, $5, $6, $6) + ON CONFLICT(contract_address) DO UPDATE SET + contract_type = EXCLUDED.contract_type, + head = EXCLUDED.head, + last_block_timestamp = EXCLUDED.last_block_timestamp, + last_pending_block_tx = EXCLUDED.last_pending_block_tx, + updated_at = EXCLUDED.updated_at" + } + }; + sqlx::query(sql) + .bind(felt_hex(contract_address)) + .bind(contract_type as i32) + .bind(head as i64) + .bind(executed_at as i64) + .bind(last_pending_block_tx.map(felt_hex)) + .bind(now as i64) + .execute(&self.state.pool) + .await?; + + if self.state.contract_subscriptions.lock().await.is_empty() { + return Ok(()); + } + + let contract = self + .load_contracts(&types::ContractQuery { + contract_addresses: vec![contract_address.to_bytes_be().to_vec()], + contract_types: vec![], + }) + .await? + .into_iter() + .next(); + + if let Some(contract) = contract { + self.publish_contract_update(contract).await; + } + Ok(()) + } + + pub async fn record_table_kind( + &self, + world_address: Felt, + table_id: Felt, + kind: TableKind, + ) -> Result<()> { + let now = Utc::now().timestamp(); + let sql = match self.state.backend { + DbBackend::Sqlite => { + "INSERT INTO torii_ecs_table_kinds (table_id, world_address, kind, updated_at) + VALUES (?1, ?2, ?3, ?4) + ON CONFLICT(table_id) DO UPDATE SET + world_address = excluded.world_address, + kind = excluded.kind, + updated_at = excluded.updated_at" + } + DbBackend::Postgres => { + "INSERT INTO torii_ecs_table_kinds (table_id, world_address, kind, updated_at) + VALUES ($1, $2, $3, $4) + ON CONFLICT(table_id) DO UPDATE SET + world_address = EXCLUDED.world_address, + kind = EXCLUDED.kind, + updated_at = EXCLUDED.updated_at" + } + }; + sqlx::query(sql) + .bind(felt_hex(table_id)) + .bind(felt_hex(world_address)) + .bind(kind.as_str()) + .bind(now) + .execute(&self.state.pool) + .await?; + self.update_managed_table_kind(table_id, world_address, kind) + .await; + Ok(()) + } + + pub async fn cache_created_table(&self, world_address: Felt, table: &CreateTable) { + let schema = TableSchema { + id: table.id, + name: table.name.clone(), + attributes: table.attributes.clone(), + primary: table.primary.clone(), + columns: table.columns.clone(), + }; + self.put_managed_table(world_address, DojoTable::from(schema)) + .await; + } + + pub async fn cache_updated_table(&self, world_address: Felt, table: &UpdateTable) { + let schema = TableSchema { + id: table.id, + name: table.name.clone(), + attributes: table.attributes.clone(), + primary: table.primary.clone(), + columns: table.columns.clone(), + }; + self.put_managed_table(world_address, DojoTable::from(schema)) + .await; + } + + pub async fn upsert_entity_meta( + &self, + kind: TableKind, + world_address: Felt, + table_id: Felt, + entity_id: Felt, + executed_at: u64, + deleted: bool, + ) -> Result<()> { + let sql = match self.state.backend { + DbBackend::Sqlite => { + "INSERT INTO torii_ecs_entity_meta ( + kind, world_address, table_id, entity_id, created_at, updated_at, executed_at, deleted + ) VALUES (?1, ?2, ?3, ?4, ?5, ?5, ?5, ?6) + ON CONFLICT(kind, world_address, table_id, entity_id) DO UPDATE SET + updated_at = excluded.updated_at, + executed_at = excluded.executed_at, + deleted = excluded.deleted" + } + DbBackend::Postgres => { + "INSERT INTO torii_ecs_entity_meta ( + kind, world_address, table_id, entity_id, created_at, updated_at, executed_at, deleted + ) VALUES ($1, $2, $3, $4, $5, $5, $5, $6) + ON CONFLICT(kind, world_address, table_id, entity_id) DO UPDATE SET + updated_at = EXCLUDED.updated_at, + executed_at = EXCLUDED.executed_at, + deleted = EXCLUDED.deleted" + } + }; + let mut query = sqlx::query(sql) + .bind(kind.as_str()) + .bind(felt_hex(world_address)) + .bind(felt_hex(table_id)) + .bind(felt_hex(entity_id)) + .bind(executed_at as i64); + query = match self.state.backend { + DbBackend::Sqlite => query.bind(i64::from(deleted)), + DbBackend::Postgres => query.bind(deleted), + }; + query.execute(&self.state.pool).await?; + Ok(()) + } + + pub async fn upsert_entity_model( + &self, + kind: TableKind, + world_address: Felt, + table_id: Felt, + columns: &[Felt], + record: &Record, + executed_at: u64, + ) -> Result<()> { + let Some(table) = self.load_managed_table(table_id).await? else { + return Ok(()); + }; + let row = record_to_json_map(&table.table, columns, record)?; + let row_json = serde_json::to_string(&row)?; + let sql = match self.state.backend { + DbBackend::Sqlite => { + "INSERT INTO torii_ecs_entity_models ( + kind, world_address, table_id, entity_id, row_json, updated_at + ) VALUES (?1, ?2, ?3, ?4, ?5, ?6) + ON CONFLICT(kind, world_address, table_id, entity_id) DO UPDATE SET + row_json = excluded.row_json, + updated_at = excluded.updated_at" + } + DbBackend::Postgres => { + "INSERT INTO torii_ecs_entity_models ( + kind, world_address, table_id, entity_id, row_json, updated_at + ) VALUES ($1, $2, $3, $4, $5, $6) + ON CONFLICT(kind, world_address, table_id, entity_id) DO UPDATE SET + row_json = EXCLUDED.row_json, + updated_at = EXCLUDED.updated_at" + } + }; + sqlx::query(sql) + .bind(kind.as_str()) + .bind(felt_hex(world_address)) + .bind(felt_hex(table_id)) + .bind(felt_hex(Felt::from_bytes_be(&record.id))) + .bind(row_json) + .bind(executed_at as i64) + .execute(&self.state.pool) + .await?; + Ok(()) + } + + pub async fn delete_entity_model( + &self, + kind: TableKind, + world_address: Felt, + table_id: Felt, + entity_id: Felt, + ) -> Result<()> { + let sql = match self.state.backend { + DbBackend::Sqlite => { + "DELETE FROM torii_ecs_entity_models + WHERE kind = ?1 AND world_address = ?2 AND table_id = ?3 AND entity_id = ?4" + } + DbBackend::Postgres => { + "DELETE FROM torii_ecs_entity_models + WHERE kind = $1 AND world_address = $2 AND table_id = $3 AND entity_id = $4" + } + }; + sqlx::query(sql) + .bind(kind.as_str()) + .bind(felt_hex(world_address)) + .bind(felt_hex(table_id)) + .bind(felt_hex(entity_id)) + .execute(&self.state.pool) + .await?; + Ok(()) + } + + pub async fn table_kind(&self, table_id: Felt) -> Result { + let sql = match self.state.backend { + DbBackend::Sqlite => "SELECT kind FROM torii_ecs_table_kinds WHERE table_id = ?1", + DbBackend::Postgres => "SELECT kind FROM torii_ecs_table_kinds WHERE table_id = $1", + }; + let row = sqlx::query(sql) + .bind(felt_hex(table_id)) + .fetch_optional(&self.state.pool) + .await?; + Ok(row + .and_then(|row| row.try_get::("kind").ok()) + .and_then(|kind| TableKind::from_str(&kind)) + .unwrap_or(TableKind::Entity)) + } + + pub async fn store_event( + &self, + world_address: Felt, + transaction_hash: Felt, + block_number: u64, + executed_at: u64, + keys: &[Felt], + data: &[Felt], + ordinal: usize, + ) -> Result<()> { + let event_id = format!( + "{}:{}:{}", + felt_hex(world_address), + felt_hex(transaction_hash), + ordinal + ); + let keys_json = + serde_json::to_string(&keys.iter().map(|felt| felt_hex(*felt)).collect::>())?; + let data_json = + serde_json::to_string(&data.iter().map(|felt| felt_hex(*felt)).collect::>())?; + + let sql = match self.state.backend { + DbBackend::Sqlite => { + "INSERT INTO torii_ecs_events ( + event_id, world_address, block_number, executed_at, transaction_hash, keys_json, data_json + ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7) + ON CONFLICT(event_id) DO NOTHING" + } + DbBackend::Postgres => { + "INSERT INTO torii_ecs_events ( + event_id, world_address, block_number, executed_at, transaction_hash, keys_json, data_json + ) VALUES ($1, $2, $3, $4, $5, $6, $7) + ON CONFLICT(event_id) DO NOTHING" + } + }; + sqlx::query(sql) + .bind(event_id) + .bind(felt_hex(world_address)) + .bind(block_number as i64) + .bind(executed_at as i64) + .bind(felt_hex(transaction_hash)) + .bind(keys_json) + .bind(data_json) + .execute(&self.state.pool) + .await?; + + self.publish_event_update(types::Event { + keys: keys + .iter() + .map(|felt| felt.to_bytes_be().to_vec()) + .collect(), + data: data + .iter() + .map(|felt| felt.to_bytes_be().to_vec()) + .collect(), + transaction_hash: transaction_hash.to_bytes_be().to_vec(), + }) + .await; + + Ok(()) + } + + pub async fn publish_entity_update( + &self, + kind: TableKind, + world_address: Felt, + entity_id: Felt, + ) -> Result<()> { + let subscriptions = match kind { + TableKind::Entity => &self.state.entity_subscriptions, + TableKind::EventMessage => &self.state.event_message_subscriptions, + }; + if subscriptions.lock().await.is_empty() { + return Ok(()); + } + + let entity = self + .load_entity_by_id(kind, world_address, entity_id) + .await? + .map(EntityAggregate::into_proto); + let Some(entity) = entity else { + return Ok(()); + }; + let subscriptions = subscriptions.lock().await; + for (&subscription_id, subscription) in subscriptions.iter() { + if !subscription.world_addresses.is_empty() + && !subscription.world_addresses.contains(&entity.world_address) + { + continue; + } + if let Some(clause) = &subscription.clause { + if !entity_matches_clause(&entity, clause) { + continue; + } + } + let _ = subscription.sender.try_send(Ok(SubscribeEntityResponse { + entity: Some(entity.clone()), + subscription_id, + })); + } + Ok(()) + } + + async fn publish_event_update(&self, event: types::Event) { + let subscriptions = self.state.event_subscriptions.lock().await; + if subscriptions.is_empty() { + return; + } + for subscription in subscriptions.values() { + if !match_keys(&event.keys, &subscription.keys) { + continue; + } + let _ = subscription.sender.try_send(Ok(SubscribeEventsResponse { + event: Some(event.clone()), + })); + } + } + + async fn publish_contract_update(&self, contract: types::Contract) { + let subscriptions = self.state.contract_subscriptions.lock().await; + if subscriptions.is_empty() { + return; + } + for subscription in subscriptions.values() { + if !contract_matches_query(&contract, &subscription.query) { + continue; + } + let _ = subscription.sender.try_send(Ok(SubscribeContractsResponse { + contract: Some(contract.clone()), + })); + } + } + + async fn load_contracts(&self, query: &types::ContractQuery) -> Result> { + let sql = "SELECT contract_address, contract_type, head, tps, last_block_timestamp, \ + last_pending_block_tx, updated_at, created_at \ + FROM torii_ecs_contracts"; + let rows = sqlx::query(sql).fetch_all(&self.state.pool).await?; + let mut contracts = Vec::new(); + for row in rows { + let contract_address = row.try_get::("contract_address")?; + let contract_type = row.try_get::("contract_type")?; + let contract = types::Contract { + contract_address: felt_from_hex(&contract_address)?.to_bytes_be().to_vec(), + contract_type, + head: row + .try_get::, _>("head")? + .map(|value| value as u64), + tps: row + .try_get::, _>("tps")? + .map(|value| value as u64), + last_block_timestamp: row + .try_get::, _>("last_block_timestamp")? + .map(|value| value as u64), + last_pending_block_tx: row + .try_get::, _>("last_pending_block_tx")? + .map(|value| felt_from_hex(&value)) + .transpose()? + .map(|felt| felt.to_bytes_be().to_vec()), + updated_at: row.try_get::("updated_at")? as u64, + created_at: row.try_get::("created_at")? as u64, + }; + if contract_matches_query(&contract, query) { + contracts.push(contract); + } + } + Ok(contracts) + } + + async fn load_worlds(&self, requested: &[Felt]) -> Result> { + let mut by_world: HashMap, Vec> = HashMap::new(); + for table in self.load_managed_tables(None).await? { + if !requested.is_empty() && !requested.contains(&table.world_address) { + continue; + } + by_world + .entry(table.world_address.to_bytes_be().to_vec()) + .or_default() + .push(model_from_table(&table)); + } + + let mut worlds = by_world + .into_iter() + .map(|(world_address, models)| types::World { + world_address: felt_from_bytes(&world_address) + .unwrap_or_default() + .to_string(), + models, + }) + .collect::>(); + worlds.sort_by(|a, b| a.world_address.cmp(&b.world_address)); + Ok(worlds) + } + + async fn put_managed_table(&self, world_address: Felt, table: DojoTable) { + if self.state.managed_tables.lock().await.is_none() { + let _ = self.load_managed_table_map().await; + } + let mut managed_tables = self.state.managed_tables.lock().await; + if let Some(tables) = managed_tables.as_mut() { + let key = felt_hex(table.id); + let kind = tables + .get(&key) + .map_or(TableKind::Entity, |table| table.kind); + tables.insert( + key, + ManagedTable { + world_address, + table, + kind, + }, + ); + } + } + + async fn update_managed_table_kind( + &self, + table_id: Felt, + world_address: Felt, + kind: TableKind, + ) { + let mut managed_tables = self.state.managed_tables.lock().await; + let Some(tables) = managed_tables.as_mut() else { + return; + }; + let key = felt_hex(table_id); + if let Some(table) = tables.get_mut(&key) { + table.world_address = world_address; + table.kind = kind; + } else { + *managed_tables = None; + } + } + + async fn load_managed_table(&self, table_id: Felt) -> Result> { + let key = felt_hex(table_id); + Ok(self.load_managed_table_map().await?.get(&key).cloned()) + } + + async fn load_managed_table_map(&self) -> Result> { + { + let managed_tables = self.state.managed_tables.lock().await; + if let Some(tables) = managed_tables.as_ref() { + return Ok(tables.clone()); + } + } + + let kinds = self.load_table_kind_rows().await?; + let tables = self.load_dojo_tables().await?; + let mut managed = HashMap::with_capacity(tables.len()); + for table in tables { + let key = felt_hex(table.id); + let (world_address, table_kind) = if let Some((world, kind_name)) = kinds.get(&key) { + let kind = TableKind::from_str(kind_name).unwrap_or(TableKind::Entity); + (felt_from_hex(world)?, kind) + } else { + (Felt::ZERO, TableKind::Entity) + }; + managed.insert( + key, + ManagedTable { + world_address, + table, + kind: table_kind, + }, + ); + } + + let mut managed_tables = self.state.managed_tables.lock().await; + *managed_tables = Some(managed.clone()); + Ok(managed) + } + + async fn load_table_kind_rows(&self) -> Result> { + let table_kind_sql = "SELECT table_id, world_address, kind FROM torii_ecs_table_kinds"; + let kind_rows = sqlx::query(table_kind_sql) + .fetch_all(&self.state.pool) + .await?; + let mut kinds = HashMap::new(); + for row in kind_rows { + kinds.insert( + row.try_get::("table_id")?, + ( + row.try_get::("world_address")?, + row.try_get::("kind")?, + ), + ); + } + Ok(kinds) + } + + async fn load_dojo_tables(&self) -> Result> { + match self.state.backend { + DbBackend::Sqlite => { + let store = SqliteStore::new(&self.state.database_url).await?; + Ok(store.load_tables(&[]).await?) + } + DbBackend::Postgres => { + let store = PostgresStore::new(&self.state.database_url).await?; + Ok(store.load_tables(&[]).await?) + } + } + } + + async fn load_managed_tables(&self, kind: Option) -> Result> { + Ok(self + .load_managed_table_map() + .await? + .into_values() + .filter(|table| kind.is_none_or(|required| table.kind == required)) + .collect()) + } + + async fn load_entity_page( + &self, + kind: TableKind, + query: &types::Query, + ) -> Result<(Vec, String)> { + let world_filters = query + .world_addresses + .iter() + .map(|bytes| felt_from_bytes(bytes)) + .collect::>>()?; + let models = self + .load_managed_tables(Some(kind)) + .await? + .into_iter() + .filter(|table| { + (world_filters.is_empty() || world_filters.contains(&table.world_address)) + && (query.models.is_empty() || query.models.contains(&table.table.name)) + }) + .collect::>(); + if models.is_empty() { + return Ok((Vec::new(), String::new())); + } + + let model_map = models + .into_iter() + .map(|table| (felt_hex(table.table.id), table)) + .collect::>(); + let candidate_meta = self + .load_entity_page_meta(kind, &world_filters, model_map.keys().cloned().collect()) + .await?; + let mut candidate_keys = candidate_meta.keys().cloned().collect::>(); + candidate_keys.sort_by(|a, b| a.1.cmp(&b.1).then_with(|| a.0.cmp(&b.0))); + if query + .pagination + .as_ref() + .is_some_and(|p| p.direction == PaginationDirection::Backward as i32) + { + candidate_keys.reverse(); + } + + if let Some(pagination) = &query.pagination { + if !pagination.cursor.is_empty() { + candidate_keys.retain(|(_, entity_id)| { + let key = entity_id.trim_start_matches("0x"); + if pagination.direction == PaginationDirection::Backward as i32 { + key < pagination.cursor.as_str() + } else { + key > pagination.cursor.as_str() + } + }); + } + } + + let target_limit = query.pagination.as_ref().map_or(usize::MAX, |pagination| { + if pagination.limit == 0 { + 100 + } else { + pagination.limit as usize + } + }); + + let mut items = Vec::new(); + let mut built_any_snapshot_entity = false; + for chunk in candidate_keys.chunks(256) { + let rows = self + .load_entity_page_rows( + kind, + &world_filters, + model_map.keys().cloned().collect(), + chunk, + ) + .await?; + let row_map = rows.into_iter().fold( + HashMap::<(String, String), Vec<(String, String)>>::new(), + |mut acc, row| { + acc.entry((row.world_address, row.entity_id)) + .or_default() + .push((row.table_id, row.row_json)); + acc + }, + ); + + for key in chunk { + let Some(meta) = candidate_meta.get(key) else { + continue; + }; + let Some(entity) = + build_entity_from_snapshot(key, meta, row_map.get(key), &model_map)? + else { + continue; + }; + built_any_snapshot_entity = true; + let entity = entity.into_proto(); + if query + .clause + .as_ref() + .is_some_and(|clause| !entity_matches_clause(&entity, clause)) + { + continue; + } + items.push(entity); + if items.len() >= target_limit { + break; + } + } + + if items.len() >= target_limit { + break; + } + } + + if !built_any_snapshot_entity && !candidate_meta.is_empty() { + return self.load_entity_page_from_storage(kind, query).await; + } + + let next_cursor = if target_limit != usize::MAX && items.len() == target_limit { + items + .last() + .map(|entity| { + felt_from_bytes(&entity.hashed_keys) + .map(|felt| felt_hex(felt).trim_start_matches("0x").to_string()) + }) + .transpose()? + .unwrap_or_default() + } else { + String::new() + }; + + if query.no_hashed_keys { + for entity in &mut items { + entity.hashed_keys.clear(); + } + } + Ok((items, next_cursor)) + } + + async fn load_entity_page_from_storage( + &self, + kind: TableKind, + query: &types::Query, + ) -> Result<(Vec, String)> { + let world_filters = query + .world_addresses + .iter() + .map(|bytes| felt_from_bytes(bytes)) + .collect::>>()?; + let models = self + .load_managed_tables(Some(kind)) + .await? + .into_iter() + .filter(|table| { + (world_filters.is_empty() || world_filters.contains(&table.world_address)) + && (query.models.is_empty() || query.models.contains(&table.table.name)) + }) + .collect::>(); + + let mut entities: HashMap<(Vec, Vec), EntityAggregate> = HashMap::new(); + for table in models { + let rows = self.fetch_table_rows(&table).await?; + let meta = self + .load_entity_meta(table.kind, table.world_address, table.table.id) + .await?; + for row in rows { + let entity_id = row + .get(&table.table.primary.name) + .cloned() + .unwrap_or(Value::Null); + let entity_felt = value_to_primary_felt(&entity_id, &table.table.primary.type_def) + .unwrap_or(Felt::ZERO); + let entity_key = entity_felt.to_bytes_be().to_vec(); + if meta + .get(&felt_hex(entity_felt)) + .is_some_and(|meta| meta.deleted) + { + continue; + } + + let model = row_to_model_struct(&table.table, &row)?; + let aggregate = entities + .entry(( + table.world_address.to_bytes_be().to_vec(), + entity_key.clone(), + )) + .or_insert_with(|| EntityAggregate { + world_address: table.world_address.to_bytes_be().to_vec(), + hashed_keys: entity_key.clone(), + ..EntityAggregate::default() + }); + aggregate.models.push(model); + if let Some(meta) = meta.get(&felt_hex(entity_felt)) { + if aggregate.created_at == 0 || meta.created_at < aggregate.created_at { + aggregate.created_at = meta.created_at; + } + aggregate.updated_at = aggregate.updated_at.max(meta.updated_at); + aggregate.executed_at = aggregate.executed_at.max(meta.executed_at); + } + } + } + + let mut items = entities + .into_values() + .map(EntityAggregate::into_proto) + .filter(|entity| { + query + .clause + .as_ref() + .is_none_or(|clause| entity_matches_clause(entity, clause)) + }) + .collect::>(); + items.sort_by(|a, b| a.hashed_keys.cmp(&b.hashed_keys)); + if query + .pagination + .as_ref() + .is_some_and(|p| p.direction == PaginationDirection::Backward as i32) + { + items.reverse(); + } + + if let Some(pagination) = &query.pagination { + if !pagination.cursor.is_empty() { + items.retain(|entity| { + let key = hex::encode(&entity.hashed_keys); + if pagination.direction == PaginationDirection::Backward as i32 { + key < pagination.cursor + } else { + key > pagination.cursor + } + }); + } + let limit = if pagination.limit == 0 { + 100 + } else { + pagination.limit as usize + }; + let next_cursor = items + .get(limit.saturating_sub(1)) + .map(|entity| hex::encode(&entity.hashed_keys)) + .unwrap_or_default(); + items.truncate(limit); + if query.no_hashed_keys { + for entity in &mut items { + entity.hashed_keys.clear(); + } + } + return Ok((items, next_cursor)); + } + + if query.no_hashed_keys { + for entity in &mut items { + entity.hashed_keys.clear(); + } + } + Ok((items, String::new())) + } + + async fn load_entity_by_id( + &self, + kind: TableKind, + world_address: Felt, + entity_id: Felt, + ) -> Result> { + if let Some(entity) = self + .load_entity_snapshot(kind, world_address, entity_id) + .await? + { + return Ok(Some(entity)); + } + + let query = types::Query { + clause: Some(types::Clause { + clause_type: Some(ClauseType::HashedKeys(types::HashedKeysClause { + hashed_keys: vec![entity_id.to_bytes_be().to_vec()], + })), + }), + no_hashed_keys: false, + models: vec![], + pagination: None, + historical: false, + world_addresses: vec![world_address.to_bytes_be().to_vec()], + }; + Ok(self + .load_entity_page(kind, &query) + .await? + .0 + .into_iter() + .next() + .map(|entity| EntityAggregate { + world_address: entity.world_address, + hashed_keys: entity.hashed_keys, + models: entity.models, + created_at: entity.created_at, + updated_at: entity.updated_at, + executed_at: entity.executed_at, + })) + } + + async fn load_entity_page_meta( + &self, + kind: TableKind, + world_filters: &[Felt], + table_ids: Vec, + ) -> Result>> { + let deleted_sql = match self.state.backend { + DbBackend::Sqlite => "CAST(deleted AS INTEGER)", + DbBackend::Postgres => "CASE WHEN deleted THEN 1 ELSE 0 END", + }; + let mut builder = QueryBuilder::::new(format!( + "SELECT world_address, table_id, entity_id, created_at, updated_at, executed_at, \ + {deleted_sql} AS deleted \ + FROM torii_ecs_entity_meta WHERE kind = " + )); + builder.push_bind(kind.as_str()); + builder.push(" AND table_id IN ("); + { + let mut separated = builder.separated(", "); + for table_id in &table_ids { + separated.push_bind(table_id); + } + } + builder.push(")"); + if !world_filters.is_empty() { + builder.push(" AND world_address IN ("); + let mut separated = builder.separated(", "); + for world_address in world_filters { + separated.push_bind(felt_hex(*world_address)); + } + builder.push(")"); + } + + let rows = builder.build().fetch_all(&self.state.pool).await?; + let mut meta = HashMap::new(); + for row in rows { + let world_address = row.try_get::("world_address")?; + let entity_id = row.try_get::("entity_id")?; + meta.entry((world_address, entity_id)) + .or_insert_with(HashMap::new) + .insert( + row.try_get::("table_id")?, + EntityMetaRow { + created_at: row.try_get::("created_at")? as u64, + updated_at: row.try_get::("updated_at")? as u64, + executed_at: row.try_get::("executed_at")? as u64, + deleted: row.try_get::("deleted")? != 0, + }, + ); + } + meta.retain(|_, tables| tables.values().any(|meta| !meta.deleted)); + Ok(meta) + } + + async fn load_entity_page_rows( + &self, + kind: TableKind, + world_filters: &[Felt], + table_ids: Vec, + entity_keys: &[(String, String)], + ) -> Result> { + if entity_keys.is_empty() { + return Ok(Vec::new()); + } + + let entity_ids = entity_keys + .iter() + .map(|(_, entity_id)| entity_id.clone()) + .collect::>(); + let world_addresses = if world_filters.is_empty() { + entity_keys + .iter() + .map(|(world_address, _)| world_address.clone()) + .collect::>() + } else { + world_filters.iter().map(|world| felt_hex(*world)).collect() + }; + + let mut builder = QueryBuilder::::new( + "SELECT world_address, table_id, entity_id, row_json \ + FROM torii_ecs_entity_models WHERE kind = ", + ); + builder.push_bind(kind.as_str()); + builder.push(" AND table_id IN ("); + { + let mut separated = builder.separated(", "); + for table_id in &table_ids { + separated.push_bind(table_id); + } + } + builder.push(") AND world_address IN ("); + { + let mut separated = builder.separated(", "); + for world_address in &world_addresses { + separated.push_bind(world_address); + } + } + builder.push(") AND entity_id IN ("); + { + let mut separated = builder.separated(", "); + for entity_id in &entity_ids { + separated.push_bind(entity_id); + } + } + builder.push(")"); + + let allowed = entity_keys.iter().cloned().collect::>(); + let rows = builder.build().fetch_all(&self.state.pool).await?; + let mut output = Vec::new(); + for row in rows { + let world_address = row.try_get::("world_address")?; + let entity_id = row.try_get::("entity_id")?; + if !allowed.contains(&(world_address.clone(), entity_id.clone())) { + continue; + } + output.push(EntityModelRow { + world_address, + table_id: row.try_get("table_id")?, + entity_id, + row_json: row.try_get("row_json")?, + }); + } + Ok(output) + } + + async fn load_entity_snapshot( + &self, + kind: TableKind, + world_address: Felt, + entity_id: Felt, + ) -> Result> { + let sql = match self.state.backend { + DbBackend::Sqlite => { + "SELECT table_id, row_json + FROM torii_ecs_entity_models + WHERE kind = ?1 AND world_address = ?2 AND entity_id = ?3" + } + DbBackend::Postgres => { + "SELECT table_id, row_json + FROM torii_ecs_entity_models + WHERE kind = $1 AND world_address = $2 AND entity_id = $3" + } + }; + let rows = sqlx::query(sql) + .bind(kind.as_str()) + .bind(felt_hex(world_address)) + .bind(felt_hex(entity_id)) + .fetch_all(&self.state.pool) + .await?; + if rows.is_empty() { + return Ok(None); + } + + let meta = self + .load_entity_meta_rows(kind, world_address, entity_id) + .await?; + let managed_tables = self.load_managed_table_map().await?; + let mut aggregate = EntityAggregate { + world_address: world_address.to_bytes_be().to_vec(), + hashed_keys: entity_id.to_bytes_be().to_vec(), + ..EntityAggregate::default() + }; + + for row in rows { + let table_id = row.try_get::("table_id")?; + let Some(table) = managed_tables.get(&table_id) else { + continue; + }; + let Some(table_meta) = meta.get(&table_id) else { + continue; + }; + if table_meta.deleted { + continue; + } + + let row_json: String = row.try_get("row_json")?; + let value: Value = serde_json::from_str(&row_json)?; + let object = value + .as_object() + .cloned() + .ok_or_else(|| anyhow!("entity snapshot row must be an object"))?; + aggregate + .models + .push(row_to_model_struct(&table.table, &object)?); + if aggregate.created_at == 0 || table_meta.created_at < aggregate.created_at { + aggregate.created_at = table_meta.created_at; + } + aggregate.updated_at = aggregate.updated_at.max(table_meta.updated_at); + aggregate.executed_at = aggregate.executed_at.max(table_meta.executed_at); + } + + if aggregate.models.is_empty() { + return Ok(None); + } + Ok(Some(aggregate)) + } + + async fn load_entity_meta_rows( + &self, + kind: TableKind, + world_address: Felt, + entity_id: Felt, + ) -> Result> { + let sql = match self.state.backend { + DbBackend::Sqlite => "SELECT table_id, created_at, updated_at, executed_at, + CAST(deleted AS INTEGER) AS deleted + FROM torii_ecs_entity_meta + WHERE kind = ? AND world_address = ? AND entity_id = ?" + .to_string(), + DbBackend::Postgres => "SELECT table_id, created_at, updated_at, executed_at, + CASE WHEN deleted THEN 1 ELSE 0 END AS deleted + FROM torii_ecs_entity_meta + WHERE kind = $1 AND world_address = $2 AND entity_id = $3" + .to_string(), + }; + let rows = sqlx::query(&sql) + .bind(kind.as_str()) + .bind(felt_hex(world_address)) + .bind(felt_hex(entity_id)) + .fetch_all(&self.state.pool) + .await?; + let mut meta = HashMap::new(); + for row in rows { + meta.insert( + row.try_get::("table_id")?, + EntityMetaRow { + created_at: row.try_get::("created_at")? as u64, + updated_at: row.try_get::("updated_at")? as u64, + executed_at: row.try_get::("executed_at")? as u64, + deleted: row.try_get::("deleted")? != 0, + }, + ); + } + Ok(meta) + } + + async fn load_entity_meta( + &self, + kind: TableKind, + world_address: Felt, + table_id: Felt, + ) -> Result> { + let sql = match self.state.backend { + DbBackend::Sqlite => "SELECT entity_id, created_at, updated_at, executed_at, + CAST(deleted AS INTEGER) AS deleted + FROM torii_ecs_entity_meta + WHERE kind = ? AND world_address = ? AND table_id = ?" + .to_string(), + DbBackend::Postgres => "SELECT entity_id, created_at, updated_at, executed_at, + CASE WHEN deleted THEN 1 ELSE 0 END AS deleted + FROM torii_ecs_entity_meta + WHERE kind = $1 AND world_address = $2 AND table_id = $3" + .to_string(), + }; + let rows = sqlx::query(&sql) + .bind(kind.as_str()) + .bind(felt_hex(world_address)) + .bind(felt_hex(table_id)) + .fetch_all(&self.state.pool) + .await?; + let mut meta = HashMap::new(); + for row in rows { + meta.insert( + row.try_get::("entity_id")?, + EntityMetaRow { + created_at: row.try_get::("created_at")? as u64, + updated_at: row.try_get::("updated_at")? as u64, + executed_at: row.try_get::("executed_at")? as u64, + deleted: row.try_get::("deleted")? != 0, + }, + ); + } + Ok(meta) + } + + async fn fetch_table_rows(&self, table: &ManagedTable) -> Result>> { + match self.state.backend { + DbBackend::Sqlite => { + let sql = format!( + "SELECT * FROM \"{}\"", + table.table.name.replace('"', "\"\"") + ); + let rows = sqlx::query(&sql).fetch_all(&self.state.pool).await?; + rows.into_iter() + .map(|row| any_row_to_json_map(&row, &table.table)) + .collect() + } + DbBackend::Postgres => { + let sql = format!( + "SELECT row_to_json(t)::text AS data FROM (SELECT * FROM \"{}\") t", + table.table.name.replace('"', "\"\"") + ); + let rows = sqlx::query(&sql).fetch_all(&self.state.pool).await?; + rows.into_iter() + .map(|row| { + let data: String = row.try_get("data")?; + let value: Value = serde_json::from_str(&data)?; + value + .as_object() + .cloned() + .ok_or_else(|| anyhow!("row_to_json must return an object")) + }) + .collect() + } + } + } + + async fn load_events_page( + &self, + query: &types::EventQuery, + ) -> Result<(Vec, String)> { + let pagination = query.pagination.clone().unwrap_or(types::Pagination { + cursor: String::new(), + limit: 100, + direction: PaginationDirection::Forward as i32, + order_by: Vec::new(), + }); + let limit = if pagination.limit == 0 { + 100 + } else { + pagination.limit as usize + }; + let direction_is_backward = pagination.direction == PaginationDirection::Backward as i32; + let order_sql = if direction_is_backward { "DESC" } else { "ASC" }; + + let mut builder = QueryBuilder::::new( + "SELECT event_id, transaction_hash, keys_json, data_json \ + FROM torii_ecs_events", + ); + if !pagination.cursor.is_empty() { + builder.push(" WHERE event_id "); + builder.push(if direction_is_backward { "< " } else { "> " }); + builder.push_bind(pagination.cursor.clone()); + } + builder.push(" ORDER BY event_id "); + builder.push(order_sql); + + let fetch_limit = if query.keys.is_some() { + limit.saturating_mul(20).max(100) + } else { + limit + }; + builder.push(" LIMIT "); + builder.push_bind(fetch_limit as i64); + + let rows = builder.build().fetch_all(&self.state.pool).await?; + + let mut events = Vec::new(); + for row in rows { + let keys_hex: Vec = + serde_json::from_str(&row.try_get::("keys_json")?)?; + let data_hex: Vec = + serde_json::from_str(&row.try_get::("data_json")?)?; + let event = types::Event { + keys: keys_hex + .into_iter() + .map(|value| felt_from_hex(&value).map(|felt| felt.to_bytes_be().to_vec())) + .collect::>>()?, + data: data_hex + .into_iter() + .map(|value| felt_from_hex(&value).map(|felt| felt.to_bytes_be().to_vec())) + .collect::>>()?, + transaction_hash: felt_from_hex(&row.try_get::("transaction_hash")?)? + .to_bytes_be() + .to_vec(), + }; + if query + .keys + .as_ref() + .is_none_or(|keys| match_keys(&event.keys, std::slice::from_ref(keys))) + { + events.push((row.try_get::("event_id")?, event)); + } + if events.len() >= limit { + break; + } + } + + if direction_is_backward { + events.reverse(); + } + let next_cursor = events + .last() + .map(|(cursor, _)| cursor.clone()) + .unwrap_or_default(); + Ok(( + events.into_iter().map(|(_, event)| event).collect(), + next_cursor, + )) + } + + async fn next_subscription_id() -> u64 { + use std::sync::atomic::{AtomicU64, Ordering}; + static NEXT_ID: AtomicU64 = AtomicU64::new(1); + NEXT_ID.fetch_add(1, Ordering::Relaxed) + } +} + +#[tonic::async_trait] +impl World for EcsService { + type SubscribeContractsStream = ReceiverStream>; + type SubscribeEntitiesStream = ReceiverStream>; + type SubscribeEventMessagesStream = ReceiverStream>; + type SubscribeEventsStream = ReceiverStream>; + + async fn subscribe_contracts( + &self, + request: Request, + ) -> Result, Status> { + let query = request + .into_inner() + .query + .ok_or_else(|| Status::invalid_argument("Missing query argument"))?; + let (sender, receiver) = mpsc::channel(256); + for contract in self.load_contracts(&query).await.map_err(internal_status)? { + let _ = sender + .send(Ok(SubscribeContractsResponse { + contract: Some(contract), + })) + .await; + } + self.state.contract_subscriptions.lock().await.insert( + Self::next_subscription_id().await, + ContractSubscription { query, sender }, + ); + Ok(Response::new(ReceiverStream::new(receiver))) + } + + async fn worlds( + &self, + request: Request, + ) -> Result, Status> { + let world_addresses = request + .into_inner() + .world_addresses + .iter() + .map(|bytes| felt_from_bytes(bytes)) + .collect::>>() + .map_err(internal_status)?; + let worlds = self + .load_worlds(&world_addresses) + .await + .map_err(internal_status)?; + Ok(Response::new(WorldsResponse { worlds })) + } + + async fn subscribe_entities( + &self, + request: Request, + ) -> Result, Status> { + let request = request.into_inner(); + let subscription_id = Self::next_subscription_id().await; + let (sender, receiver) = mpsc::channel(256); + sender + .send(Ok(SubscribeEntityResponse { + entity: None, + subscription_id, + })) + .await + .map_err(|_| Status::internal("subscription setup failed"))?; + self.state.entity_subscriptions.lock().await.insert( + subscription_id, + EntitySubscription { + clause: request.clause, + world_addresses: request.world_addresses, + sender, + }, + ); + Ok(Response::new(ReceiverStream::new(receiver))) + } + + async fn update_entities_subscription( + &self, + request: Request, + ) -> Result, Status> { + let request = request.into_inner(); + if let Some(subscription) = self + .state + .entity_subscriptions + .lock() + .await + .get_mut(&request.subscription_id) + { + subscription.clause = request.clause; + subscription.world_addresses = request.world_addresses; + } + Ok(Response::new(())) + } + + async fn retrieve_entities( + &self, + request: Request, + ) -> Result, Status> { + let query = request + .into_inner() + .query + .ok_or_else(|| Status::invalid_argument("Missing query argument"))?; + let (entities, next_cursor) = self + .load_entity_page(TableKind::Entity, &query) + .await + .map_err(internal_status)?; + Ok(Response::new(RetrieveEntitiesResponse { + next_cursor, + entities, + })) + } + + async fn subscribe_event_messages( + &self, + request: Request, + ) -> Result, Status> { + let request = request.into_inner(); + let subscription_id = Self::next_subscription_id().await; + let (sender, receiver) = mpsc::channel(256); + sender + .send(Ok(SubscribeEntityResponse { + entity: None, + subscription_id, + })) + .await + .map_err(|_| Status::internal("subscription setup failed"))?; + self.state.event_message_subscriptions.lock().await.insert( + subscription_id, + EntitySubscription { + clause: request.clause, + world_addresses: request.world_addresses, + sender, + }, + ); + Ok(Response::new(ReceiverStream::new(receiver))) + } + + async fn update_event_messages_subscription( + &self, + request: Request, + ) -> Result, Status> { + let request = request.into_inner(); + if let Some(subscription) = self + .state + .event_message_subscriptions + .lock() + .await + .get_mut(&request.subscription_id) + { + subscription.clause = request.clause; + subscription.world_addresses = request.world_addresses; + } + Ok(Response::new(())) + } + + async fn retrieve_event_messages( + &self, + request: Request, + ) -> Result, Status> { + let query = request + .into_inner() + .query + .ok_or_else(|| Status::invalid_argument("Missing query argument"))?; + let (entities, next_cursor) = self + .load_entity_page(TableKind::EventMessage, &query) + .await + .map_err(internal_status)?; + Ok(Response::new(RetrieveEntitiesResponse { + next_cursor, + entities, + })) + } + + async fn retrieve_events( + &self, + request: Request, + ) -> Result, Status> { + let query = request + .into_inner() + .query + .ok_or_else(|| Status::invalid_argument("Missing query argument"))?; + let (events, next_cursor) = self + .load_events_page(&query) + .await + .map_err(internal_status)?; + Ok(Response::new(RetrieveEventsResponse { + next_cursor, + events, + })) + } + + async fn subscribe_events( + &self, + request: Request, + ) -> Result, Status> { + let request = request.into_inner(); + let (sender, receiver) = mpsc::channel(256); + sender + .send(Ok(SubscribeEventsResponse { event: None })) + .await + .map_err(|_| Status::internal("subscription setup failed"))?; + self.state.event_subscriptions.lock().await.insert( + Self::next_subscription_id().await, + EventSubscription { + keys: request.keys, + sender, + }, + ); + Ok(Response::new(ReceiverStream::new(receiver))) + } +} + +#[derive(Clone, Copy)] +struct EntityMetaRow { + created_at: u64, + updated_at: u64, + executed_at: u64, + deleted: bool, +} + +struct EntityModelRow { + world_address: String, + table_id: String, + entity_id: String, + row_json: String, +} + +struct SnapshotJsonSerializer; + +fn internal_status(error: anyhow::Error) -> Status { + Status::internal(error.to_string()) +} + +fn sqlite_url(path: &str) -> Result { + if path == ":memory:" || path == "sqlite::memory:" { + return Ok("sqlite::memory:".to_string()); + } + if path.starts_with("sqlite:") { + return Ok(path.to_string()); + } + let options = SqliteConnectOptions::from_str(&format!("sqlite://{path}")) + .or_else(|_| Ok::<_, sqlx::Error>(SqliteConnectOptions::new().filename(path)))?; + if let Some(parent) = options + .get_filename() + .parent() + .filter(|path| !path.as_os_str().is_empty()) + { + std::fs::create_dir_all(parent)?; + } + Ok(options.to_url_lossy().to_string()) +} + +impl CairoTypeSerialization for SnapshotJsonSerializer { + fn serialize_byte_array( + &self, + serializer: S, + value: &[u8], + ) -> Result { + serializer.serialize_str(&format!("0x{}", hex::encode(value))) + } + + fn serialize_felt( + &self, + serializer: S, + value: &[u8; 32], + ) -> Result { + self.serialize_byte_array(serializer, value) + } + + fn serialize_eth_address( + &self, + serializer: S, + value: &[u8; 20], + ) -> Result { + self.serialize_byte_array(serializer, value) + } + + fn serialize_tuple<'a, S: Serializer>( + &'a self, + data: &mut impl CairoDeserializer, + serializer: S, + tuple: &'a TupleDef, + ) -> Result { + let mut seq = serializer.serialize_map(Some(tuple.elements.len()))?; + for (index, element) in tuple.elements.iter().enumerate() { + seq.serialize_entry(&format!("_{index}"), &element.to_de_se(data, self))?; + } + seq.end() + } + + fn serialize_variant<'a, S: Serializer>( + &'a self, + data: &mut impl CairoDeserializer, + serializer: S, + name: &str, + type_def: &'a TypeDef, + ) -> Result { + if type_def == &TypeDef::None { + let mut map = serializer.serialize_map(Some(1))?; + map.serialize_entry("variant", name)?; + map + } else { + let mut map = serializer.serialize_map(Some(2))?; + map.serialize_entry("variant", name)?; + map.serialize_entry(&format!("_{name}"), &type_def.to_de_se(data, self))?; + map + } + .end() + } + + fn serialize_result<'a, S: Serializer>( + &'a self, + data: &mut impl CairoDeserializer, + serializer: S, + result: &'a ResultDef, + is_ok: bool, + ) -> Result { + let mut map = serializer.serialize_map(Some(1))?; + map.serialize_entry("is_ok", &is_ok)?; + if is_ok { + map.serialize_entry("Ok", &result.ok.to_de_se(data, self))?; + } else { + map.serialize_entry("Err", &result.err.to_de_se(data, self))?; + } + map.end() + } +} + +fn model_from_table(table: &ManagedTable) -> types::Model { + let (namespace, name) = split_table_name(&table.table.name); + types::Model { + selector: table.table.id.to_bytes_be().to_vec(), + namespace, + name, + packed_size: table.table.columns.len() as u32, + unpacked_size: table.table.columns.len() as u32, + class_hash: Vec::new(), + layout: Vec::new(), + schema: serde_json::to_vec(&table.table).unwrap_or_default(), + contract_address: Vec::new(), + use_legacy_store: table.table.legacy, + world_address: table.world_address.to_bytes_be().to_vec(), + } +} + +fn split_table_name(name: &str) -> (String, String) { + match name.split_once('-') { + Some((namespace, model)) => (namespace.to_string(), model.to_string()), + None => (String::new(), name.to_string()), + } +} + +fn row_to_model_struct(table: &DojoTable, row: &Map) -> Result { + let children = table + .key_fields + .iter() + .chain(table.value_fields.iter()) + .filter_map(|column_id| table.columns.get(column_id)) + .map(|column| { + let value = row.get(&column.name).cloned().unwrap_or(Value::Null); + Ok(types::Member { + name: column.name.clone(), + ty: Some(type_to_proto_value(&column.type_def, value)?), + key: column.attributes.has_attribute("key"), + }) + }) + .collect::>>()?; + + Ok(types::Struct { + name: table.name.clone(), + children, + }) +} + +fn record_to_json_map( + table: &DojoTable, + columns: &[Felt], + record: &Record, +) -> Result> { + let schema_table = table.to_schema(); + let columns_by_id = schema_table + .columns + .iter() + .cloned() + .map(|column| (column.id, column)) + .collect::>(); + let schema_columns = columns + .iter() + .map(|column_id| { + columns_by_id + .get(column_id) + .ok_or_else(|| anyhow!("column {column_id:#x} not found in table {}", table.name)) + }) + .collect::>>()?; + let schema = torii_introspect::tables::RecordSchema::new(&schema_table.primary, schema_columns); + + let mut bytes = Vec::new(); + let mut serializer = JsonSerializer::new(&mut bytes); + schema.parse_records_with_metadata( + std::slice::from_ref(record), + &(), + &mut serializer, + &SnapshotJsonSerializer, + )?; + + let value = serde_json::from_slice::>(&bytes)? + .into_iter() + .next() + .ok_or_else(|| anyhow!("record serializer returned no rows"))?; + value + .as_object() + .cloned() + .ok_or_else(|| anyhow!("record serializer must return an object")) +} + +fn build_entity_from_snapshot( + key: &(String, String), + meta: &HashMap, + rows: Option<&Vec<(String, String)>>, + models: &HashMap, +) -> Result> { + let Some(rows) = rows else { + return Ok(None); + }; + + let mut aggregate = EntityAggregate { + world_address: felt_from_hex(&key.0)?.to_bytes_be().to_vec(), + hashed_keys: felt_from_hex(&key.1)?.to_bytes_be().to_vec(), + ..EntityAggregate::default() + }; + + for (table_id, row_json) in rows { + let Some(table_meta) = meta.get(table_id) else { + continue; + }; + if table_meta.deleted { + continue; + } + let Some(table) = models.get(table_id) else { + continue; + }; + + let value: Value = serde_json::from_str(row_json)?; + let object = value + .as_object() + .cloned() + .ok_or_else(|| anyhow!("entity snapshot row must be an object"))?; + aggregate + .models + .push(row_to_model_struct(&table.table, &object)?); + if aggregate.created_at == 0 || table_meta.created_at < aggregate.created_at { + aggregate.created_at = table_meta.created_at; + } + aggregate.updated_at = aggregate.updated_at.max(table_meta.updated_at); + aggregate.executed_at = aggregate.executed_at.max(table_meta.executed_at); + } + + if aggregate.models.is_empty() { + return Ok(None); + } + Ok(Some(aggregate)) +} + +fn any_row_to_json_map(row: &sqlx::any::AnyRow, table: &DojoTable) -> Result> { + let mut map = Map::new(); + let primary_name = &table.primary.name; + map.insert( + primary_name.clone(), + read_row_value(row, primary_name, &TypeDef::from(&table.primary.type_def))?, + ); + for column in table.columns.values() { + map.insert( + column.name.clone(), + read_row_value(row, &column.name, &column.type_def)?, + ); + } + Ok(map) +} + +fn read_row_value(row: &sqlx::any::AnyRow, name: &str, type_def: &TypeDef) -> Result { + if is_integer_type(type_def) { + let value = row.try_get::, _>(name)?; + return Ok(value.map_or(Value::Null, |value| Value::Number(value.into()))); + } + + let value = row.try_get::, _>(name)?; + let Some(value) = value else { + return Ok(Value::Null); + }; + if is_json_text_type(type_def) { + Ok(serde_json::from_str(&value).unwrap_or(Value::String(value))) + } else { + Ok(Value::String(value)) + } +} + +fn is_integer_type(type_def: &TypeDef) -> bool { + matches!( + type_def, + TypeDef::Bool + | TypeDef::I8 + | TypeDef::I16 + | TypeDef::I32 + | TypeDef::I64 + | TypeDef::U8 + | TypeDef::U16 + | TypeDef::U32 + ) +} + +fn is_json_text_type(type_def: &TypeDef) -> bool { + matches!( + type_def, + TypeDef::Tuple(_) + | TypeDef::Array(_) + | TypeDef::FixedArray(_) + | TypeDef::Struct(_) + | TypeDef::Enum(_) + | TypeDef::Option(_) + | TypeDef::Nullable(_) + | TypeDef::Result(_) + | TypeDef::Felt252Dict(_) + ) +} + +fn type_to_proto_value(type_def: &TypeDef, value: Value) -> Result { + use types::ty::TyType; + + let ty_type = match type_def { + TypeDef::Bool => TyType::Primitive(types::Primitive { + primitive_type: Some(types::primitive::PrimitiveType::Bool( + value.as_bool().unwrap_or_default(), + )), + }), + TypeDef::I8 | TypeDef::I16 | TypeDef::I32 => TyType::Primitive(types::Primitive { + primitive_type: Some(types::primitive::PrimitiveType::I32( + value.as_i64().unwrap_or_default() as i32, + )), + }), + TypeDef::I64 => TyType::Primitive(types::Primitive { + primitive_type: Some(types::primitive::PrimitiveType::I64( + value.as_i64().unwrap_or_default(), + )), + }), + TypeDef::U8 | TypeDef::U16 | TypeDef::U32 => TyType::Primitive(types::Primitive { + primitive_type: Some(types::primitive::PrimitiveType::U32( + value.as_u64().unwrap_or_default() as u32, + )), + }), + TypeDef::U64 => TyType::Primitive(types::Primitive { + primitive_type: Some(types::primitive::PrimitiveType::U64(value_as_u64(&value))), + }), + TypeDef::Felt252 + | TypeDef::ClassHash + | TypeDef::ContractAddress + | TypeDef::StorageAddress + | TypeDef::StorageBaseAddress + | TypeDef::EthAddress => { + let bytes = value_as_felt_bytes(&value)?; + let primitive = match type_def { + TypeDef::ClassHash => types::primitive::PrimitiveType::ClassHash(bytes), + TypeDef::ContractAddress + | TypeDef::StorageAddress + | TypeDef::StorageBaseAddress => { + types::primitive::PrimitiveType::ContractAddress(bytes) + } + TypeDef::EthAddress => types::primitive::PrimitiveType::EthAddress(bytes), + _ => types::primitive::PrimitiveType::Felt252(bytes), + }; + TyType::Primitive(types::Primitive { + primitive_type: Some(primitive), + }) + } + TypeDef::U128 | TypeDef::I128 | TypeDef::U256 | TypeDef::U512 => { + TyType::Primitive(types::Primitive { + primitive_type: Some(types::primitive::PrimitiveType::U256(value_as_bytes( + &value, + ))), + }) + } + TypeDef::Utf8String + | TypeDef::ShortUtf8 + | TypeDef::ByteArray + | TypeDef::ByteArrayEncoded(_) + | TypeDef::Bytes31 + | TypeDef::Bytes31Encoded(_) => TyType::Bytearray(value_as_string(&value)), + TypeDef::Struct(def) => { + let object = value.as_object().cloned().unwrap_or_default(); + TyType::Struct(types::Struct { + name: def.name.clone(), + children: def + .members + .iter() + .map(|member| { + Ok(types::Member { + name: member.name.clone(), + ty: Some(type_to_proto_value( + &member.type_def, + object.get(&member.name).cloned().unwrap_or(Value::Null), + )?), + key: member.attributes.has_attribute("key"), + }) + }) + .collect::>>()?, + }) + } + TypeDef::Tuple(def) => TyType::Tuple(types::Array { + children: value + .as_array() + .cloned() + .unwrap_or_default() + .into_iter() + .zip(def.elements.iter()) + .map(|(item, inner)| type_to_proto_value(inner, item)) + .collect::>>()?, + }), + TypeDef::Array(def) => TyType::Array(types::Array { + children: value + .as_array() + .cloned() + .unwrap_or_default() + .into_iter() + .map(|item| type_to_proto_value(&def.type_def, item)) + .collect::>>()?, + }), + TypeDef::FixedArray(def) => TyType::FixedSizeArray(types::FixedSizeArray { + children: value + .as_array() + .cloned() + .unwrap_or_default() + .into_iter() + .map(|item| type_to_proto_value(&def.type_def, item)) + .collect::>>()?, + size: def.size, + }), + TypeDef::Enum(def) => { + let object = value.as_object().cloned().unwrap_or_default(); + let variant_name = object + .get("option") + .and_then(Value::as_str) + .unwrap_or_default() + .to_string(); + let (option, options) = def + .order + .iter() + .enumerate() + .filter_map(|(index, selector)| { + def.variants.get(selector).map(|variant| (index, variant)) + }) + .map(|(index, variant)| { + let ty = object + .get(&variant.name) + .cloned() + .map(|value| type_to_proto_value(&variant.type_def, value)) + .transpose()? + .unwrap_or_default(); + Ok::<_, anyhow::Error>(( + if variant.name == variant_name { + index as u32 + } else { + u32::MAX + }, + types::EnumOption { + name: variant.name.clone(), + ty: Some(ty), + }, + )) + }) + .collect::>>()? + .into_iter() + .fold( + (0, Vec::new()), + |(selected, mut options), (index, option)| { + let selected = if index == u32::MAX { selected } else { index }; + options.push(option); + (selected, options) + }, + ); + TyType::Enum(types::Enum { + name: def.name.clone(), + option, + options, + }) + } + TypeDef::Option(def) => { + if value.is_null() { + TyType::Bytearray(String::new()) + } else { + return type_to_proto_value(&def.type_def, value); + } + } + TypeDef::Nullable(def) => { + if value.is_null() { + TyType::Bytearray(String::new()) + } else { + return type_to_proto_value(&def.type_def, value); + } + } + TypeDef::Result(def) => { + let object = value.as_object().cloned().unwrap_or_default(); + if let Some(ok) = object.get("Ok").cloned() { + return type_to_proto_value(&def.ok, ok); + } + if let Some(err) = object.get("Err").cloned() { + return type_to_proto_value(&def.err, err); + } + TyType::Bytearray(value_as_string(&value)) + } + TypeDef::None | TypeDef::Felt252Dict(_) | TypeDef::Ref(_) | TypeDef::Custom(_) => { + TyType::Bytearray(value_as_string(&value)) + } + }; + + Ok(types::Ty { + ty_type: Some(ty_type), + }) +} + +fn value_as_u64(value: &Value) -> u64 { + value + .as_u64() + .or_else(|| value.as_i64().map(|value| value as u64)) + .or_else(|| value.as_str().and_then(|value| value.parse::().ok())) + .unwrap_or_default() +} + +fn value_as_string(value: &Value) -> String { + match value { + Value::String(value) => value.clone(), + Value::Null => String::new(), + other => other.to_string(), + } +} + +fn value_as_bytes(value: &Value) -> Vec { + value.as_str().map_or_else( + || value_as_string(value).into_bytes(), + |value| value.as_bytes().to_vec(), + ) +} + +fn value_as_felt_bytes(value: &Value) -> Result> { + let string = value_as_string(value); + Ok(felt_from_hex(&string)?.to_bytes_be().to_vec()) +} + +fn value_to_primary_felt(value: &Value, type_def: &PrimaryTypeDef) -> Result { + match type_def { + PrimaryTypeDef::Bool => Ok(Felt::from(value.as_bool().unwrap_or_default() as u64)), + PrimaryTypeDef::U8 + | PrimaryTypeDef::U16 + | PrimaryTypeDef::U32 + | PrimaryTypeDef::U64 + | PrimaryTypeDef::U128 + | PrimaryTypeDef::I8 + | PrimaryTypeDef::I16 + | PrimaryTypeDef::I32 + | PrimaryTypeDef::I64 + | PrimaryTypeDef::I128 => Ok(Felt::from(value_as_u64(value))), + _ => felt_from_hex(&value_as_string(value)), + } +} + +fn entity_matches_clause(entity: &types::Entity, clause: &types::Clause) -> bool { + match &clause.clause_type { + Some(ClauseType::HashedKeys(hashed)) => { + hashed.hashed_keys.is_empty() || hashed.hashed_keys.contains(&entity.hashed_keys) + } + Some(ClauseType::Keys(keys)) => { + if !keys.models.is_empty() + && !entity + .models + .iter() + .any(|model| keys.models.contains(&model.name)) + { + return false; + } + let entity_keys = entity_key_values(entity); + if keys.pattern_matching == PatternMatching::FixedLen as i32 + && entity_keys.len() != keys.keys.len() + { + return false; + } + entity_keys.iter().enumerate().all(|(index, key)| { + keys.keys + .get(index) + .is_none_or(|expected| expected.is_empty() || expected == key) + }) + } + Some(ClauseType::Member(member)) => entity.models.iter().any(|model| { + model.name == member.model + && model_member_matches( + model, + &member.member, + member.operator, + member.value.as_ref(), + ) + }), + Some(ClauseType::Composite(composite)) => match composite.operator { + value if value == LogicalOperator::And as i32 => composite + .clauses + .iter() + .all(|clause| entity_matches_clause(entity, clause)), + _ => composite + .clauses + .iter() + .any(|clause| entity_matches_clause(entity, clause)), + }, + None => true, + } +} + +fn match_keys(keys: &[Vec], clauses: &[types::KeysClause]) -> bool { + clauses.is_empty() + || clauses.iter().any(|clause| { + if clause.pattern_matching == PatternMatching::FixedLen as i32 + && keys.len() != clause.keys.len() + { + return false; + } + keys.iter().enumerate().all(|(index, key)| { + clause + .keys + .get(index) + .is_none_or(|expected| expected.is_empty() || expected == key) + }) + }) +} + +fn model_member_matches( + model: &types::Struct, + path: &str, + operator: i32, + value: Option<&types::MemberValue>, +) -> bool { + let mut current = find_member_value(model, path); + let Some(current) = current.take() else { + return false; + }; + let Some(value) = value else { + return false; + }; + compare_member_value(current, operator, value) +} + +fn find_member_value<'a>(model: &'a types::Struct, path: &str) -> Option<&'a types::Ty> { + let mut members = &model.children; + let parts = path.split('.').collect::>(); + let mut current = None; + for (index, part) in parts.iter().enumerate() { + let member = members.iter().find(|member| member.name == *part)?; + current = member.ty.as_ref(); + if index + 1 == parts.len() { + return current; + } + let ty = current?.ty_type.as_ref()?; + match ty { + types::ty::TyType::Struct(value) => { + members = &value.children; + } + _ => return None, + } + } + current +} + +fn compare_member_value(current: &types::Ty, operator: i32, expected: &types::MemberValue) -> bool { + let current_value = scalar_from_ty(current); + let expected_value = scalar_from_member_value(expected); + match (current_value, expected_value) { + (Some(current), Some(expected)) => compare_scalars(¤t, operator, &expected), + _ => false, + } +} + +fn scalar_from_ty(value: &types::Ty) -> Option { + match value.ty_type.as_ref()? { + types::ty::TyType::Primitive(primitive) => match primitive.primitive_type.as_ref()? { + types::primitive::PrimitiveType::Bool(value) => Some(ScalarValue::Bool(*value)), + types::primitive::PrimitiveType::I32(value) => Some(ScalarValue::Signed(*value as i64)), + types::primitive::PrimitiveType::I64(value) => Some(ScalarValue::Signed(*value)), + types::primitive::PrimitiveType::U32(value) => { + Some(ScalarValue::Unsigned(*value as u64)) + } + types::primitive::PrimitiveType::U64(value) => Some(ScalarValue::Unsigned(*value)), + types::primitive::PrimitiveType::Felt252(value) + | types::primitive::PrimitiveType::ClassHash(value) + | types::primitive::PrimitiveType::ContractAddress(value) + | types::primitive::PrimitiveType::EthAddress(value) + | types::primitive::PrimitiveType::U256(value) + | types::primitive::PrimitiveType::U128(value) + | types::primitive::PrimitiveType::I128(value) => { + Some(ScalarValue::Bytes(value.clone())) + } + _ => None, + }, + types::ty::TyType::Bytearray(value) => Some(ScalarValue::Text(value.clone())), + _ => None, + } +} + +fn scalar_from_member_value(value: &types::MemberValue) -> Option { + match value.value_type.as_ref()? { + ValueType::Primitive(primitive) => scalar_from_ty(&types::Ty { + ty_type: Some(types::ty::TyType::Primitive(primitive.clone())), + }), + ValueType::String(value) => Some(ScalarValue::Text(value.clone())), + ValueType::List(_) => None, + } +} + +#[derive(Clone)] +enum ScalarValue { + Bool(bool), + Signed(i64), + Unsigned(u64), + Text(String), + Bytes(Vec), +} + +fn compare_scalars(current: &ScalarValue, operator: i32, expected: &ScalarValue) -> bool { + match (current, expected) { + (ScalarValue::Bool(current), ScalarValue::Bool(expected)) => match operator { + value if value == ComparisonOperator::Eq as i32 => current == expected, + value if value == ComparisonOperator::Neq as i32 => current != expected, + _ => false, + }, + (ScalarValue::Signed(current), ScalarValue::Signed(expected)) => { + compare_ord(current, operator, expected) + } + (ScalarValue::Unsigned(current), ScalarValue::Unsigned(expected)) => { + compare_ord(current, operator, expected) + } + (ScalarValue::Text(current), ScalarValue::Text(expected)) => { + compare_ord(current, operator, expected) + } + (ScalarValue::Bytes(current), ScalarValue::Bytes(expected)) => { + compare_ord(current, operator, expected) + } + _ => false, + } +} + +fn compare_ord(current: &T, operator: i32, expected: &T) -> bool { + match operator { + value if value == ComparisonOperator::Eq as i32 => current == expected, + value if value == ComparisonOperator::Neq as i32 => current != expected, + value if value == ComparisonOperator::Gt as i32 => current > expected, + value if value == ComparisonOperator::Gte as i32 => current >= expected, + value if value == ComparisonOperator::Lt as i32 => current < expected, + value if value == ComparisonOperator::Lte as i32 => current <= expected, + _ => false, + } +} + +fn entity_key_values(entity: &types::Entity) -> Vec> { + entity + .models + .iter() + .find_map(|model| { + let keys = model + .children + .iter() + .filter(|member| member.key) + .filter_map(|member| { + scalar_from_ty(member.ty.as_ref()?).map(|value| match value { + ScalarValue::Bool(value) => vec![u8::from(value)], + ScalarValue::Signed(value) => value.to_be_bytes().to_vec(), + ScalarValue::Unsigned(value) => value.to_be_bytes().to_vec(), + ScalarValue::Text(value) => value.into_bytes(), + ScalarValue::Bytes(value) => value, + }) + }) + .collect::>(); + (!keys.is_empty()).then_some(keys) + }) + .unwrap_or_default() +} + +fn contract_matches_query(contract: &types::Contract, query: &types::ContractQuery) -> bool { + (query.contract_addresses.is_empty() + || query + .contract_addresses + .contains(&contract.contract_address)) + && (query.contract_types.is_empty() + || query.contract_types.contains(&contract.contract_type)) +} + +fn felt_hex(value: Felt) -> String { + format!("{value:#x}") +} + +fn felt_from_hex(value: &str) -> Result { + Felt::from_str(value).map_err(|err| anyhow!("invalid felt {value}: {err}")) +} + +fn felt_from_bytes(value: &[u8]) -> Result { + Ok(Felt::from_bytes_be_slice(value)) +} diff --git a/crates/torii-ecs-sink/src/lib.rs b/crates/torii-ecs-sink/src/lib.rs new file mode 100644 index 00000000..0a3d28c9 --- /dev/null +++ b/crates/torii-ecs-sink/src/lib.rs @@ -0,0 +1,19 @@ +pub mod grpc_service; +pub mod sink; + +pub mod proto { + pub mod world { + tonic::include_proto!("world"); + + pub const FILE_DESCRIPTOR_SET: &[u8] = + tonic::include_file_descriptor_set!("world_descriptor"); + } + + pub mod types { + tonic::include_proto!("types"); + } +} + +pub use grpc_service::EcsService; +pub use proto::world::FILE_DESCRIPTOR_SET; +pub use sink::EcsSink; diff --git a/crates/torii-ecs-sink/src/sink.rs b/crates/torii-ecs-sink/src/sink.rs new file mode 100644 index 00000000..047e6daa --- /dev/null +++ b/crates/torii-ecs-sink/src/sink.rs @@ -0,0 +1,222 @@ +use std::sync::Arc; + +use anyhow::Result; +use async_trait::async_trait; +use dojo_introspect::events::{ + EventEmitted, StoreDelRecord, StoreSetRecord, StoreUpdateMember, StoreUpdateRecord, +}; +use introspect_types::event::CairoEventInfo; +use starknet::core::types::Felt; +use torii::axum::Router; +use torii::etl::{ + envelope::{Envelope, TypeId}, + extractor::ExtractionBatch, + sink::{EventBus, Sink, SinkContext, TopicInfo}, +}; +use torii_introspect::events::{IntrospectBody, IntrospectMsg}; + +use crate::grpc_service::{EcsService, TableKind}; + +pub struct EcsSink { + service: Arc, +} + +impl EcsSink { + pub async fn new(database_url: &str, max_connections: Option) -> Result { + Ok(Self { + service: Arc::new(EcsService::new(database_url, max_connections).await?), + }) + } + + pub fn get_grpc_service_impl(&self) -> Arc { + self.service.clone() + } +} + +#[async_trait] +impl Sink for EcsSink { + fn name(&self) -> &'static str { + "ecs" + } + + fn interested_types(&self) -> Vec { + vec![TypeId::new("introspect")] + } + + async fn process(&self, envelopes: &[Envelope], batch: &ExtractionBatch) -> Result<()> { + for (ordinal, event) in batch.events.iter().enumerate() { + let context = batch + .get_event_context(&event.transaction_hash, event.from_address) + .unwrap_or_default(); + self.service + .store_event( + event.from_address, + event.transaction_hash, + context.transaction.block_number, + context.block.timestamp, + &event.keys, + &event.data, + ordinal, + ) + .await?; + + self.service + .record_contract_progress( + event.from_address, + crate::proto::types::ContractType::World, + context.transaction.block_number, + context.block.timestamp, + None, + ) + .await?; + + let Some(selector) = event.keys.first() else { + continue; + }; + let selector_raw = selector.to_raw(); + if matches!( + selector_raw, + StoreSetRecord::SELECTOR_RAW + | StoreUpdateRecord::SELECTOR_RAW + | StoreUpdateMember::SELECTOR_RAW + | StoreDelRecord::SELECTOR_RAW + ) { + if event.keys.len() >= 3 { + self.service + .record_table_kind(event.from_address, event.keys[1], TableKind::Entity) + .await?; + self.service + .upsert_entity_meta( + TableKind::Entity, + event.from_address, + event.keys[1], + event.keys[2], + context.block.timestamp, + selector_raw == StoreDelRecord::SELECTOR_RAW, + ) + .await?; + } + } else if selector_raw == EventEmitted::SELECTOR_RAW && event.keys.len() >= 2 { + self.service + .record_table_kind(event.from_address, event.keys[1], TableKind::EventMessage) + .await?; + } + } + + for envelope in envelopes { + if envelope.type_id != TypeId::new("introspect") { + continue; + } + + let Some(body) = envelope.downcast_ref::() else { + continue; + }; + let context = batch + .get_event_context(&body.transaction_hash, body.from_address) + .unwrap_or_default(); + + match &body.msg { + IntrospectMsg::CreateTable(table) => { + self.service + .cache_created_table(body.from_address, table) + .await; + self.service + .record_table_kind(body.from_address, table.id, TableKind::Entity) + .await + .ok(); + } + IntrospectMsg::UpdateTable(table) => { + self.service + .cache_updated_table(body.from_address, table) + .await; + self.service + .record_table_kind(body.from_address, table.id, TableKind::Entity) + .await + .ok(); + } + IntrospectMsg::InsertsFields(insert) => { + let kind = if batch.events.iter().any(|event| { + event.keys.first().is_some_and(|selector| { + selector.to_raw() == EventEmitted::SELECTOR_RAW + && event.keys.get(1).copied() == Some(insert.table) + }) + }) { + TableKind::EventMessage + } else { + TableKind::Entity + }; + self.service + .record_table_kind(body.from_address, insert.table, kind) + .await?; + for record in &insert.records { + let entity_id = Felt::from_bytes_be(&record.id); + self.service + .upsert_entity_meta( + kind, + body.from_address, + insert.table, + entity_id, + context.block.timestamp, + false, + ) + .await?; + self.service + .upsert_entity_model( + kind, + body.from_address, + insert.table, + &insert.columns, + record, + context.block.timestamp, + ) + .await?; + self.service + .publish_entity_update(kind, body.from_address, entity_id) + .await?; + } + } + IntrospectMsg::DeleteRecords(delete) => { + let kind = self.service.table_kind(delete.table).await?; + for row in &delete.rows { + let entity_id = row.to_felt(); + self.service + .upsert_entity_meta( + kind, + body.from_address, + delete.table, + entity_id, + context.block.timestamp, + true, + ) + .await?; + self.service + .delete_entity_model(kind, body.from_address, delete.table, entity_id) + .await?; + self.service + .publish_entity_update(kind, body.from_address, entity_id) + .await?; + } + } + _ => {} + } + } + + Ok(()) + } + + fn topics(&self) -> Vec { + Vec::new() + } + + fn build_routes(&self) -> Router { + Router::new() + } + + async fn initialize( + &mut self, + _event_bus: Arc, + _context: &SinkContext, + ) -> Result<()> { + Ok(()) + } +} diff --git a/scripts/test-ecs-grpc-e2e.sh b/scripts/test-ecs-grpc-e2e.sh new file mode 100755 index 00000000..241983af --- /dev/null +++ b/scripts/test-ecs-grpc-e2e.sh @@ -0,0 +1,240 @@ +#!/usr/bin/env bash + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" + +GRPC_ADDR="${GRPC_ADDR:-localhost:3000}" +DB_PATH="${DB_PATH:-${REPO_ROOT}/torii-data/introspect.db}" +DB_URL="${DB_URL:-${DATABASE_URL:-}}" +LIMIT="${LIMIT:-1}" + +log() { + printf '[ecs-e2e] %s\n' "$*" +} + +fail() { + printf '[ecs-e2e] error: %s\n' "$*" >&2 + exit 1 +} + +require_cmd() { + command -v "$1" >/dev/null 2>&1 || fail "missing required command: $1" +} + +sql_one() { + local query="$1" + case "$DB_BACKEND" in + sqlite) + sqlite3 -batch -noheader "$DB_PATH" "$query" | head -n 1 + ;; + postgres) + psql "$DB_URL" -Atqc "$query" | head -n 1 + ;; + *) + fail "unsupported DB_BACKEND: $DB_BACKEND" + ;; + esac +} + +model_name_query() { + local kind="$1" + case "$DB_BACKEND" in + sqlite) + cat <> 'name' +FROM torii_ecs_table_kinds k +JOIN torii_dojo_manager_state m + ON ltrim(lower(encode(m.owner, 'hex')), '0') = ltrim(substr(k.world_address, 3), '0') + AND ltrim(lower(encode(m.table_id, 'hex')), '0') = ltrim(substr(k.table_id, 3), '0') +WHERE k.world_address = '$WORLD_ADDRESS' + AND k.kind = '$kind' +ORDER BY m.table_json::json ->> 'name' +LIMIT 1; +EOF + ;; + *) + fail "unsupported DB_BACKEND: $DB_BACKEND" + ;; + esac +} + +felt_to_base64() { + local hex="${1#0x}" + hex="$(printf '%064s' "$hex" | tr ' ' '0')" + printf '%s' "$hex" | xxd -r -p | base64 | tr -d '\n' +} + +base64_decode() { + if base64 --help 2>&1 | rg -q '\-d'; then + base64 -d + else + base64 -D + fi +} + +base64_to_felt() { + local value="$1" + local hex + hex="$(printf '%s' "$value" | base64_decode | xxd -p -c 256 | tr -d '\n')" + hex="$(printf '%s' "$hex" | sed 's/^0*//')" + if [[ -z "$hex" ]]; then + hex="0" + fi + printf '0x%s' "$hex" +} + +grpc_call() { + local method="$1" + local payload="$2" + local output="$3" + grpcurl -plaintext -d "$payload" "$GRPC_ADDR" "$method" >"$output" +} + +assert_contains() { + local needle="$1" + local file="$2" + rg -F -q "$needle" "$file" || fail "expected to find '$needle' in $file" +} + +assert_non_empty_entities() { + local file="$1" + assert_contains '"entities": [' "$file" + if rg -q '"entities": \[\]' "$file"; then + fail "expected non-empty entities response in $file" + fi +} + +assert_non_empty_events() { + local file="$1" + assert_contains '"events": [' "$file" + if rg -q '"events": \[\]' "$file"; then + fail "expected non-empty events response in $file" + fi +} + +require_cmd grpcurl +require_cmd rg +require_cmd xxd +require_cmd base64 +require_cmd mktemp + +if [[ -n "$DB_URL" && "$DB_URL" =~ ^postgres(ql)?:// ]]; then + DB_BACKEND="postgres" +else + DB_BACKEND="sqlite" +fi + +if [[ -z "${WORLD_ADDRESS:-}" || -z "${ENTITY_MODEL:-}" || -z "${EVENT_MESSAGE_MODEL:-}" ]]; then + if [[ "$DB_BACKEND" == "sqlite" ]]; then + require_cmd sqlite3 + [[ -f "$DB_PATH" ]] || fail "sqlite database not found at $DB_PATH" + else + require_cmd psql + [[ -n "$DB_URL" ]] || fail "postgres DB_URL is required for metadata autodiscovery" + fi +fi + +discovery_dir="$(mktemp -d)" +trap 'rm -rf "$discovery_dir"' EXIT +discovered_worlds_output="$discovery_dir/worlds_discovery.json" +grpc_call "world.World/Worlds" "{}" "$discovered_worlds_output" + +if [[ -z "${WORLD_ADDRESS:-}" ]]; then + world_address_b64="$( + rg -o '"worldAddress": "[^"]+"' "$discovered_worlds_output" \ + | rg -v '"worldAddress": "[0-9]+"' \ + | sed -E 's/"worldAddress": "([^"]+)"/\1/' \ + | while IFS= read -r candidate; do + candidate_hex="$(base64_to_felt "$candidate")" + if [[ "$candidate_hex" != "0x0" ]]; then + printf '%s\n' "$candidate" + break + fi + done + )" + [[ -n "$world_address_b64" ]] || fail "could not discover a world from live Worlds response" + WORLD_ADDRESS="$(base64_to_felt "$world_address_b64")" +fi + +[[ -n "$WORLD_ADDRESS" ]] || fail "could not discover a world address" + +ENTITY_MODEL="${ENTITY_MODEL:-$(sql_one "$(model_name_query entity)")}" +EVENT_MESSAGE_MODEL="${EVENT_MESSAGE_MODEL:-$(sql_one "$(model_name_query event_message)")}" + +[[ -n "$ENTITY_MODEL" ]] || fail "could not discover an entity model for world $WORLD_ADDRESS" +[[ -n "$EVENT_MESSAGE_MODEL" ]] || fail "could not discover an event_message model for world $WORLD_ADDRESS" + +WORLD_ADDRESS_B64="$(felt_to_base64 "$WORLD_ADDRESS")" + +tmp_dir="$(mktemp -d)" +if [[ "${KEEP_TMP:-0}" != "1" ]]; then + trap 'rm -rf "$tmp_dir" "$discovery_dir"' EXIT +else + trap 'rm -rf "$discovery_dir"' EXIT +fi + +list_output="$tmp_dir/list.txt" +describe_output="$tmp_dir/describe.txt" +worlds_output="$tmp_dir/worlds.json" +entities_output="$tmp_dir/entities.json" +event_messages_output="$tmp_dir/event_messages.json" +events_output="$tmp_dir/events.json" + +log "gRPC target: $GRPC_ADDR" +log "db backend: $DB_BACKEND" +log "world: $WORLD_ADDRESS" +log "entity model: $ENTITY_MODEL" +log "event message model: $EVENT_MESSAGE_MODEL" + +grpcurl -plaintext "$GRPC_ADDR" list >"$list_output" +assert_contains 'world.World' "$list_output" +assert_contains 'torii.Torii' "$list_output" + +grpcurl -plaintext "$GRPC_ADDR" describe world.World >"$describe_output" +assert_contains 'rpc RetrieveEntities' "$describe_output" +assert_contains 'rpc RetrieveEventMessages' "$describe_output" +assert_contains 'rpc RetrieveEvents' "$describe_output" + +grpc_call "world.World/Worlds" "{\"world_addresses\":[\"$WORLD_ADDRESS_B64\"]}" "$worlds_output" +assert_contains '"worlds": [' "$worlds_output" +assert_contains "\"worldAddress\": \"$WORLD_ADDRESS_B64\"" "$worlds_output" + +grpc_call \ + "world.World/RetrieveEntities" \ + "{\"query\":{\"world_addresses\":[\"$WORLD_ADDRESS_B64\"],\"models\":[\"$ENTITY_MODEL\"],\"pagination\":{\"limit\":$LIMIT,\"direction\":\"FORWARD\"}}}" \ + "$entities_output" +assert_non_empty_entities "$entities_output" +assert_contains "\"name\": \"$ENTITY_MODEL\"" "$entities_output" + +grpc_call \ + "world.World/RetrieveEventMessages" \ + "{\"query\":{\"world_addresses\":[\"$WORLD_ADDRESS_B64\"],\"models\":[\"$EVENT_MESSAGE_MODEL\"],\"pagination\":{\"limit\":$LIMIT,\"direction\":\"FORWARD\"}}}" \ + "$event_messages_output" +assert_non_empty_entities "$event_messages_output" +assert_contains "\"name\": \"$EVENT_MESSAGE_MODEL\"" "$event_messages_output" + +grpc_call \ + "world.World/RetrieveEvents" \ + "{\"query\":{\"pagination\":{\"limit\":$LIMIT,\"direction\":\"FORWARD\"}}}" \ + "$events_output" +assert_non_empty_events "$events_output" + +log "verification passed" +if [[ "${KEEP_TMP:-0}" == "1" ]]; then + log "artifacts kept in $tmp_dir" +fi From 93c62f019c4dcf2c44d8ba8c8cab53becd163ceb Mon Sep 17 00:00:00 2001 From: Valentin Dosimont Date: Tue, 17 Mar 2026 09:36:00 +0100 Subject: [PATCH 03/10] fix(ci): restore introspect and ecs build compatibility --- bins/torii-introspect-bin/Cargo.toml | 3 ++ bins/torii-introspect-bin/src/main.rs | 23 +++------ .../introspect-postgres-sink/src/processor.rs | 39 --------------- crates/introspect-postgres-sink/src/sink.rs | 48 ------------------- crates/torii-ecs-sink/src/grpc_service.rs | 18 +++++-- 5 files changed, 23 insertions(+), 108 deletions(-) diff --git a/bins/torii-introspect-bin/Cargo.toml b/bins/torii-introspect-bin/Cargo.toml index de096c99..f2583c14 100644 --- a/bins/torii-introspect-bin/Cargo.toml +++ b/bins/torii-introspect-bin/Cargo.toml @@ -25,6 +25,9 @@ url = "2.5" clap = { version = "4.5", features = ["derive", "env"] } anyhow = "1.0" sqlx = { version = "0.8", features = ["postgres", "sqlite", "runtime-tokio-rustls", "any"] } +tonic.workspace = true +tonic-reflection.workspace = true +tonic-web.workspace = true [lints] workspace = true diff --git a/bins/torii-introspect-bin/src/main.rs b/bins/torii-introspect-bin/src/main.rs index 4771b8df..5a3c7aab 100644 --- a/bins/torii-introspect-bin/src/main.rs +++ b/bins/torii-introspect-bin/src/main.rs @@ -6,9 +6,10 @@ use anyhow::{bail, Result}; use clap::Parser; use config::{Config, StorageBackend}; use sqlx::postgres::PgPoolOptions; -use sqlx::sqlite::SqlitePoolOptions; +use sqlx::sqlite::{SqliteConnectOptions, SqlitePoolOptions}; use starknet::core::types::Felt; use std::path::Path; +use std::str::FromStr; use std::sync::Arc; use torii::etl::decoder::DecoderId; use torii::etl::engine_db::{EngineDb, EngineDbConfig}; @@ -19,23 +20,11 @@ use torii_config_common::apply_observability_env; use torii_dojo::decoder::DojoDecoder; use torii_dojo::store::postgres::PgStore; use torii_dojo::store::sqlite::SqliteStore; +use torii_ecs_sink::proto::world::world_server::WorldServer; +use torii_ecs_sink::{EcsSink, FILE_DESCRIPTOR_SET as ECS_DESCRIPTOR_SET}; use torii_introspect_postgres_sink::processor::IntrospectPgDb; use torii_introspect_sqlite_sink::processor::IntrospectSqliteDb; -use torii_sqlite::{is_sqlite_memory_path, sqlite_connect_options}; - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -enum DbBackend { - Postgres, - Sqlite, -} - -fn detect_database_backend(database_url: &str) -> DbBackend { - if database_url.starts_with("postgres://") || database_url.starts_with("postgresql://") { - DbBackend::Postgres - } else { - DbBackend::Sqlite - } -} +use torii_sqlite::is_sqlite_memory_path; fn sqlite_connect_options(path: &str) -> Result { if path == ":memory:" || path == "sqlite::memory:" { @@ -200,7 +189,7 @@ async fn run_with_postgres( let decoder: Arc = Arc::new(decoder); - let ecs_sink = EcsSink::new(&storage_database_url, config.max_db_connections).await?; + let ecs_sink = EcsSink::new(storage_database_url, config.max_db_connections).await?; let ecs_grpc_service = ecs_sink.get_grpc_service_impl(); let reflection = tonic_reflection::server::Builder::configure() diff --git a/crates/introspect-postgres-sink/src/processor.rs b/crates/introspect-postgres-sink/src/processor.rs index 0f6589a5..053b2437 100644 --- a/crates/introspect-postgres-sink/src/processor.rs +++ b/crates/introspect-postgres-sink/src/processor.rs @@ -182,45 +182,6 @@ impl PostgresTables { Ok(()) } - pub fn prepared_insert_fields( - &self, - event: &InsertsFields, - ) -> PGSinkResult> { - let table = self.get_living_table(&event.table)?; - let schema = table.get_schema(&event.columns)?; - let table_name = table.name(); - - let mut json_writer = Vec::new(); - schema.parse_records_with_metadata( - &event.records, - &(), - &mut JsonSerializer::new(&mut json_writer), - &PostgresJsonSerializer, - )?; - let payload = unsafe { String::from_utf8_unchecked(json_writer) }; - - let mut sql_writer = Vec::new(); - write!( - sql_writer, - r#"INSERT INTO "{table_name}" - SELECT * FROM jsonb_populate_recordset(NULL::"{table_name}", $1::jsonb) - ON CONFLICT ("{}") DO "#, - schema.primary().name - )?; - if let Some((coln, cols)) = schema.columns().split_last() { - write!(&mut sql_writer, "UPDATE SET ")?; - for column in cols { - write_conflict_res::(&mut sql_writer, table_name, &column.name)?; - } - write_conflict_res::(&mut sql_writer, table_name, &coln.name)?; - } else { - write!(&mut sql_writer, "NOTHING")?; - } - - let sql = unsafe { String::from_utf8_unchecked(sql_writer) }; - Ok(Some((sql, payload))) - } - pub fn handle_message( &self, schema: &Rc, diff --git a/crates/introspect-postgres-sink/src/sink.rs b/crates/introspect-postgres-sink/src/sink.rs index ac841806..188be42e 100644 --- a/crates/introspect-postgres-sink/src/sink.rs +++ b/crates/introspect-postgres-sink/src/sink.rs @@ -14,51 +14,6 @@ use torii_postgres::PostgresConnection; pub const LOGGING_TARGET: &str = "torii::sinks::introspect"; const INTROSPECT_TYPE: TypeId = TypeId::new("introspect"); -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -enum DbBackend { - Postgres, - Sqlite, -} - -enum IntrospectDb { - Postgres(PostgresSimpleDb), - Sqlite(SqliteSimpleDb), -} - -impl IntrospectDb { - async fn initialize(&mut self) -> Result<()> { - match self { - Self::Postgres(db) => db.initialize().await.map_err(Into::into), - Self::Sqlite(db) => db.initialize().await.map_err(Into::into), - } - } - - fn has_tables(&self) -> bool { - match self { - Self::Postgres(db) => db.has_tables(), - Self::Sqlite(db) => db.has_tables(), - } - } - - async fn bootstrap_tables(&mut self, tables: &[CreateTable]) -> Result<()> { - match self { - Self::Postgres(db) => db.bootstrap_tables(tables).await.map_err(Into::into), - Self::Sqlite(db) => db.bootstrap_tables(tables).await.map_err(Into::into), - } - } - - async fn process_message( - &mut self, - msg: &IntrospectMsg, - context: &torii::etl::EventContext, - ) -> Result<()> { - match self { - Self::Postgres(db) => db.process_message(msg, context).await.map_err(Into::into), - Self::Sqlite(db) => db.process_message(msg, context).await.map_err(Into::into), - } - } -} - #[async_trait] impl Sink for IntrospectPgDb { fn name(&self) -> &'static str { @@ -140,11 +95,8 @@ impl Sink for IntrospectPgDb { self.initialize_introspect_pg_sink().await?; tracing::info!( target: LOGGING_TARGET, - backend = ?backend, "Initialized introspect sink" ); Ok(()) } } - -pub type IntrospectSqlSink = IntrospectPostgresSink; diff --git a/crates/torii-ecs-sink/src/grpc_service.rs b/crates/torii-ecs-sink/src/grpc_service.rs index f31165f7..ce10f954 100644 --- a/crates/torii-ecs-sink/src/grpc_service.rs +++ b/crates/torii-ecs-sink/src/grpc_service.rs @@ -13,13 +13,15 @@ use serde::ser::SerializeMap; use serde::Serializer; use serde_json::{Map, Serializer as JsonSerializer, Value}; use sqlx::{ - any::AnyPoolOptions, sqlite::SqliteConnectOptions, Any, ConnectOptions, Pool, QueryBuilder, Row, + any::AnyPoolOptions, postgres::PgPoolOptions, sqlite::SqliteConnectOptions, + sqlite::SqlitePoolOptions, Any, ConnectOptions, Pool, QueryBuilder, Row, }; use starknet::core::types::Felt; use tokio::sync::{mpsc, Mutex}; use tokio_stream::wrappers::ReceiverStream; use tonic::{Request, Response, Status}; -use torii_dojo::manager::{PostgresStore, SqliteStore}; +use torii_dojo::store::postgres::PgStore; +use torii_dojo::store::sqlite::SqliteStore; use torii_dojo::store::DojoStoreTrait; use torii_dojo::DojoTable; use torii_introspect::events::{CreateTable, Record, UpdateTable}; @@ -831,11 +833,19 @@ impl EcsService { async fn load_dojo_tables(&self) -> Result> { match self.state.backend { DbBackend::Sqlite => { - let store = SqliteStore::new(&self.state.database_url).await?; + let pool = SqlitePoolOptions::new() + .max_connections(1) + .connect_with(SqliteConnectOptions::from_str(&self.state.database_url)?) + .await?; + let store = SqliteStore(Arc::new(pool)); Ok(store.load_tables(&[]).await?) } DbBackend::Postgres => { - let store = PostgresStore::new(&self.state.database_url).await?; + let pool = PgPoolOptions::new() + .max_connections(1) + .connect(&self.state.database_url) + .await?; + let store = PgStore(Arc::new(pool)); Ok(store.load_tables(&[]).await?) } } From 36454dd5046b60dd00647ee0696b8eef3648cbc2 Mon Sep 17 00:00:00 2001 From: Valentin Dosimont Date: Tue, 17 Mar 2026 11:35:25 +0100 Subject: [PATCH 04/10] `feat(introspect): add ecs grpc sink` --- bins/torii-introspect-bin/src/main.rs | 21 ++++++++++ .../introspect-sqlite-sink/src/processor.rs | 16 ++++++-- src/etl/decoder/context.rs | 38 +++++++++++++++++-- 3 files changed, 67 insertions(+), 8 deletions(-) diff --git a/bins/torii-introspect-bin/src/main.rs b/bins/torii-introspect-bin/src/main.rs index 5a3c7aab..b28f116e 100644 --- a/bins/torii-introspect-bin/src/main.rs +++ b/bins/torii-introspect-bin/src/main.rs @@ -212,6 +212,7 @@ async fn run_with_postgres( .engine_database_url(engine_database_url) .with_extractor(extractor) .add_decoder(decoder) + .add_sink_boxed(Box::new(ecs_sink)) .add_sink_boxed(Box::new(IntrospectPgDb::new(pool.clone(), ()))); let decoder_id = DecoderId::new("dojo-introspect"); @@ -270,11 +271,30 @@ async fn run_with_sqlite( let decoder: Arc = Arc::new(decoder); + let ecs_sink = EcsSink::new(storage_database_url, config.max_db_connections).await?; + let ecs_grpc_service = ecs_sink.get_grpc_service_impl(); + + let reflection = tonic_reflection::server::Builder::configure() + .register_encoded_file_descriptor_set(torii::TORII_DESCRIPTOR_SET) + .register_encoded_file_descriptor_set(ECS_DESCRIPTOR_SET) + .build_v1() + .expect("failed to build ECS reflection service"); + + let grpc_router = tonic::transport::Server::builder() + .accept_http1(true) + .add_service(tonic_web::enable(WorldServer::new( + (*ecs_grpc_service).clone(), + ))) + .add_service(tonic_web::enable(reflection)); + let mut torii_config = torii::ToriiConfig::builder() .port(config.port) + .with_grpc_router(grpc_router) + .with_custom_reflection(true) .engine_database_url(engine_database_url) .with_extractor(extractor) .add_decoder(decoder) + .add_sink_boxed(Box::new(ecs_sink)) .add_sink_boxed(Box::new(IntrospectSqliteDb::new(pool.clone(), ()))); let decoder_id = DecoderId::new("dojo-introspect"); @@ -286,6 +306,7 @@ async fn run_with_sqlite( tracing::info!("Torii configured, starting ETL pipeline..."); tracing::info!("gRPC service available on port {}", config.port); tracing::info!(" - torii.Torii (core subscriptions and metrics endpoint)"); + tracing::info!(" - world.World (legacy ECS gRPC service)"); torii::run(torii_config.build()) .await diff --git a/crates/introspect-sqlite-sink/src/processor.rs b/crates/introspect-sqlite-sink/src/processor.rs index 12ff6e4a..33ecca87 100644 --- a/crates/introspect-sqlite-sink/src/processor.rs +++ b/crates/introspect-sqlite-sink/src/processor.rs @@ -1,7 +1,7 @@ use crate::json::SqliteJsonSerializer; use crate::table::{SqliteTable, SqliteTableError}; use crate::INTROSPECT_SQLITE_SINK_MIGRATIONS; -use serde_json::Value; +use serde_json::{Serializer as JsonSerializer, Value}; use starknet_types_core::felt::Felt; use std::collections::HashMap; use std::fmt::Display; @@ -269,10 +269,18 @@ impl IntrospectSqliteDb { update_columns ); + let mut bytes = Vec::new(); + let mut serializer = JsonSerializer::new(&mut bytes); + record_schema.parse_records_with_metadata( + &event.records, + &(), + &mut serializer, + &SqliteJsonSerializer, + )?; + let rows = serde_json::from_slice::>(&bytes)?; + let mut tx = self.begin().await?; - for record in &event.records { - let value = - serde_json::to_value(record_schema.to_frame(record, &(), &SqliteJsonSerializer))?; + for value in rows { let object = value.as_object().ok_or(SqliteDbError::InvalidRecordFrame)?; let mut query = sqlx::query(&sql); diff --git a/src/etl/decoder/context.rs b/src/etl/decoder/context.rs index 5732db39..d5f592ac 100644 --- a/src/etl/decoder/context.rs +++ b/src/etl/decoder/context.rs @@ -22,6 +22,34 @@ use super::{ContractFilter, Decoder, DecoderId}; use crate::etl::engine_db::EngineDb; use crate::etl::envelope::Envelope; +fn felt_to_utf8_lossy(felt: &Felt) -> String { + let bytes = felt.to_bytes_be(); + let start = bytes.iter().position(|b| *b != 0).unwrap_or(bytes.len()); + String::from_utf8_lossy(&bytes[start..]).into_owned() +} + +fn event_preview(event: &EmittedEvent) -> String { + let key0 = event.keys.first(); + let data0 = event.data.first(); + let key0_hex = key0 + .map(|k| format!("{k:#x}")) + .unwrap_or_else(|| "".to_string()); + let key0_utf8 = key0 + .map(felt_to_utf8_lossy) + .unwrap_or_else(|| "".to_string()); + let data0_hex = data0 + .map(|d| format!("{d:#x}")) + .unwrap_or_else(|| "".to_string()); + let data0_utf8 = data0 + .map(felt_to_utf8_lossy) + .unwrap_or_else(|| "".to_string()); + + format!( + "contract={:#x} tx={:#x} key0_hex={} key0_utf8={:?} data0_hex={} data0_utf8={:?}", + event.from_address, event.transaction_hash, key0_hex, key0_utf8, data0_hex, data0_utf8 + ) +} + /// DecoderContext manages multiple decoders with contract filtering. /// /// Routes events to decoders based on: @@ -237,9 +265,10 @@ impl DecoderContext { Err(e) => { tracing::warn!( target: "torii::etl::decoder_context", - "Decoder '{}' failed: {}", + "Decoder '{}' failed: {} | {}", decoder.decoder_name(), - e + e, + event_preview(event) ); } } @@ -280,9 +309,10 @@ impl DecoderContext { Err(e) => { tracing::warn!( target: "torii::etl::decoder_context", - "Decoder '{}' failed: {}", + "Decoder '{}' failed: {} | {}", decoder.decoder_name(), - e + e, + event_preview(event) ); } } From 20fd690823550f355c8c9a41f060a920ef5c88b0 Mon Sep 17 00:00:00 2001 From: Valentin Dosimont Date: Tue, 17 Mar 2026 17:07:54 +0100 Subject: [PATCH 05/10] feat(introspect): persist schema state --- bins/torii-introspect-bin/src/config.rs | 3 - bins/torii-introspect-bin/src/main.rs | 48 +-- .../migrations/002_schema_state.sql | 6 + .../introspect-postgres-sink/src/processor.rs | 129 ++---- .../migrations/002_schema_state.sql | 6 + .../introspect-sqlite-sink/src/processor.rs | 92 ++++- crates/torii-ecs-sink/src/grpc_service.rs | 377 +++++++++++++++++- scripts/test-ecs-grpc-e2e.sh | 24 +- 8 files changed, 519 insertions(+), 166 deletions(-) create mode 100644 crates/introspect-postgres-sink/migrations/002_schema_state.sql create mode 100644 crates/introspect-sqlite-sink/migrations/002_schema_state.sql diff --git a/bins/torii-introspect-bin/src/config.rs b/bins/torii-introspect-bin/src/config.rs index 784824b1..e3cf6f0d 100644 --- a/bins/torii-introspect-bin/src/config.rs +++ b/bins/torii-introspect-bin/src/config.rs @@ -81,9 +81,6 @@ pub struct Config { pub max_db_connections: Option, /// Ignore persisted extractor state and force extraction from `from_block`. - /// - /// This binary no longer supports historical schema bootstrap, so resumed - /// runs must opt into a fresh extraction window explicitly. #[arg(long)] pub ignore_saved_state: bool, } diff --git a/bins/torii-introspect-bin/src/main.rs b/bins/torii-introspect-bin/src/main.rs index b28f116e..09e0ddf9 100644 --- a/bins/torii-introspect-bin/src/main.rs +++ b/bins/torii-introspect-bin/src/main.rs @@ -2,7 +2,7 @@ mod config; -use anyhow::{bail, Result}; +use anyhow::Result; use clap::Parser; use config::{Config, StorageBackend}; use sqlx::postgres::PgPoolOptions; @@ -12,7 +12,6 @@ use std::path::Path; use std::str::FromStr; use std::sync::Arc; use torii::etl::decoder::DecoderId; -use torii::etl::engine_db::{EngineDb, EngineDbConfig}; use torii::etl::extractor::{ ContractEventConfig, EventExtractor, EventExtractorConfig, RetryPolicy, }; @@ -88,7 +87,7 @@ async fn run_indexer(config: Config) -> Result<()> { if config.ignore_saved_state { "ignoring persisted extractor state" } else { - "failing closed if persisted extractor state exists" + "resuming from persisted extractor state when available" } ); tracing::info!( @@ -130,12 +129,6 @@ async fn run_indexer(config: Config) -> Result<()> { tokio::fs::create_dir_all(db_dir).await?; } - let engine_db = EngineDb::new(EngineDbConfig { - path: engine_database_url.clone(), - }) - .await?; - ensure_no_saved_state(&engine_db, &contracts, config.ignore_saved_state).await?; - match backend { StorageBackend::Postgres => { run_with_postgres( @@ -313,40 +306,3 @@ async fn run_with_sqlite( .map_err(|e| anyhow::anyhow!("Torii error: {e}"))?; Ok(()) } - -async fn ensure_no_saved_state( - engine_db: &EngineDb, - contracts: &[Felt], - ignore_saved_state: bool, -) -> Result<()> { - if ignore_saved_state { - return Ok(()); - } - - let mut owners_with_saved_state = Vec::new(); - for owner in contracts { - let state_key = format!("{owner:#x}"); - if engine_db - .get_extractor_state("event", &state_key) - .await? - .is_some() - { - owners_with_saved_state.push(*owner); - } - } - - if owners_with_saved_state.is_empty() { - return Ok(()); - } - - bail!( - "persisted extractor state exists for contract(s): {}. \ - Historical schema bootstrap is not supported on this branch; \ - re-run with --ignore-saved-state or use a fresh database", - owners_with_saved_state - .iter() - .map(|owner| format!("{owner:#x}")) - .collect::>() - .join(", ") - ) -} diff --git a/crates/introspect-postgres-sink/migrations/002_schema_state.sql b/crates/introspect-postgres-sink/migrations/002_schema_state.sql new file mode 100644 index 00000000..21dcbf3b --- /dev/null +++ b/crates/introspect-postgres-sink/migrations/002_schema_state.sql @@ -0,0 +1,6 @@ +CREATE TABLE IF NOT EXISTS introspect.introspect_sink_schema_state ( + table_id TEXT PRIMARY KEY, + table_schema_json TEXT NOT NULL, + alive BOOLEAN NOT NULL DEFAULT TRUE, + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); diff --git a/crates/introspect-postgres-sink/src/processor.rs b/crates/introspect-postgres-sink/src/processor.rs index 053b2437..d34aa2c0 100644 --- a/crates/introspect-postgres-sink/src/processor.rs +++ b/crates/introspect-postgres-sink/src/processor.rs @@ -1,11 +1,12 @@ use crate::json::PostgresJsonSerializer; use crate::query::CreatePgTable; +use crate::sql::write_conflict_res; use crate::table::{DeadField, PgTable}; use crate::{PgDbError, PgDbResult, PgSchema, INTROSPECT_PG_SINK_MIGRATIONS}; use introspect_types::{ColumnInfo, ResultInto, TypeDef}; use serde_json::Serializer as JsonSerializer; use sqlx::types::Json; -use sqlx::{FromRow, PgPool}; +use sqlx::{FromRow, PgPool, Row}; use starknet_types_core::felt::Felt; use std::collections::HashMap; use std::io::Write; @@ -330,6 +331,30 @@ impl IntrospectPgDb { .handle_message(&schema, msg, metadata, &mut queries)?; } self.execute_queries(&queries).await?; + + match msg { + IntrospectMsg::CreateTable(event) => { + self.persist_table_state(&event.clone().into(), true) + .await?; + } + IntrospectMsg::UpdateTable(event) => { + self.persist_alive_state(event.id, false).await?; + } + IntrospectMsg::AddColumns(event) => { + self.persist_alive_state(event.table, false).await?; + } + IntrospectMsg::DropColumns(event) => { + self.persist_alive_state(event.table, false).await?; + } + IntrospectMsg::RetypeColumns(event) => { + self.persist_alive_state(event.table, false).await?; + } + IntrospectMsg::RetypePrimary(event) => { + self.persist_alive_state(event.table, false).await?; + } + _ => {} + } + Ok(()) } @@ -365,105 +390,3 @@ pub struct MessageWithContext<'a, M> { pub msg: &'a M, pub context: &'a MetaData, } - -// #[cfg(test)] -// mod tests { -// use super::*; -// use introspect_types::{ColumnDef, PrimaryDef, PrimaryTypeDef, TypeDef}; -// use torii_introspect::{CreateTable, UpdateTable}; - -// fn primary() -> PrimaryDef { -// PrimaryDef { -// name: "entity_id".to_string(), -// attributes: vec![], -// type_def: PrimaryTypeDef::Felt252, -// } -// } - -// fn column(id: u64, name: &str) -> ColumnDef { -// ColumnDef { -// id: Felt::from(id), -// name: name.to_string(), -// attributes: vec![], -// type_def: TypeDef::U32, -// } -// } - -// fn create_table(id: u64, columns: Vec) -> CreateTable { -// CreateTable { -// id: Felt::from(id), -// name: "duel-state".to_string(), -// attributes: vec![], -// primary: primary(), -// columns, -// } -// } - -// fn update_table(id: u64, columns: Vec) -> UpdateTable { -// UpdateTable { -// id: Felt::from(id), -// name: "duel-state".to_string(), -// attributes: vec![], -// primary: primary(), -// columns, -// } -// } - -// // #[test] -// // fn update_table_creates_unknown_table() { -// // let mut tables = PostgresTables::default(); -// // let mut queries = Vec::new(); -// // let schema = PgSchema::Public; -// // tables -// // .update_table( -// // &schema, -// // update_table(1, vec![column(10, "duelist_count")]), -// // &mut queries, -// // ) -// // .unwrap(); - -// // assert!(!queries.is_empty()); -// // assert!(queries -// // .iter() -// // .any(|query| query.contains("CREATE TABLE IF NOT EXISTS"))); -// // assert!(tables.read().unwrap().contains_key(&Felt::from(1_u64))); -// // } - -// // #[test] -// // fn update_table_emits_alter_queries_for_new_columns() { -// // let mut tables = PostgresTables::default(); -// // let mut create_queries = Vec::new(); -// // let schema = PgSchema::Public; -// // tables -// // .create_table( -// // &schema, -// // create_table(1, vec![column(10, "duelist_count")]), -// // &mut create_queries, -// // ) -// // .unwrap(); - -// // let mut update_queries = Vec::new(); -// // tables -// // .update_table( -// // &schema, -// // update_table( -// // 1, -// // vec![ -// // column(10, "duelist_count"), -// // column(11, "alive_duelist_count"), -// // ], -// // ), -// // &EventContext::default(), -// // &mut update_queries, -// // ) -// // .unwrap(); - -// // assert!(update_queries -// // .iter() -// // .any(|query| query.contains("ALTER TABLE") && query.contains("alive_duelist_count"))); -// // assert!(tables.read().unwrap()[&Felt::from(1_u64)] -// // .columns -// // .values() -// // .any(|column| column.name == "alive_duelist_count")); -// // } -// } diff --git a/crates/introspect-sqlite-sink/migrations/002_schema_state.sql b/crates/introspect-sqlite-sink/migrations/002_schema_state.sql new file mode 100644 index 00000000..caa564d4 --- /dev/null +++ b/crates/introspect-sqlite-sink/migrations/002_schema_state.sql @@ -0,0 +1,6 @@ +CREATE TABLE IF NOT EXISTS introspect_sink_schema_state ( + table_id TEXT PRIMARY KEY, + table_schema_json TEXT NOT NULL, + alive INTEGER NOT NULL DEFAULT 1, + updated_at INTEGER NOT NULL DEFAULT (unixepoch()) +); diff --git a/crates/introspect-sqlite-sink/src/processor.rs b/crates/introspect-sqlite-sink/src/processor.rs index 33ecca87..8b86a2b1 100644 --- a/crates/introspect-sqlite-sink/src/processor.rs +++ b/crates/introspect-sqlite-sink/src/processor.rs @@ -2,6 +2,7 @@ use crate::json::SqliteJsonSerializer; use crate::table::{SqliteTable, SqliteTableError}; use crate::INTROSPECT_SQLITE_SINK_MIGRATIONS; use serde_json::{Serializer as JsonSerializer, Value}; +use sqlx::Row; use starknet_types_core::felt::Felt; use std::collections::HashMap; use std::fmt::Display; @@ -181,6 +182,70 @@ impl IntrospectSqliteDb { pub async fn initialize_introspect_sqlite_sink(&self) -> SqliteDbResult<()> { self.migrate(Some("introspect"), INTROSPECT_SQLITE_SINK_MIGRATIONS) .await?; + self.load_persisted_state().await?; + Ok(()) + } + + async fn load_persisted_state(&self) -> SqliteDbResult<()> { + let rows = sqlx::query( + r#" + SELECT table_schema_json, alive + FROM introspect_sink_schema_state + ORDER BY updated_at ASC + "#, + ) + .fetch_all(self.pool()) + .await?; + + let mut tables = self.tables.write()?; + for row in rows { + let schema_json: String = row.try_get("table_schema_json")?; + let alive: i64 = row.try_get("alive")?; + let table_schema: TableSchema = serde_json::from_str(&schema_json)?; + let (id, mut table) = + SqliteTable::new_from_table(self.namespace.prefix(), table_schema); + table.alive = alive != 0; + tables.insert(id, table); + } + + Ok(()) + } + + async fn persist_table_state(&self, table: &TableSchema, alive: bool) -> SqliteDbResult<()> { + let schema_json = serde_json::to_string(table)?; + let alive = i64::from(alive); + sqlx::query( + r#" + INSERT INTO introspect_sink_schema_state (table_id, table_schema_json, alive, updated_at) + VALUES (?1, ?2, ?3, unixepoch()) + ON CONFLICT (table_id) + DO UPDATE SET + table_schema_json = excluded.table_schema_json, + alive = excluded.alive, + updated_at = unixepoch() + "#, + ) + .bind(format!("{:#x}", table.id)) + .bind(schema_json) + .bind(alive) + .execute(self.pool()) + .await?; + Ok(()) + } + + async fn persist_alive_state(&self, table_id: Felt, alive: bool) -> SqliteDbResult<()> { + let alive = i64::from(alive); + sqlx::query( + r#" + UPDATE introspect_sink_schema_state + SET alive = ?1, updated_at = unixepoch() + WHERE table_id = ?2 + "#, + ) + .bind(alive) + .bind(format!("{table_id:#x}")) + .execute(self.pool()) + .await?; Ok(()) } @@ -202,13 +267,30 @@ impl IntrospectSqliteDb { IntrospectMsg::CreateTable(event) => { let (_, query) = self.tables.create_table(&self.namespace, event.clone())?; self.execute_queries(&[query]).await?; + self.persist_table_state(&event.clone().into(), true) + .await?; Ok(()) } - IntrospectMsg::UpdateTable(event) => self.tables.set_table_dead(&event.id), - IntrospectMsg::AddColumns(event) => self.tables.set_table_dead(&event.table), - IntrospectMsg::DropColumns(event) => self.tables.set_table_dead(&event.table), - IntrospectMsg::RetypeColumns(event) => self.tables.set_table_dead(&event.table), - IntrospectMsg::RetypePrimary(event) => self.tables.set_table_dead(&event.table), + IntrospectMsg::UpdateTable(event) => { + self.tables.set_table_dead(&event.id)?; + self.persist_alive_state(event.id, false).await + } + IntrospectMsg::AddColumns(event) => { + self.tables.set_table_dead(&event.table)?; + self.persist_alive_state(event.table, false).await + } + IntrospectMsg::DropColumns(event) => { + self.tables.set_table_dead(&event.table)?; + self.persist_alive_state(event.table, false).await + } + IntrospectMsg::RetypeColumns(event) => { + self.tables.set_table_dead(&event.table)?; + self.persist_alive_state(event.table, false).await + } + IntrospectMsg::RetypePrimary(event) => { + self.tables.set_table_dead(&event.table)?; + self.persist_alive_state(event.table, false).await + } IntrospectMsg::RenameTable(_) | IntrospectMsg::DropTable(_) | IntrospectMsg::RenameColumns(_) diff --git a/crates/torii-ecs-sink/src/grpc_service.rs b/crates/torii-ecs-sink/src/grpc_service.rs index ce10f954..80262010 100644 --- a/crates/torii-ecs-sink/src/grpc_service.rs +++ b/crates/torii-ecs-sink/src/grpc_service.rs @@ -2293,8 +2293,17 @@ fn value_as_bytes(value: &Value) -> Vec { } fn value_as_felt_bytes(value: &Value) -> Result> { + if value.is_null() { + return Ok(Felt::ZERO.to_bytes_be().to_vec()); + } + let string = value_as_string(value); - Ok(felt_from_hex(&string)?.to_bytes_be().to_vec()) + let trimmed = string.trim(); + if trimmed.is_empty() { + return Ok(Felt::ZERO.to_bytes_be().to_vec()); + } + + Ok(felt_from_hex(trimmed)?.to_bytes_be().to_vec()) } fn value_to_primary_felt(value: &Value, type_def: &PrimaryTypeDef) -> Result { @@ -2550,3 +2559,369 @@ fn felt_from_hex(value: &str) -> Result { fn felt_from_bytes(value: &[u8]) -> Result { Ok(Felt::from_bytes_be_slice(value)) } + +#[cfg(test)] +mod tests { + use super::*; + use introspect_types::{ColumnDef, PrimaryDef, PrimaryTypeDef, TypeDef}; + use tokio::time::{timeout, Duration}; + use tokio_stream::StreamExt; + + fn test_db_path(name: &str) -> String { + let nonce = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .expect("clock before epoch") + .as_nanos(); + format!("sqlite:file:torii-ecs-sink-{name}-{nonce}?mode=memory&cache=shared") + } + + fn member_bool_clause(model: &str, member: &str, value: bool) -> types::Clause { + types::Clause { + clause_type: Some(ClauseType::Member(types::MemberClause { + model: model.to_string(), + member: member.to_string(), + operator: ComparisonOperator::Eq as i32, + value: Some(types::MemberValue { + value_type: Some(ValueType::Primitive(types::Primitive { + primitive_type: Some(types::primitive::PrimitiveType::Bool(value)), + })), + }), + })), + } + } + + async fn seed_entity( + service: &EcsService, + kind: TableKind, + world_address: Felt, + table_id: Felt, + table_name: &str, + entity_id: Felt, + open: bool, + ) { + sqlx::query( + "CREATE TABLE IF NOT EXISTS dojo_tables ( + owner BLOB NOT NULL, + id BLOB NOT NULL, + name TEXT NOT NULL, + attributes TEXT NOT NULL, + keys_json TEXT NOT NULL, + values_json TEXT NOT NULL, + legacy INTEGER NOT NULL, + created_at INTEGER, + updated_at INTEGER, + created_block INTEGER, + updated_block INTEGER, + created_tx BLOB, + updated_tx BLOB, + PRIMARY KEY(owner, id) + )", + ) + .execute(&service.state.pool) + .await + .expect("create dojo_tables"); + sqlx::query( + "CREATE TABLE IF NOT EXISTS dojo_columns ( + owner BLOB NOT NULL, + table_id BLOB NOT NULL, + id BLOB NOT NULL, + payload TEXT NOT NULL, + PRIMARY KEY(owner, table_id, id) + )", + ) + .execute(&service.state.pool) + .await + .expect("create dojo_columns"); + + let table = CreateTable { + id: table_id, + name: table_name.to_string(), + attributes: vec![], + primary: PrimaryDef { + name: "entity_id".to_string(), + attributes: vec![], + type_def: PrimaryTypeDef::Felt252, + }, + columns: vec![ColumnDef { + id: Felt::from(1_u64), + name: "open".to_string(), + attributes: vec![], + type_def: TypeDef::Bool, + }], + }; + + service.cache_created_table(world_address, &table).await; + service + .record_table_kind(world_address, table_id, kind) + .await + .expect("record table kind"); + service + .upsert_entity_meta(kind, world_address, table_id, entity_id, 1, false) + .await + .expect("upsert entity meta"); + + let row_json = serde_json::json!({ "open": open }).to_string(); + sqlx::query( + "INSERT INTO torii_ecs_entity_models ( + kind, world_address, table_id, entity_id, row_json, updated_at + ) VALUES (?1, ?2, ?3, ?4, ?5, ?6) + ON CONFLICT(kind, world_address, table_id, entity_id) DO UPDATE SET + row_json = excluded.row_json, + updated_at = excluded.updated_at", + ) + .bind(kind.as_str()) + .bind(felt_hex(world_address)) + .bind(felt_hex(table_id)) + .bind(felt_hex(entity_id)) + .bind(row_json) + .bind(1_i64) + .execute(&service.state.pool) + .await + .expect("insert entity model"); + } + + #[tokio::test] + async fn subscribe_entities_update_flow() { + let db_path = test_db_path("entities-sub"); + let service = EcsService::new(&db_path, Some(1)) + .await + .expect("service init"); + let world = Felt::from(10_u64); + let table = Felt::from(20_u64); + let entity = Felt::from(30_u64); + seed_entity( + &service, + TableKind::Entity, + world, + table, + "test-Lobby", + entity, + true, + ) + .await; + + let response = service + .subscribe_entities(Request::new(SubscribeEntitiesRequest { + clause: Some(member_bool_clause("test-Lobby", "open", true)), + world_addresses: vec![world.to_bytes_be().to_vec()], + })) + .await + .expect("subscribe entities"); + let mut stream = response.into_inner(); + + let setup = timeout(Duration::from_secs(1), stream.next()) + .await + .expect("setup timeout") + .expect("setup frame") + .expect("setup ok"); + let subscription_id = setup.subscription_id; + assert!(setup.entity.is_none()); + + service + .publish_entity_update(TableKind::Entity, world, entity) + .await + .expect("publish entity update"); + let matched = timeout(Duration::from_secs(1), stream.next()) + .await + .expect("entity timeout") + .expect("entity frame") + .expect("entity ok"); + assert_eq!(matched.subscription_id, subscription_id); + let matched_entity = matched.entity.expect("entity payload"); + assert_eq!(matched_entity.world_address, world.to_bytes_be().to_vec()); + + service + .update_entities_subscription(Request::new(UpdateEntitiesSubscriptionRequest { + subscription_id, + clause: Some(member_bool_clause("test-Lobby", "open", false)), + world_addresses: vec![world.to_bytes_be().to_vec()], + })) + .await + .expect("update entities subscription"); + + service + .publish_entity_update(TableKind::Entity, world, entity) + .await + .expect("publish filtered entity update"); + let filtered = timeout(Duration::from_millis(150), stream.next()).await; + assert!(filtered.is_err(), "entity should be filtered out"); + } + + #[tokio::test] + async fn subscribe_event_messages_update_flow() { + let db_path = test_db_path("event-messages-sub"); + let service = EcsService::new(&db_path, Some(1)) + .await + .expect("service init"); + let world = Felt::from(101_u64); + let table = Felt::from(201_u64); + let entity = Felt::from(301_u64); + seed_entity( + &service, + TableKind::EventMessage, + world, + table, + "test-EventMessage", + entity, + true, + ) + .await; + + let response = service + .subscribe_event_messages(Request::new(SubscribeEntitiesRequest { + clause: Some(member_bool_clause("test-EventMessage", "open", true)), + world_addresses: vec![world.to_bytes_be().to_vec()], + })) + .await + .expect("subscribe event messages"); + let mut stream = response.into_inner(); + + let setup = timeout(Duration::from_secs(1), stream.next()) + .await + .expect("setup timeout") + .expect("setup frame") + .expect("setup ok"); + let subscription_id = setup.subscription_id; + assert!(setup.entity.is_none()); + + service + .publish_entity_update(TableKind::EventMessage, world, entity) + .await + .expect("publish event message update"); + let matched = timeout(Duration::from_secs(1), stream.next()) + .await + .expect("event message timeout") + .expect("event message frame") + .expect("event message ok"); + assert_eq!(matched.subscription_id, subscription_id); + assert!(matched.entity.is_some()); + + service + .update_event_messages_subscription(Request::new(UpdateEntitiesSubscriptionRequest { + subscription_id, + clause: Some(member_bool_clause("test-EventMessage", "open", false)), + world_addresses: vec![world.to_bytes_be().to_vec()], + })) + .await + .expect("update event messages subscription"); + + service + .publish_entity_update(TableKind::EventMessage, world, entity) + .await + .expect("publish filtered event message update"); + let filtered = timeout(Duration::from_millis(150), stream.next()).await; + assert!(filtered.is_err(), "event message should be filtered out"); + } + + #[tokio::test] + async fn subscribe_events_keys_filter() { + let db_path = test_db_path("events-sub"); + let service = EcsService::new(&db_path, Some(1)) + .await + .expect("service init"); + let world = Felt::from(1_u64); + let tx1 = Felt::from(2_u64); + let tx2 = Felt::from(3_u64); + let key_match = Felt::from(444_u64); + + let response = service + .subscribe_events(Request::new(SubscribeEventsRequest { + keys: vec![types::KeysClause { + keys: vec![key_match.to_bytes_be().to_vec()], + pattern_matching: PatternMatching::VariableLen as i32, + models: vec![], + }], + })) + .await + .expect("subscribe events"); + let mut stream = response.into_inner(); + + let setup = timeout(Duration::from_secs(1), stream.next()) + .await + .expect("setup timeout") + .expect("setup frame") + .expect("setup ok"); + assert!(setup.event.is_none()); + + service + .store_event( + world, + tx1, + 1, + 1, + &[Felt::from(999_u64)], + &[Felt::from(7_u64)], + 0, + ) + .await + .expect("store non matching event"); + let no_match = timeout(Duration::from_millis(150), stream.next()).await; + assert!( + no_match.is_err(), + "non matching event should be filtered out" + ); + + service + .store_event(world, tx2, 1, 1, &[key_match, Felt::from(123_u64)], &[], 1) + .await + .expect("store matching event"); + let matched = timeout(Duration::from_secs(1), stream.next()) + .await + .expect("matched timeout") + .expect("matched frame") + .expect("matched ok"); + let event = matched.event.expect("event payload"); + assert_eq!(event.transaction_hash, tx2.to_bytes_be().to_vec()); + assert_eq!( + event.keys.first().cloned(), + Some(key_match.to_bytes_be().to_vec()) + ); + } + + #[tokio::test] + async fn subscribe_contracts_query_filter() { + let db_path = test_db_path("contracts-sub"); + let service = EcsService::new(&db_path, Some(1)) + .await + .expect("service init"); + let world_contract = Felt::from(1111_u64); + let other_contract = Felt::from(2222_u64); + + let response = service + .subscribe_contracts(Request::new(SubscribeContractsRequest { + query: Some(types::ContractQuery { + contract_addresses: vec![world_contract.to_bytes_be().to_vec()], + contract_types: vec![ContractType::World as i32], + }), + })) + .await + .expect("subscribe contracts"); + let mut stream = response.into_inner(); + + service + .record_contract_progress(other_contract, ContractType::World, 7, 77, None) + .await + .expect("record other contract progress"); + let no_match = timeout(Duration::from_millis(150), stream.next()).await; + assert!( + no_match.is_err(), + "non matching contract should be filtered out" + ); + + service + .record_contract_progress(world_contract, ContractType::World, 9, 99, None) + .await + .expect("record world contract progress"); + let matched = timeout(Duration::from_secs(1), stream.next()) + .await + .expect("matched timeout") + .expect("matched frame") + .expect("matched ok"); + let contract = matched.contract.expect("contract payload"); + assert_eq!( + contract.contract_address, + world_contract.to_bytes_be().to_vec() + ); + assert_eq!(contract.contract_type, ContractType::World as i32); + } +} diff --git a/scripts/test-ecs-grpc-e2e.sh b/scripts/test-ecs-grpc-e2e.sh index 241983af..1ef12ce4 100755 --- a/scripts/test-ecs-grpc-e2e.sh +++ b/scripts/test-ecs-grpc-e2e.sh @@ -9,6 +9,7 @@ GRPC_ADDR="${GRPC_ADDR:-localhost:3000}" DB_PATH="${DB_PATH:-${REPO_ROOT}/torii-data/introspect.db}" DB_URL="${DB_URL:-${DATABASE_URL:-}}" LIMIT="${LIMIT:-1}" +RUN_SUBSCRIPTIONS_SMOKE="${RUN_SUBSCRIPTIONS_SMOKE:-1}" log() { printf '[ecs-e2e] %s\n' "$*" @@ -43,27 +44,27 @@ model_name_query() { case "$DB_BACKEND" in sqlite) cat <> 'name' +SELECT m.name FROM torii_ecs_table_kinds k -JOIN torii_dojo_manager_state m +JOIN dojo_tables m ON ltrim(lower(encode(m.owner, 'hex')), '0') = ltrim(substr(k.world_address, 3), '0') - AND ltrim(lower(encode(m.table_id, 'hex')), '0') = ltrim(substr(k.table_id, 3), '0') + AND ltrim(lower(encode(m.id, 'hex')), '0') = ltrim(substr(k.table_id, 3), '0') WHERE k.world_address = '$WORLD_ADDRESS' AND k.kind = '$kind' -ORDER BY m.table_json::json ->> 'name' +ORDER BY m.name LIMIT 1; EOF ;; @@ -132,6 +133,7 @@ require_cmd rg require_cmd xxd require_cmd base64 require_cmd mktemp +require_cmd python3 if [[ -n "$DB_URL" && "$DB_URL" =~ ^postgres(ql)?:// ]]; then DB_BACKEND="postgres" @@ -235,6 +237,12 @@ grpc_call \ assert_non_empty_events "$events_output" log "verification passed" + +if [[ "$RUN_SUBSCRIPTIONS_SMOKE" == "1" ]]; then + log "running subscriptions smoke checks" + GRPC_ADDR="$GRPC_ADDR" python3 "$REPO_ROOT/scripts/test-ecs-subscriptions-smoke.py" +fi + if [[ "${KEEP_TMP:-0}" == "1" ]]; then log "artifacts kept in $tmp_dir" fi From 3568339f36b961c877ff2c678ecf689097c3ea75 Mon Sep 17 00:00:00 2001 From: Valentin Dosimont Date: Tue, 17 Mar 2026 20:31:29 +0100 Subject: [PATCH 06/10] perf(ecs-sink): add metrics and query pushdown --- Cargo.lock | 1 + crates/torii-ecs-sink/Cargo.toml | 1 + crates/torii-ecs-sink/src/grpc_service.rs | 588 +++++++++++++++++----- 3 files changed, 461 insertions(+), 129 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3bb64755..a139cdb7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4694,6 +4694,7 @@ dependencies = [ "futures-util", "hex", "introspect-types", + "metrics", "prost 0.13.5", "prost-types 0.13.5", "serde", diff --git a/crates/torii-ecs-sink/Cargo.toml b/crates/torii-ecs-sink/Cargo.toml index 03760390..0d402e5f 100644 --- a/crates/torii-ecs-sink/Cargo.toml +++ b/crates/torii-ecs-sink/Cargo.toml @@ -25,6 +25,7 @@ tokio.workspace = true tokio-stream.workspace = true tonic.workspace = true tracing.workspace = true +metrics.workspace = true hex.workspace = true [build-dependencies] diff --git a/crates/torii-ecs-sink/src/grpc_service.rs b/crates/torii-ecs-sink/src/grpc_service.rs index 80262010..834bf928 100644 --- a/crates/torii-ecs-sink/src/grpc_service.rs +++ b/crates/torii-ecs-sink/src/grpc_service.rs @@ -1,6 +1,7 @@ use std::collections::{HashMap, HashSet}; use std::str::FromStr; use std::sync::Arc; +use std::time::Instant; use anyhow::{anyhow, Result}; use chrono::Utc; @@ -17,6 +18,7 @@ use sqlx::{ sqlite::SqlitePoolOptions, Any, ConnectOptions, Pool, QueryBuilder, Row, }; use starknet::core::types::Felt; +use tokio::sync::mpsc::error::TrySendError; use tokio::sync::{mpsc, Mutex}; use tokio_stream::wrappers::ReceiverStream; use tonic::{Request, Response, Status}; @@ -86,7 +88,7 @@ struct EcsState { pool: Pool, backend: DbBackend, database_url: String, - managed_tables: Mutex>>, + managed_tables: Mutex>>>, entity_subscriptions: Mutex>, event_message_subscriptions: Mutex>, event_subscriptions: Mutex>, @@ -95,7 +97,7 @@ struct EcsState { struct EntitySubscription { clause: Option, - world_addresses: Vec>, + world_addresses: HashSet>, sender: mpsc::Sender>, } @@ -275,6 +277,13 @@ impl EcsService { .execute(&self.state.pool) .await?; + sqlx::query( + "CREATE INDEX IF NOT EXISTS torii_ecs_entity_meta_table_lookup_idx + ON torii_ecs_entity_meta(kind, table_id, world_address, entity_id, deleted)", + ) + .execute(&self.state.pool) + .await?; + sqlx::query( "CREATE INDEX IF NOT EXISTS torii_ecs_entity_models_lookup_idx ON torii_ecs_entity_models(kind, world_address, entity_id)", @@ -282,6 +291,20 @@ impl EcsService { .execute(&self.state.pool) .await?; + sqlx::query( + "CREATE INDEX IF NOT EXISTS torii_ecs_entity_models_table_lookup_idx + ON torii_ecs_entity_models(kind, table_id, world_address, entity_id)", + ) + .execute(&self.state.pool) + .await?; + + sqlx::query( + "CREATE INDEX IF NOT EXISTS torii_ecs_events_cursor_idx + ON torii_ecs_events(event_id)", + ) + .execute(&self.state.pool) + .await?; + Ok(()) } @@ -607,6 +630,7 @@ impl EcsService { world_address: Felt, entity_id: Felt, ) -> Result<()> { + let start = Instant::now(); let subscriptions = match kind { TableKind::Entity => &self.state.entity_subscriptions, TableKind::EventMessage => &self.state.event_message_subscriptions, @@ -622,61 +646,183 @@ impl EcsService { let Some(entity) = entity else { return Ok(()); }; - let subscriptions = subscriptions.lock().await; - for (&subscription_id, subscription) in subscriptions.iter() { - if !subscription.world_addresses.is_empty() - && !subscription.world_addresses.contains(&entity.world_address) - { - continue; - } - if let Some(clause) = &subscription.clause { - if !entity_matches_clause(&entity, clause) { - continue; - } - } - let _ = subscription.sender.try_send(Ok(SubscribeEntityResponse { + + let (checked, targets) = { + let subscriptions = subscriptions.lock().await; + let checked = subscriptions.len(); + let targets = subscriptions + .iter() + .filter_map(|(&subscription_id, subscription)| { + if !subscription.world_addresses.is_empty() + && !subscription.world_addresses.contains(&entity.world_address) + { + return None; + } + if let Some(clause) = &subscription.clause { + if !entity_matches_clause(&entity, clause) { + return None; + } + } + Some((subscription_id, subscription.sender.clone())) + }) + .collect::>(); + (checked, targets) + }; + let matched = targets.len(); + + let mut closed = Vec::new(); + for (subscription_id, sender) in targets { + if let Err(err) = sender.try_send(Ok(SubscribeEntityResponse { entity: Some(entity.clone()), subscription_id, - })); + })) { + if matches!(err, TrySendError::Closed(_)) { + closed.push(subscription_id); + } + } } + + if !closed.is_empty() { + let mut subscriptions = subscriptions.lock().await; + for subscription_id in closed { + subscriptions.remove(&subscription_id); + } + } + ::metrics::histogram!("torii_ecs_publish_entity_update_seconds") + .record(start.elapsed().as_secs_f64()); + ::metrics::counter!("torii_ecs_publish_entity_subscribers_checked_total") + .increment(checked as u64); + ::metrics::counter!("torii_ecs_publish_entity_subscribers_matched_total") + .increment(matched as u64); Ok(()) } async fn publish_event_update(&self, event: types::Event) { - let subscriptions = self.state.event_subscriptions.lock().await; - if subscriptions.is_empty() { - return; - } - for subscription in subscriptions.values() { - if !match_keys(&event.keys, &subscription.keys) { - continue; + let start = Instant::now(); + let (checked, targets) = { + let subscriptions = self.state.event_subscriptions.lock().await; + if subscriptions.is_empty() { + return; } - let _ = subscription.sender.try_send(Ok(SubscribeEventsResponse { + let checked = subscriptions.len(); + let targets = subscriptions + .iter() + .filter_map(|(&subscription_id, subscription)| { + if !match_keys(&event.keys, &subscription.keys) { + return None; + } + Some((subscription_id, subscription.sender.clone())) + }) + .collect::>(); + (checked, targets) + }; + let matched = targets.len(); + + let mut closed = Vec::new(); + for (subscription_id, sender) in targets { + if let Err(err) = sender.try_send(Ok(SubscribeEventsResponse { event: Some(event.clone()), - })); + })) { + if matches!(err, TrySendError::Closed(_)) { + closed.push(subscription_id); + } + } } + + if !closed.is_empty() { + let mut subscriptions = self.state.event_subscriptions.lock().await; + for subscription_id in closed { + subscriptions.remove(&subscription_id); + } + } + ::metrics::histogram!("torii_ecs_publish_event_update_seconds") + .record(start.elapsed().as_secs_f64()); + ::metrics::counter!("torii_ecs_publish_event_subscribers_checked_total") + .increment(checked as u64); + ::metrics::counter!("torii_ecs_publish_event_subscribers_matched_total") + .increment(matched as u64); } async fn publish_contract_update(&self, contract: types::Contract) { - let subscriptions = self.state.contract_subscriptions.lock().await; - if subscriptions.is_empty() { - return; - } - for subscription in subscriptions.values() { - if !contract_matches_query(&contract, &subscription.query) { - continue; + let start = Instant::now(); + let (checked, targets) = { + let subscriptions = self.state.contract_subscriptions.lock().await; + if subscriptions.is_empty() { + return; } - let _ = subscription.sender.try_send(Ok(SubscribeContractsResponse { + let checked = subscriptions.len(); + let targets = subscriptions + .iter() + .filter_map(|(&subscription_id, subscription)| { + if !contract_matches_query(&contract, &subscription.query) { + return None; + } + Some((subscription_id, subscription.sender.clone())) + }) + .collect::>(); + (checked, targets) + }; + let matched = targets.len(); + + let mut closed = Vec::new(); + for (subscription_id, sender) in targets { + if let Err(err) = sender.try_send(Ok(SubscribeContractsResponse { contract: Some(contract.clone()), - })); + })) { + if matches!(err, TrySendError::Closed(_)) { + closed.push(subscription_id); + } + } } + + if !closed.is_empty() { + let mut subscriptions = self.state.contract_subscriptions.lock().await; + for subscription_id in closed { + subscriptions.remove(&subscription_id); + } + } + ::metrics::histogram!("torii_ecs_publish_contract_update_seconds") + .record(start.elapsed().as_secs_f64()); + ::metrics::counter!("torii_ecs_publish_contract_subscribers_checked_total") + .increment(checked as u64); + ::metrics::counter!("torii_ecs_publish_contract_subscribers_matched_total") + .increment(matched as u64); } async fn load_contracts(&self, query: &types::ContractQuery) -> Result> { - let sql = "SELECT contract_address, contract_type, head, tps, last_block_timestamp, \ - last_pending_block_tx, updated_at, created_at \ - FROM torii_ecs_contracts"; - let rows = sqlx::query(sql).fetch_all(&self.state.pool).await?; + let mut builder = QueryBuilder::::new( + "SELECT contract_address, contract_type, head, tps, last_block_timestamp, \ + last_pending_block_tx, updated_at, created_at \ + FROM torii_ecs_contracts", + ); + if !query.contract_addresses.is_empty() || !query.contract_types.is_empty() { + builder.push(" WHERE "); + if !query.contract_addresses.is_empty() { + builder.push("contract_address IN ("); + { + let mut separated = builder.separated(", "); + for address in &query.contract_addresses { + separated.push_bind(felt_hex(felt_from_bytes(address)?)); + } + } + builder.push(")"); + if !query.contract_types.is_empty() { + builder.push(" AND "); + } + } + if !query.contract_types.is_empty() { + builder.push("contract_type IN ("); + { + let mut separated = builder.separated(", "); + for contract_type in &query.contract_types { + separated.push_bind(*contract_type); + } + } + builder.push(")"); + } + } + + let rows = builder.build().fetch_all(&self.state.pool).await?; let mut contracts = Vec::new(); for row in rows { let contract_address = row.try_get::("contract_address")?; @@ -701,14 +847,13 @@ impl EcsService { updated_at: row.try_get::("updated_at")? as u64, created_at: row.try_get::("created_at")? as u64, }; - if contract_matches_query(&contract, query) { - contracts.push(contract); - } + contracts.push(contract); } Ok(contracts) } async fn load_worlds(&self, requested: &[Felt]) -> Result> { + let requested = requested.iter().copied().collect::>(); let mut by_world: HashMap, Vec> = HashMap::new(); for table in self.load_managed_tables(None).await? { if !requested.is_empty() && !requested.contains(&table.world_address) { @@ -738,12 +883,11 @@ impl EcsService { let _ = self.load_managed_table_map().await; } let mut managed_tables = self.state.managed_tables.lock().await; - if let Some(tables) = managed_tables.as_mut() { + if let Some(tables) = managed_tables.as_ref() { + let mut next = (**tables).clone(); let key = felt_hex(table.id); - let kind = tables - .get(&key) - .map_or(TableKind::Entity, |table| table.kind); - tables.insert( + let kind = next.get(&key).map_or(TableKind::Entity, |table| table.kind); + next.insert( key, ManagedTable { world_address, @@ -751,6 +895,7 @@ impl EcsService { kind, }, ); + *managed_tables = Some(Arc::new(next)); } } @@ -761,13 +906,15 @@ impl EcsService { kind: TableKind, ) { let mut managed_tables = self.state.managed_tables.lock().await; - let Some(tables) = managed_tables.as_mut() else { + let Some(tables) = managed_tables.as_ref() else { return; }; + let mut next = (**tables).clone(); let key = felt_hex(table_id); - if let Some(table) = tables.get_mut(&key) { + if let Some(table) = next.get_mut(&key) { table.world_address = world_address; table.kind = kind; + *managed_tables = Some(Arc::new(next)); } else { *managed_tables = None; } @@ -778,11 +925,11 @@ impl EcsService { Ok(self.load_managed_table_map().await?.get(&key).cloned()) } - async fn load_managed_table_map(&self) -> Result> { + async fn load_managed_table_map(&self) -> Result>> { { let managed_tables = self.state.managed_tables.lock().await; if let Some(tables) = managed_tables.as_ref() { - return Ok(tables.clone()); + return Ok(Arc::clone(tables)); } } @@ -808,7 +955,8 @@ impl EcsService { } let mut managed_tables = self.state.managed_tables.lock().await; - *managed_tables = Some(managed.clone()); + let managed = Arc::new(managed); + *managed_tables = Some(Arc::clone(&managed)); Ok(managed) } @@ -855,8 +1003,9 @@ impl EcsService { Ok(self .load_managed_table_map() .await? - .into_values() + .values() .filter(|table| kind.is_none_or(|required| table.kind == required)) + .cloned() .collect()) } @@ -870,13 +1019,20 @@ impl EcsService { .iter() .map(|bytes| felt_from_bytes(bytes)) .collect::>>()?; + let world_filter_set = world_filters.iter().copied().collect::>(); + let model_filter_set = query + .models + .iter() + .map(String::as_str) + .collect::>(); let models = self .load_managed_tables(Some(kind)) .await? .into_iter() .filter(|table| { - (world_filters.is_empty() || world_filters.contains(&table.world_address)) - && (query.models.is_empty() || query.models.contains(&table.table.name)) + (world_filter_set.is_empty() || world_filter_set.contains(&table.world_address)) + && (model_filter_set.is_empty() + || model_filter_set.contains(table.table.name.as_str())) }) .collect::>(); if models.is_empty() { @@ -887,9 +1043,20 @@ impl EcsService { .into_iter() .map(|table| (felt_hex(table.table.id), table)) .collect::>(); + let table_ids = model_map.keys().cloned().collect::>(); + let member_pushdown = member_pushdown_from_clause(query.clause.as_ref(), &model_map); + let meta_start = Instant::now(); let candidate_meta = self - .load_entity_page_meta(kind, &world_filters, model_map.keys().cloned().collect()) + .load_entity_page_meta( + kind, + &world_filters, + table_ids.clone(), + member_pushdown.as_ref(), + ) .await?; + ::metrics::histogram!("torii_ecs_load_entity_page_meta_seconds") + .record(meta_start.elapsed().as_secs_f64()); + ::metrics::gauge!("torii_ecs_load_entity_page_candidates").set(candidate_meta.len() as f64); let mut candidate_keys = candidate_meta.keys().cloned().collect::>(); candidate_keys.sort_by(|a, b| a.1.cmp(&b.1).then_with(|| a.0.cmp(&b.0))); if query @@ -924,14 +1091,14 @@ impl EcsService { let mut items = Vec::new(); let mut built_any_snapshot_entity = false; for chunk in candidate_keys.chunks(256) { + let rows_start = Instant::now(); let rows = self - .load_entity_page_rows( - kind, - &world_filters, - model_map.keys().cloned().collect(), - chunk, - ) + .load_entity_page_rows(kind, &world_filters, table_ids.as_slice(), chunk) .await?; + ::metrics::histogram!("torii_ecs_load_entity_page_rows_seconds") + .record(rows_start.elapsed().as_secs_f64()); + ::metrics::counter!("torii_ecs_load_entity_page_rows_total") + .increment(rows.len() as u64); let row_map = rows.into_iter().fold( HashMap::<(String, String), Vec<(String, String)>>::new(), |mut acc, row| { @@ -1006,13 +1173,20 @@ impl EcsService { .iter() .map(|bytes| felt_from_bytes(bytes)) .collect::>>()?; + let world_filter_set = world_filters.iter().copied().collect::>(); + let model_filter_set = query + .models + .iter() + .map(String::as_str) + .collect::>(); let models = self .load_managed_tables(Some(kind)) .await? .into_iter() .filter(|table| { - (world_filters.is_empty() || world_filters.contains(&table.world_address)) - && (query.models.is_empty() || query.models.contains(&table.table.name)) + (world_filter_set.is_empty() || world_filter_set.contains(&table.world_address)) + && (model_filter_set.is_empty() + || model_filter_set.contains(table.table.name.as_str())) }) .collect::>(); @@ -1161,7 +1335,12 @@ impl EcsService { kind: TableKind, world_filters: &[Felt], table_ids: Vec, + member_pushdown: Option<&MemberPushdown>, ) -> Result>> { + if table_ids.is_empty() { + return Ok(HashMap::new()); + } + let deleted_sql = match self.state.backend { DbBackend::Sqlite => "CAST(deleted AS INTEGER)", DbBackend::Postgres => "CASE WHEN deleted THEN 1 ELSE 0 END", @@ -1169,10 +1348,10 @@ impl EcsService { let mut builder = QueryBuilder::::new(format!( "SELECT world_address, table_id, entity_id, created_at, updated_at, executed_at, \ {deleted_sql} AS deleted \ - FROM torii_ecs_entity_meta WHERE kind = " + FROM torii_ecs_entity_meta m WHERE m.kind = " )); builder.push_bind(kind.as_str()); - builder.push(" AND table_id IN ("); + builder.push(" AND m.table_id IN ("); { let mut separated = builder.separated(", "); for table_id in &table_ids { @@ -1181,13 +1360,24 @@ impl EcsService { } builder.push(")"); if !world_filters.is_empty() { - builder.push(" AND world_address IN ("); + builder.push(" AND m.world_address IN ("); let mut separated = builder.separated(", "); for world_address in world_filters { separated.push_bind(felt_hex(*world_address)); } builder.push(")"); } + match self.state.backend { + DbBackend::Sqlite => { + builder.push(" AND deleted = 0"); + } + DbBackend::Postgres => { + builder.push(" AND deleted = FALSE"); + } + } + if let Some(member_pushdown) = member_pushdown { + append_member_pushdown_sql(&mut builder, self.state.backend, member_pushdown); + } let rows = builder.build().fetch_all(&self.state.pool).await?; let mut meta = HashMap::new(); @@ -1206,34 +1396,20 @@ impl EcsService { }, ); } - meta.retain(|_, tables| tables.values().any(|meta| !meta.deleted)); Ok(meta) } async fn load_entity_page_rows( &self, kind: TableKind, - world_filters: &[Felt], - table_ids: Vec, + _world_filters: &[Felt], + table_ids: &[String], entity_keys: &[(String, String)], ) -> Result> { - if entity_keys.is_empty() { + if entity_keys.is_empty() || table_ids.is_empty() { return Ok(Vec::new()); } - let entity_ids = entity_keys - .iter() - .map(|(_, entity_id)| entity_id.clone()) - .collect::>(); - let world_addresses = if world_filters.is_empty() { - entity_keys - .iter() - .map(|(world_address, _)| world_address.clone()) - .collect::>() - } else { - world_filters.iter().map(|world| felt_hex(*world)).collect() - }; - let mut builder = QueryBuilder::::new( "SELECT world_address, table_id, entity_id, row_json \ FROM torii_ecs_entity_models WHERE kind = ", @@ -1242,43 +1418,34 @@ impl EcsService { builder.push(" AND table_id IN ("); { let mut separated = builder.separated(", "); - for table_id in &table_ids { + for table_id in table_ids { separated.push_bind(table_id); } } - builder.push(") AND world_address IN ("); + builder.push(") AND ("); { - let mut separated = builder.separated(", "); - for world_address in &world_addresses { + let mut separated = builder.separated(" OR "); + for (world_address, entity_id) in entity_keys { + separated.push("(world_address = "); separated.push_bind(world_address); - } - } - builder.push(") AND entity_id IN ("); - { - let mut separated = builder.separated(", "); - for entity_id in &entity_ids { + separated.push(" AND entity_id = "); separated.push_bind(entity_id); + separated.push(")"); } } builder.push(")"); - let allowed = entity_keys.iter().cloned().collect::>(); let rows = builder.build().fetch_all(&self.state.pool).await?; - let mut output = Vec::new(); - for row in rows { - let world_address = row.try_get::("world_address")?; - let entity_id = row.try_get::("entity_id")?; - if !allowed.contains(&(world_address.clone(), entity_id.clone())) { - continue; - } - output.push(EntityModelRow { - world_address, - table_id: row.try_get("table_id")?, - entity_id, - row_json: row.try_get("row_json")?, - }); - } - Ok(output) + rows.into_iter() + .map(|row| { + Ok(EntityModelRow { + world_address: row.try_get("world_address")?, + table_id: row.try_get("table_id")?, + entity_id: row.try_get("entity_id")?, + row_json: row.try_get("row_json")?, + }) + }) + .collect() } async fn load_entity_snapshot( @@ -1502,6 +1669,8 @@ impl EcsService { builder.push_bind(fetch_limit as i64); let rows = builder.build().fetch_all(&self.state.pool).await?; + ::metrics::counter!("torii_ecs_retrieve_events_rows_fetched_total") + .increment(rows.len() as u64); let mut events = Vec::new(); for row in rows { @@ -1533,6 +1702,8 @@ impl EcsService { break; } } + ::metrics::counter!("torii_ecs_retrieve_events_rows_decoded_total") + .increment(events.len() as u64); if direction_is_backward { events.reverse(); @@ -1588,6 +1759,7 @@ impl World for EcsService { &self, request: Request, ) -> Result, Status> { + let start = Instant::now(); let world_addresses = request .into_inner() .world_addresses @@ -1595,10 +1767,10 @@ impl World for EcsService { .map(|bytes| felt_from_bytes(bytes)) .collect::>>() .map_err(internal_status)?; - let worlds = self - .load_worlds(&world_addresses) - .await - .map_err(internal_status)?; + let worlds = self.load_worlds(&world_addresses).await; + ::metrics::histogram!("torii_ecs_worlds_latency_seconds") + .record(start.elapsed().as_secs_f64()); + let worlds = worlds.map_err(internal_status)?; Ok(Response::new(WorldsResponse { worlds })) } @@ -1620,7 +1792,7 @@ impl World for EcsService { subscription_id, EntitySubscription { clause: request.clause, - world_addresses: request.world_addresses, + world_addresses: request.world_addresses.into_iter().collect(), sender, }, ); @@ -1640,7 +1812,7 @@ impl World for EcsService { .get_mut(&request.subscription_id) { subscription.clause = request.clause; - subscription.world_addresses = request.world_addresses; + subscription.world_addresses = request.world_addresses.into_iter().collect(); } Ok(Response::new(())) } @@ -1649,14 +1821,17 @@ impl World for EcsService { &self, request: Request, ) -> Result, Status> { + let start = Instant::now(); let query = request .into_inner() .query .ok_or_else(|| Status::invalid_argument("Missing query argument"))?; - let (entities, next_cursor) = self - .load_entity_page(TableKind::Entity, &query) - .await - .map_err(internal_status)?; + let result = self.load_entity_page(TableKind::Entity, &query).await; + ::metrics::histogram!("torii_ecs_retrieve_entities_latency_seconds") + .record(start.elapsed().as_secs_f64()); + let (entities, next_cursor) = result.map_err(internal_status)?; + ::metrics::counter!("torii_ecs_retrieve_entities_returned_total") + .increment(entities.len() as u64); Ok(Response::new(RetrieveEntitiesResponse { next_cursor, entities, @@ -1681,7 +1856,7 @@ impl World for EcsService { subscription_id, EntitySubscription { clause: request.clause, - world_addresses: request.world_addresses, + world_addresses: request.world_addresses.into_iter().collect(), sender, }, ); @@ -1701,7 +1876,7 @@ impl World for EcsService { .get_mut(&request.subscription_id) { subscription.clause = request.clause; - subscription.world_addresses = request.world_addresses; + subscription.world_addresses = request.world_addresses.into_iter().collect(); } Ok(Response::new(())) } @@ -1710,14 +1885,17 @@ impl World for EcsService { &self, request: Request, ) -> Result, Status> { + let start = Instant::now(); let query = request .into_inner() .query .ok_or_else(|| Status::invalid_argument("Missing query argument"))?; - let (entities, next_cursor) = self - .load_entity_page(TableKind::EventMessage, &query) - .await - .map_err(internal_status)?; + let result = self.load_entity_page(TableKind::EventMessage, &query).await; + ::metrics::histogram!("torii_ecs_retrieve_event_messages_latency_seconds") + .record(start.elapsed().as_secs_f64()); + let (entities, next_cursor) = result.map_err(internal_status)?; + ::metrics::counter!("torii_ecs_retrieve_event_messages_returned_total") + .increment(entities.len() as u64); Ok(Response::new(RetrieveEntitiesResponse { next_cursor, entities, @@ -1728,14 +1906,17 @@ impl World for EcsService { &self, request: Request, ) -> Result, Status> { + let start = Instant::now(); let query = request .into_inner() .query .ok_or_else(|| Status::invalid_argument("Missing query argument"))?; - let (events, next_cursor) = self - .load_events_page(&query) - .await - .map_err(internal_status)?; + let result = self.load_events_page(&query).await; + ::metrics::histogram!("torii_ecs_retrieve_events_latency_seconds") + .record(start.elapsed().as_secs_f64()); + let (events, next_cursor) = result.map_err(internal_status)?; + ::metrics::counter!("torii_ecs_retrieve_events_returned_total") + .increment(events.len() as u64); Ok(Response::new(RetrieveEventsResponse { next_cursor, events, @@ -2323,6 +2504,148 @@ fn value_to_primary_felt(value: &Value, type_def: &PrimaryTypeDef) -> Result, + models: &HashMap, +) -> Option { + let ClauseType::Member(member) = clause?.clause_type.as_ref()? else { + return None; + }; + if member.member.contains('.') { + return None; + } + if !member + .member + .chars() + .all(|ch| ch.is_ascii_alphanumeric() || ch == '_') + { + return None; + } + + let expected = scalar_from_member_value(member.value.as_ref()?)?; + if matches!(expected, ScalarValue::Bytes(_)) { + return None; + } + + let table = models + .values() + .find(|table| table.table.name == member.model)?; + if !table + .table + .columns + .values() + .any(|column| column.name == member.member) + { + return None; + } + + Some(MemberPushdown { + table_id: felt_hex(table.table.id), + member: member.member.clone(), + operator: member.operator, + expected, + }) +} + +fn sql_operator(operator: i32) -> Option<&'static str> { + match operator { + value if value == ComparisonOperator::Eq as i32 => Some("="), + value if value == ComparisonOperator::Neq as i32 => Some("!="), + value if value == ComparisonOperator::Gt as i32 => Some(">"), + value if value == ComparisonOperator::Gte as i32 => Some(">="), + value if value == ComparisonOperator::Lt as i32 => Some("<"), + value if value == ComparisonOperator::Lte as i32 => Some("<="), + _ => None, + } +} + +fn append_member_pushdown_sql( + builder: &mut QueryBuilder<'_, Any>, + backend: DbBackend, + pushdown: &MemberPushdown, +) { + let Some(operator) = sql_operator(pushdown.operator) else { + return; + }; + + builder.push(" AND EXISTS (SELECT 1 FROM torii_ecs_entity_models em WHERE em.kind = m.kind"); + builder.push( + " AND em.world_address = m.world_address AND em.table_id = m.table_id AND em.entity_id = m.entity_id", + ); + builder.push(" AND em.table_id = "); + builder.push_bind(pushdown.table_id.clone()); + + match (&pushdown.expected, backend) { + (ScalarValue::Bool(value), DbBackend::Sqlite) => { + builder.push(" AND CAST(json_extract(em.row_json, '$."); + builder.push(pushdown.member.as_str()); + builder.push("') AS INTEGER) "); + builder.push(operator); + builder.push(" "); + builder.push_bind(i64::from(*value)); + } + (ScalarValue::Bool(value), DbBackend::Postgres) => { + builder.push(" AND CASE WHEN lower((em.row_json::jsonb ->> '"); + builder.push(pushdown.member.as_str()); + builder.push("')) = 'true' THEN 1 ELSE 0 END "); + builder.push(operator); + builder.push(" "); + builder.push_bind(i64::from(*value)); + } + (ScalarValue::Signed(value), DbBackend::Sqlite) => { + builder.push(" AND CAST(json_extract(em.row_json, '$."); + builder.push(pushdown.member.as_str()); + builder.push("') AS INTEGER) "); + builder.push(operator); + builder.push(" "); + builder.push_bind(*value); + } + (ScalarValue::Signed(value), DbBackend::Postgres) => { + builder.push(" AND CAST((em.row_json::jsonb ->> '"); + builder.push(pushdown.member.as_str()); + builder.push("') AS BIGINT) "); + builder.push(operator); + builder.push(" "); + builder.push_bind(*value); + } + (ScalarValue::Unsigned(value), DbBackend::Sqlite) => { + builder.push(" AND CAST(json_extract(em.row_json, '$."); + builder.push(pushdown.member.as_str()); + builder.push("') AS INTEGER) "); + builder.push(operator); + builder.push(" "); + builder.push_bind(*value as i64); + } + (ScalarValue::Unsigned(value), DbBackend::Postgres) => { + builder.push(" AND CAST((em.row_json::jsonb ->> '"); + builder.push(pushdown.member.as_str()); + builder.push("') AS BIGINT) "); + builder.push(operator); + builder.push(" "); + builder.push_bind(*value as i64); + } + (ScalarValue::Text(value), DbBackend::Sqlite) => { + builder.push(" AND CAST(json_extract(em.row_json, '$."); + builder.push(pushdown.member.as_str()); + builder.push("') AS TEXT) "); + builder.push(operator); + builder.push(" "); + builder.push_bind(value.clone()); + } + (ScalarValue::Text(value), DbBackend::Postgres) => { + builder.push(" AND (em.row_json::jsonb ->> '"); + builder.push(pushdown.member.as_str()); + builder.push("') "); + builder.push(operator); + builder.push(" "); + builder.push_bind(value.clone()); + } + (ScalarValue::Bytes(_), _) => {} + } + + builder.push(")"); +} + fn entity_matches_clause(entity: &types::Entity, clause: &types::Clause) -> bool { match &clause.clause_type { Some(ClauseType::HashedKeys(hashed)) => { @@ -2480,6 +2803,13 @@ enum ScalarValue { Bytes(Vec), } +struct MemberPushdown { + table_id: String, + member: String, + operator: i32, + expected: ScalarValue, +} + fn compare_scalars(current: &ScalarValue, operator: i32, expected: &ScalarValue) -> bool { match (current, expected) { (ScalarValue::Bool(current), ScalarValue::Bool(expected)) => match operator { From 1392f866283d318f47141b891e85f4bbe41d0376 Mon Sep 17 00:00:00 2001 From: Valentin Dosimont Date: Tue, 17 Mar 2026 21:06:06 +0100 Subject: [PATCH 07/10] `perf(ecs): add bounded keyset pagination` --- crates/torii-ecs-sink/src/grpc_service.rs | 373 +++++++++++++++++----- scripts/test-ecs-entity-model-filters.sh | 49 +++ scripts/test-ecs-grpc-e2e.sh | 3 +- scripts/test-ecs-subscriptions-smoke.sh | 92 ++++++ 4 files changed, 427 insertions(+), 90 deletions(-) create mode 100755 scripts/test-ecs-entity-model-filters.sh create mode 100755 scripts/test-ecs-subscriptions-smoke.sh diff --git a/crates/torii-ecs-sink/src/grpc_service.rs b/crates/torii-ecs-sink/src/grpc_service.rs index 834bf928..79cb728a 100644 --- a/crates/torii-ecs-sink/src/grpc_service.rs +++ b/crates/torii-ecs-sink/src/grpc_service.rs @@ -284,6 +284,13 @@ impl EcsService { .execute(&self.state.pool) .await?; + sqlx::query( + "CREATE INDEX IF NOT EXISTS torii_ecs_entity_meta_keyset_idx + ON torii_ecs_entity_meta(kind, deleted, entity_id, world_address)", + ) + .execute(&self.state.pool) + .await?; + sqlx::query( "CREATE INDEX IF NOT EXISTS torii_ecs_entity_models_lookup_idx ON torii_ecs_entity_models(kind, world_address, entity_id)", @@ -298,6 +305,13 @@ impl EcsService { .execute(&self.state.pool) .await?; + sqlx::query( + "CREATE INDEX IF NOT EXISTS torii_ecs_entity_models_key_lookup_idx + ON torii_ecs_entity_models(kind, world_address, entity_id)", + ) + .execute(&self.state.pool) + .await?; + sqlx::query( "CREATE INDEX IF NOT EXISTS torii_ecs_events_cursor_idx ON torii_ecs_events(event_id)", @@ -1044,56 +1058,64 @@ impl EcsService { .map(|table| (felt_hex(table.table.id), table)) .collect::>(); let table_ids = model_map.keys().cloned().collect::>(); + let table_id_filter = if query.models.is_empty() { + None + } else { + Some(table_ids.as_slice()) + }; let member_pushdown = member_pushdown_from_clause(query.clause.as_ref(), &model_map); - let meta_start = Instant::now(); - let candidate_meta = self - .load_entity_page_meta( + let target_limit = query.pagination.as_ref().map_or(100, |pagination| { + if pagination.limit == 0 { + 100 + } else { + pagination.limit as usize + } + }); + + let direction = query + .pagination + .as_ref() + .map_or(PaginationDirection::Forward as i32, |pagination| { + pagination.direction + }); + let cursor = query + .pagination + .as_ref() + .map_or(String::new(), |pagination| pagination.cursor.clone()); + + let keys_start = Instant::now(); + let mut candidate_keys = self + .load_entity_page_keys( kind, &world_filters, - table_ids.clone(), + table_id_filter, member_pushdown.as_ref(), + &cursor, + direction, + target_limit.saturating_add(1), ) .await?; ::metrics::histogram!("torii_ecs_load_entity_page_meta_seconds") - .record(meta_start.elapsed().as_secs_f64()); - ::metrics::gauge!("torii_ecs_load_entity_page_candidates").set(candidate_meta.len() as f64); - let mut candidate_keys = candidate_meta.keys().cloned().collect::>(); - candidate_keys.sort_by(|a, b| a.1.cmp(&b.1).then_with(|| a.0.cmp(&b.0))); - if query - .pagination - .as_ref() - .is_some_and(|p| p.direction == PaginationDirection::Backward as i32) - { - candidate_keys.reverse(); - } + .record(keys_start.elapsed().as_secs_f64()); - if let Some(pagination) = &query.pagination { - if !pagination.cursor.is_empty() { - candidate_keys.retain(|(_, entity_id)| { - let key = entity_id.trim_start_matches("0x"); - if pagination.direction == PaginationDirection::Backward as i32 { - key < pagination.cursor.as_str() - } else { - key > pagination.cursor.as_str() - } - }); - } + let has_more = target_limit != usize::MAX && candidate_keys.len() > target_limit; + if has_more { + candidate_keys.truncate(target_limit); } - let target_limit = query.pagination.as_ref().map_or(usize::MAX, |pagination| { - if pagination.limit == 0 { - 100 - } else { - pagination.limit as usize - } - }); + let meta_start = Instant::now(); + let candidate_meta = self + .load_entity_meta_for_keys(kind, table_id_filter, candidate_keys.as_slice()) + .await?; + ::metrics::histogram!("torii_ecs_load_entity_page_meta_rows_seconds") + .record(meta_start.elapsed().as_secs_f64()); + ::metrics::gauge!("torii_ecs_load_entity_page_candidates").set(candidate_keys.len() as f64); let mut items = Vec::new(); - let mut built_any_snapshot_entity = false; for chunk in candidate_keys.chunks(256) { let rows_start = Instant::now(); let rows = self - .load_entity_page_rows(kind, &world_filters, table_ids.as_slice(), chunk) + .load_entity_page_rows(kind, &world_filters, table_id_filter, chunk) .await?; ::metrics::histogram!("torii_ecs_load_entity_page_rows_seconds") .record(rows_start.elapsed().as_secs_f64()); @@ -1118,7 +1140,6 @@ impl EcsService { else { continue; }; - built_any_snapshot_entity = true; let entity = entity.into_proto(); if query .clause @@ -1138,18 +1159,10 @@ impl EcsService { } } - if !built_any_snapshot_entity && !candidate_meta.is_empty() { - return self.load_entity_page_from_storage(kind, query).await; - } - - let next_cursor = if target_limit != usize::MAX && items.len() == target_limit { - items + let next_cursor = if has_more { + candidate_keys .last() - .map(|entity| { - felt_from_bytes(&entity.hashed_keys) - .map(|felt| felt_hex(felt).trim_start_matches("0x").to_string()) - }) - .transpose()? + .map(|(_, entity_id)| entity_id.trim_start_matches("0x").to_string()) .unwrap_or_default() } else { String::new() @@ -1330,60 +1343,168 @@ impl EcsService { })) } - async fn load_entity_page_meta( + async fn load_entity_page_keys( &self, kind: TableKind, world_filters: &[Felt], - table_ids: Vec, + table_ids: Option<&[String]>, member_pushdown: Option<&MemberPushdown>, + cursor: &str, + direction: i32, + limit: usize, + ) -> Result> { + if table_ids.is_some_and(|table_ids| table_ids.is_empty()) { + return Ok(Vec::new()); + } + let limit = limit.clamp(1, 200); + + let cursor_op = if direction == PaginationDirection::Backward as i32 { + "<" + } else { + ">" + }; + let order = if direction == PaginationDirection::Backward as i32 { + "DESC" + } else { + "ASC" + }; + + let mut builder = QueryBuilder::::new(format!( + "SELECT DISTINCT m.world_address, m.entity_id \ + FROM torii_ecs_entity_meta m \ + WHERE m.kind = " + )); + builder.push_bind(kind.as_str()); + if let Some(table_ids) = table_ids { + builder.push(" AND m.table_id IN ("); + { + let mut separated = builder.separated(", "); + for table_id in table_ids { + separated.push_bind(table_id); + } + } + builder.push(")"); + } + if !world_filters.is_empty() { + builder.push(" AND m.world_address IN ("); + { + let mut separated = builder.separated(", "); + for world_address in world_filters { + separated.push_bind(felt_hex(*world_address)); + } + } + builder.push(")"); + } + match self.state.backend { + DbBackend::Sqlite => { + builder.push(" AND m.deleted = 0"); + } + DbBackend::Postgres => { + builder.push(" AND m.deleted = FALSE"); + } + } + if !cursor.is_empty() { + builder.push(" AND m.entity_id "); + builder.push(cursor_op); + builder.push(" "); + builder.push_bind(format!("0x{cursor}")); + } + if let Some(member_pushdown) = member_pushdown { + append_member_pushdown_sql(&mut builder, self.state.backend, member_pushdown); + } + builder.push(" ORDER BY m.entity_id "); + builder.push(order); + builder.push(", m.world_address "); + builder.push(order); + builder.push(" LIMIT "); + builder.push_bind(limit as i64); + + let rows = builder.build().fetch_all(&self.state.pool).await?; + rows.into_iter() + .map(|row| { + Ok(( + row.try_get::("world_address")?, + row.try_get::("entity_id")?, + )) + }) + .collect() + } + + async fn load_entity_meta_for_keys( + &self, + kind: TableKind, + table_ids: Option<&[String]>, + entity_keys: &[(String, String)], ) -> Result>> { - if table_ids.is_empty() { + if table_ids.is_some_and(|table_ids| table_ids.is_empty()) || entity_keys.is_empty() { return Ok(HashMap::new()); } let deleted_sql = match self.state.backend { - DbBackend::Sqlite => "CAST(deleted AS INTEGER)", - DbBackend::Postgres => "CASE WHEN deleted THEN 1 ELSE 0 END", + DbBackend::Sqlite => "CAST(m.deleted AS INTEGER)", + DbBackend::Postgres => "CASE WHEN m.deleted THEN 1 ELSE 0 END", }; + let mut builder = QueryBuilder::::new(format!( - "SELECT world_address, table_id, entity_id, created_at, updated_at, executed_at, \ + "SELECT m.world_address, m.table_id, m.entity_id, m.created_at, m.updated_at, m.executed_at, \ {deleted_sql} AS deleted \ FROM torii_ecs_entity_meta m WHERE m.kind = " )); builder.push_bind(kind.as_str()); - builder.push(" AND m.table_id IN ("); + if let Some(table_ids) = table_ids { + builder.push(" AND m.table_id IN ("); + { + let mut separated = builder.separated(", "); + for table_id in table_ids { + separated.push_bind(table_id); + } + } + builder.push(")"); + } + + let world_addresses = entity_keys + .iter() + .map(|(world_address, _)| world_address.clone()) + .collect::>(); + let entity_ids = entity_keys + .iter() + .map(|(_, entity_id)| entity_id.clone()) + .collect::>(); + let allowed = entity_keys.iter().cloned().collect::>(); + + builder.push(" AND m.world_address IN ("); { let mut separated = builder.separated(", "); - for table_id in &table_ids { - separated.push_bind(table_id); + for world_address in &world_addresses { + separated.push_bind(world_address); } } builder.push(")"); - if !world_filters.is_empty() { - builder.push(" AND m.world_address IN ("); + builder.push(" AND m.entity_id IN ("); + { let mut separated = builder.separated(", "); - for world_address in world_filters { - separated.push_bind(felt_hex(*world_address)); + for entity_id in &entity_ids { + separated.push_bind(entity_id); } - builder.push(")"); } + builder.push(")"); match self.state.backend { DbBackend::Sqlite => { - builder.push(" AND deleted = 0"); + builder.push(" AND m.deleted = 0"); } DbBackend::Postgres => { - builder.push(" AND deleted = FALSE"); + builder.push(" AND m.deleted = FALSE"); } } - if let Some(member_pushdown) = member_pushdown { - append_member_pushdown_sql(&mut builder, self.state.backend, member_pushdown); - } let rows = builder.build().fetch_all(&self.state.pool).await?; let mut meta = HashMap::new(); for row in rows { let world_address = row.try_get::("world_address")?; let entity_id = row.try_get::("entity_id")?; + if !allowed.contains(&(world_address.clone(), entity_id.clone())) { + continue; + } meta.entry((world_address, entity_id)) .or_insert_with(HashMap::new) .insert( @@ -1403,10 +1524,10 @@ impl EcsService { &self, kind: TableKind, _world_filters: &[Felt], - table_ids: &[String], + table_ids: Option<&[String]>, entity_keys: &[(String, String)], ) -> Result> { - if entity_keys.is_empty() || table_ids.is_empty() { + if entity_keys.is_empty() || table_ids.is_some_and(|table_ids| table_ids.is_empty()) { return Ok(Vec::new()); } @@ -1415,37 +1536,60 @@ impl EcsService { FROM torii_ecs_entity_models WHERE kind = ", ); builder.push_bind(kind.as_str()); - builder.push(" AND table_id IN ("); + if let Some(table_ids) = table_ids { + builder.push(" AND table_id IN ("); + { + let mut separated = builder.separated(", "); + for table_id in table_ids { + separated.push_bind(table_id); + } + } + builder.push(")"); + } + + let world_addresses = entity_keys + .iter() + .map(|(world_address, _)| world_address.clone()) + .collect::>(); + let entity_ids = entity_keys + .iter() + .map(|(_, entity_id)| entity_id.clone()) + .collect::>(); + let allowed = entity_keys.iter().cloned().collect::>(); + + builder.push(" AND world_address IN ("); { let mut separated = builder.separated(", "); - for table_id in table_ids { - separated.push_bind(table_id); + for world_address in &world_addresses { + separated.push_bind(world_address); } } - builder.push(") AND ("); + builder.push(")"); + builder.push(" AND entity_id IN ("); { - let mut separated = builder.separated(" OR "); - for (world_address, entity_id) in entity_keys { - separated.push("(world_address = "); - separated.push_bind(world_address); - separated.push(" AND entity_id = "); + let mut separated = builder.separated(", "); + for entity_id in &entity_ids { separated.push_bind(entity_id); - separated.push(")"); } } builder.push(")"); let rows = builder.build().fetch_all(&self.state.pool).await?; - rows.into_iter() - .map(|row| { - Ok(EntityModelRow { - world_address: row.try_get("world_address")?, - table_id: row.try_get("table_id")?, - entity_id: row.try_get("entity_id")?, - row_json: row.try_get("row_json")?, - }) - }) - .collect() + let mut output = Vec::new(); + for row in rows { + let world_address: String = row.try_get("world_address")?; + let entity_id: String = row.try_get("entity_id")?; + if !allowed.contains(&(world_address.clone(), entity_id.clone())) { + continue; + } + output.push(EntityModelRow { + world_address, + table_id: row.try_get("table_id")?, + entity_id, + row_json: row.try_get("row_json")?, + }); + } + Ok(output) } async fn load_entity_snapshot( @@ -3254,4 +3398,57 @@ mod tests { ); assert_eq!(contract.contract_type, ContractType::World as i32); } + + #[tokio::test] + async fn retrieve_entities_without_filters_uses_bounded_query() { + let db_path = test_db_path("retrieve-entities"); + let service = EcsService::new(&db_path, Some(1)) + .await + .expect("service init"); + let world = Felt::from(900_u64); + + seed_entity( + &service, + TableKind::Entity, + world, + Felt::from(901_u64), + "test-ModelA", + Felt::from(1001_u64), + true, + ) + .await; + seed_entity( + &service, + TableKind::Entity, + world, + Felt::from(902_u64), + "test-ModelB", + Felt::from(1001_u64), + true, + ) + .await; + + let response = service + .retrieve_entities(Request::new(RetrieveEntitiesRequest { + query: Some(types::Query { + pagination: Some(types::Pagination { + cursor: String::new(), + limit: 10, + direction: PaginationDirection::Forward as i32, + order_by: vec![], + }), + world_addresses: vec![], + models: vec![], + clause: None, + no_hashed_keys: false, + historical: false, + }), + })) + .await + .expect("retrieve entities") + .into_inner(); + + assert_eq!(response.entities.len(), 1); + assert!(!response.next_cursor.is_empty() || response.entities.len() < 10); + } } diff --git a/scripts/test-ecs-entity-model-filters.sh b/scripts/test-ecs-entity-model-filters.sh new file mode 100755 index 00000000..61045ae7 --- /dev/null +++ b/scripts/test-ecs-entity-model-filters.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +set -euo pipefail + +GRPC_ADDR="${GRPC_ADDR:-localhost:3000}" +DB_PATH="${DB_PATH:-torii-data/introspect.db}" +WORLD_ADDRESS="${WORLD_ADDRESS:-}" +LIMIT="${LIMIT:-20}" + +fail(){ printf '[ecs-model-filter] error: %s\n' "$*" >&2; exit 1; } +log(){ printf '[ecs-model-filter] %s\n' "$*"; } + +for cmd in grpcurl sqlite3 rg xxd base64 mktemp; do command -v "$cmd" >/dev/null 2>&1 || fail "missing $cmd"; done +[[ -f "$DB_PATH" ]] || fail "missing db at $DB_PATH" + +if [[ -z "$WORLD_ADDRESS" ]]; then + WORLD_ADDRESS="$(sqlite3 -batch -noheader "$DB_PATH" "SELECT world_address FROM torii_ecs_entity_models WHERE kind='entity' ORDER BY world_address LIMIT 1;")" +fi +[[ -n "$WORLD_ADDRESS" ]] || fail 'cannot discover world' + +mapfile -t MODELS < <(sqlite3 -batch -noheader "$DB_PATH" " +SELECT DISTINCT d.name +FROM torii_ecs_table_kinds k +JOIN dojo_tables d + ON ltrim(lower(hex(d.owner)), '0') = ltrim(substr(k.world_address, 3), '0') + AND ltrim(lower(hex(d.id)), '0') = ltrim(substr(k.table_id, 3), '0') +WHERE k.world_address='${WORLD_ADDRESS}' AND k.kind='entity' +ORDER BY d.name LIMIT 2;") +[[ ${#MODELS[@]} -ge 2 ]] || fail 'not enough models' + +felt_to_b64(){ local h="${1#0x}"; h="$(printf '%064s' "$h" | tr ' ' '0')"; printf '%s' "$h" | xxd -r -p | base64 | tr -d '\n'; } +WORLD_B64="$(felt_to_b64 "$WORLD_ADDRESS")" + +tmp="$(mktemp -d)"; trap 'rm -rf "$tmp"' EXIT +one="$tmp/one.json"; two="$tmp/two.json"; both="$tmp/both.json"; bad="$tmp/bad.json" + +grpcurl -plaintext -d "{\"query\":{\"world_addresses\":[\"$WORLD_B64\"],\"models\":[\"${MODELS[0]}\"],\"pagination\":{\"limit\":$LIMIT,\"direction\":\"FORWARD\"}}}" "$GRPC_ADDR" world.World/RetrieveEntities >"$one" +grpcurl -plaintext -d "{\"query\":{\"world_addresses\":[\"$WORLD_B64\"],\"models\":[\"${MODELS[1]}\"],\"pagination\":{\"limit\":$LIMIT,\"direction\":\"FORWARD\"}}}" "$GRPC_ADDR" world.World/RetrieveEntities >"$two" +grpcurl -plaintext -d "{\"query\":{\"world_addresses\":[\"$WORLD_B64\"],\"models\":[\"${MODELS[0]}\",\"${MODELS[1]}\"],\"pagination\":{\"limit\":$LIMIT,\"direction\":\"FORWARD\"}}}" "$GRPC_ADDR" world.World/RetrieveEntities >"$both" +grpcurl -plaintext -d "{\"query\":{\"world_addresses\":[\"$WORLD_B64\"],\"models\":[\"definitely-not-a-real-model\"],\"pagination\":{\"limit\":$LIMIT,\"direction\":\"FORWARD\"}}}" "$GRPC_ADDR" world.World/RetrieveEntities >"$bad" + +rg -q 'hashedKeys' "$one" || fail 'single model A returned no entities' +rg -q 'hashedKeys' "$two" || fail 'single model B returned no entities' +rg -q 'hashedKeys' "$both" || fail 'union model query returned no entities' +! rg -q 'hashedKeys' "$bad" || fail 'unknown model should return no entities' + +log "world=${WORLD_ADDRESS}" +log "model_a=${MODELS[0]}" +log "model_b=${MODELS[1]}" +log 'entity model filter checks passed' diff --git a/scripts/test-ecs-grpc-e2e.sh b/scripts/test-ecs-grpc-e2e.sh index 1ef12ce4..e8d1e950 100755 --- a/scripts/test-ecs-grpc-e2e.sh +++ b/scripts/test-ecs-grpc-e2e.sh @@ -133,7 +133,6 @@ require_cmd rg require_cmd xxd require_cmd base64 require_cmd mktemp -require_cmd python3 if [[ -n "$DB_URL" && "$DB_URL" =~ ^postgres(ql)?:// ]]; then DB_BACKEND="postgres" @@ -240,7 +239,7 @@ log "verification passed" if [[ "$RUN_SUBSCRIPTIONS_SMOKE" == "1" ]]; then log "running subscriptions smoke checks" - GRPC_ADDR="$GRPC_ADDR" python3 "$REPO_ROOT/scripts/test-ecs-subscriptions-smoke.py" + GRPC_ADDR="$GRPC_ADDR" "$REPO_ROOT/scripts/test-ecs-subscriptions-smoke.sh" fi if [[ "${KEEP_TMP:-0}" == "1" ]]; then diff --git a/scripts/test-ecs-subscriptions-smoke.sh b/scripts/test-ecs-subscriptions-smoke.sh new file mode 100755 index 00000000..3ceed1c5 --- /dev/null +++ b/scripts/test-ecs-subscriptions-smoke.sh @@ -0,0 +1,92 @@ +#!/usr/bin/env bash + +set -euo pipefail + +GRPC_ADDR="${GRPC_ADDR:-localhost:3000}" +STREAM_TIMEOUT_SEC="${STREAM_TIMEOUT_SEC:-2.5}" +REQUIRE_LIVE_UPDATES="${REQUIRE_LIVE_UPDATES:-0}" +WORLD_ADDRESS_B64="${WORLD_ADDRESS_B64:-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=}" + +log() { + printf '[ecs-sub-smoke] %s\n' "$*" +} + +fail() { + printf '[ecs-sub-smoke] error: %s\n' "$*" >&2 + exit 1 +} + +require_cmd() { + command -v "$1" >/dev/null 2>&1 || fail "missing required command: $1" +} + +stream_call() { + local method="$1" + local payload="$2" + local out_file="$3" + local err_file="$4" + + set +e + grpcurl -plaintext -max-time "$STREAM_TIMEOUT_SEC" -d "$payload" "$GRPC_ADDR" "$method" >"$out_file" 2>"$err_file" + local status=$? + set -e + + if [[ $status -ne 0 ]]; then + if rg -q 'DeadlineExceeded|max-time reached' "$err_file"; then + return 0 + fi + fail "stream call failed for $method: $(<"$err_file")" + fi +} + +require_cmd grpcurl +require_cmd rg +require_cmd mktemp + +list_out="$(mktemp)" +trap 'rm -f "$list_out" "$entities_out" "$entities_err" "$messages_out" "$messages_err" "$events_out" "$events_err" "$contracts_out" "$contracts_err"' EXIT + +grpcurl -plaintext "$GRPC_ADDR" list >"$list_out" +rg -F -q 'world.World' "$list_out" || fail 'world.World service not exposed' + +entities_out="$(mktemp)"; entities_err="$(mktemp)" +messages_out="$(mktemp)"; messages_err="$(mktemp)" +events_out="$(mktemp)"; events_err="$(mktemp)" +contracts_out="$(mktemp)"; contracts_err="$(mktemp)" + +log "gRPC target: $GRPC_ADDR" +log "world filter: $WORLD_ADDRESS_B64" + +stream_call "world.World/SubscribeEntities" "{\"world_addresses\":[\"$WORLD_ADDRESS_B64\"]}" "$entities_out" "$entities_err" +rg -q '"subscriptionId":' "$entities_out" || fail 'SubscribeEntities missing subscriptionId frame' +if [[ "$REQUIRE_LIVE_UPDATES" == "1" ]]; then + rg -q '"entity": \{' "$entities_out" || fail 'SubscribeEntities did not receive live entity update' +fi +log 'SubscribeEntities setup frame: OK' + +stream_call "world.World/SubscribeEventMessages" "{\"world_addresses\":[\"$WORLD_ADDRESS_B64\"]}" "$messages_out" "$messages_err" +rg -q '"subscriptionId":' "$messages_out" || fail 'SubscribeEventMessages missing subscriptionId frame' +if [[ "$REQUIRE_LIVE_UPDATES" == "1" ]]; then + rg -q '"entity": \{' "$messages_out" || fail 'SubscribeEventMessages did not receive live update' +fi +log 'SubscribeEventMessages setup frame: OK' + +stream_call "world.World/SubscribeEvents" '{"keys":[]}' "$events_out" "$events_err" +if ! rg -q '^\{\}' "$events_out" && ! rg -q '"event": \{' "$events_out"; then + fail 'SubscribeEvents did not emit setup or live frame' +fi +if [[ "$REQUIRE_LIVE_UPDATES" == "1" ]]; then + rg -q '"event": \{' "$events_out" || fail 'SubscribeEvents did not receive live event' +fi +log 'SubscribeEvents setup frame: OK' + +stream_call "world.World/SubscribeContracts" '{"query":{"contract_addresses":["AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="]}}' "$contracts_out" "$contracts_err" +if rg -q '"contract": \{' "$contracts_out"; then + log 'SubscribeContracts initial/live frame: OK' +elif [[ "$REQUIRE_LIVE_UPDATES" == "1" ]]; then + fail 'SubscribeContracts did not receive contract frame' +else + log 'SubscribeContracts: no contract frame during window (inconclusive)' +fi + +log 'subscription smoke checks passed' From b224bff6541892aa2f0eb15dec2766d67aa118d5 Mon Sep 17 00:00:00 2001 From: Valentin Dosimont Date: Wed, 18 Mar 2026 09:11:36 +0100 Subject: [PATCH 08/10] refactor(introspect): batch process envelopes --- ..._schema_state.sql => 003_schema_state.sql} | 0 crates/introspect-postgres-sink/src/lib.rs | 1 - .../introspect-postgres-sink/src/processor.rs | 39 +++++++++++++++++- crates/introspect-postgres-sink/src/sqlite.rs | 23 +++++++---- .../introspect-sqlite-sink/src/processor.rs | 28 +++++++++---- crates/introspect-sqlite-sink/src/sink.rs | 4 +- crates/torii-ecs-sink/src/grpc_service.rs | 41 ++++++++++++++----- crates/torii-ecs-sink/src/sink.rs | 29 +++++++------ src/etl/decoder/context.rs | 25 +---------- 9 files changed, 123 insertions(+), 67 deletions(-) rename crates/introspect-postgres-sink/migrations/{002_schema_state.sql => 003_schema_state.sql} (100%) diff --git a/crates/introspect-postgres-sink/migrations/002_schema_state.sql b/crates/introspect-postgres-sink/migrations/003_schema_state.sql similarity index 100% rename from crates/introspect-postgres-sink/migrations/002_schema_state.sql rename to crates/introspect-postgres-sink/migrations/003_schema_state.sql diff --git a/crates/introspect-postgres-sink/src/lib.rs b/crates/introspect-postgres-sink/src/lib.rs index 2d397d05..f2a44973 100644 --- a/crates/introspect-postgres-sink/src/lib.rs +++ b/crates/introspect-postgres-sink/src/lib.rs @@ -4,7 +4,6 @@ pub mod json; pub mod processor; pub mod query; pub mod sink; -pub mod sql; pub mod sqlite; pub mod table; pub mod types; diff --git a/crates/introspect-postgres-sink/src/processor.rs b/crates/introspect-postgres-sink/src/processor.rs index d34aa2c0..64e4157f 100644 --- a/crates/introspect-postgres-sink/src/processor.rs +++ b/crates/introspect-postgres-sink/src/processor.rs @@ -1,12 +1,11 @@ use crate::json::PostgresJsonSerializer; use crate::query::CreatePgTable; -use crate::sql::write_conflict_res; use crate::table::{DeadField, PgTable}; use crate::{PgDbError, PgDbResult, PgSchema, INTROSPECT_PG_SINK_MIGRATIONS}; use introspect_types::{ColumnInfo, ResultInto, TypeDef}; use serde_json::Serializer as JsonSerializer; use sqlx::types::Json; -use sqlx::{FromRow, PgPool, Row}; +use sqlx::{FromRow, PgPool}; use starknet_types_core::felt::Felt; use std::collections::HashMap; use std::io::Write; @@ -297,6 +296,42 @@ impl IntrospectPgDb { Ok(()) } + async fn persist_table_state(&self, table: &TableSchema, alive: bool) -> PgDbResult<()> { + let schema_json = serde_json::to_string(table)?; + sqlx::query( + r#" + INSERT INTO introspect.introspect_sink_schema_state (table_id, table_schema_json, alive, updated_at) + VALUES ($1, $2, $3, NOW()) + ON CONFLICT (table_id) + DO UPDATE SET + table_schema_json = EXCLUDED.table_schema_json, + alive = EXCLUDED.alive, + updated_at = NOW() + "#, + ) + .bind(format!("{:#x}", table.id)) + .bind(schema_json) + .bind(alive) + .execute(self.pool()) + .await?; + Ok(()) + } + + async fn persist_alive_state(&self, table_id: Felt, alive: bool) -> PgDbResult<()> { + sqlx::query( + r#" + UPDATE introspect.introspect_sink_schema_state + SET alive = $1, updated_at = NOW() + WHERE table_id = $2 + "#, + ) + .bind(alive) + .bind(format!("{table_id:#x}")) + .execute(self.pool()) + .await?; + Ok(()) + } + pub async fn initialize_introspect_pg_sink(&self) -> PgDbResult<()> { self.migrate(Some("introspect"), INTROSPECT_PG_SINK_MIGRATIONS) .await?; diff --git a/crates/introspect-postgres-sink/src/sqlite.rs b/crates/introspect-postgres-sink/src/sqlite.rs index b8dd5cba..ec269e62 100644 --- a/crates/introspect-postgres-sink/src/sqlite.rs +++ b/crates/introspect-postgres-sink/src/sqlite.rs @@ -2,7 +2,8 @@ use anyhow::Context; use introspect_types::serialize::ToCairoDeSeFrom; use introspect_types::serialize_def::CairoTypeSerialization; use introspect_types::{ - CairoDeserializer, ColumnDef, PrimaryDef, PrimaryTypeDef, ResultDef, TupleDef, TypeDef, + CairoDeserializer, ColumnDef, ColumnInfo, PrimaryDef, PrimaryTypeDef, ResultDef, TupleDef, + TypeDef, }; use serde::ser::SerializeMap; use serde::Serializer; @@ -435,14 +436,18 @@ fn sqlite_records_for_event( table: &SqliteTable, event: &InsertsFields, ) -> SqliteResult> { - let schema = torii_introspect::tables::RecordSchema::new( - &table.primary, - event - .columns - .iter() - .map(|column_id| table.get_living_column(column_id)) - .collect::>>()?, - ); + let column_infos = event + .columns + .iter() + .map(|column_id| { + table.get_living_column(column_id).cloned().map(|column| { + let (_, info): (Felt, ColumnInfo) = column.into(); + info + }) + }) + .collect::>>()?; + let schema = + torii_introspect::tables::RecordSchema::new(&table.primary, column_infos.iter().collect()); let mut bytes = Vec::new(); let mut serializer = JsonSerializer::new(&mut bytes); diff --git a/crates/introspect-sqlite-sink/src/processor.rs b/crates/introspect-sqlite-sink/src/processor.rs index 8b86a2b1..3f6121b2 100644 --- a/crates/introspect-sqlite-sink/src/processor.rs +++ b/crates/introspect-sqlite-sink/src/processor.rs @@ -8,8 +8,8 @@ use std::collections::HashMap; use std::fmt::Display; use std::ops::Deref; use std::sync::{PoisonError, RwLock}; -use torii::etl::MetaData; -use torii_introspect::events::IntrospectMsg; +use torii::etl::envelope::MetaData; +use torii_introspect::events::{IntrospectBody, IntrospectMsg}; use torii_introspect::schema::TableSchema; use torii_introspect::InsertsFields; use torii_sqlite::{SqliteConnection, SqlxError}; @@ -188,11 +188,11 @@ impl IntrospectSqliteDb { async fn load_persisted_state(&self) -> SqliteDbResult<()> { let rows = sqlx::query( - r#" + r" SELECT table_schema_json, alive FROM introspect_sink_schema_state ORDER BY updated_at ASC - "#, + ", ) .fetch_all(self.pool()) .await?; @@ -215,7 +215,7 @@ impl IntrospectSqliteDb { let schema_json = serde_json::to_string(table)?; let alive = i64::from(alive); sqlx::query( - r#" + r" INSERT INTO introspect_sink_schema_state (table_id, table_schema_json, alive, updated_at) VALUES (?1, ?2, ?3, unixepoch()) ON CONFLICT (table_id) @@ -223,7 +223,7 @@ impl IntrospectSqliteDb { table_schema_json = excluded.table_schema_json, alive = excluded.alive, updated_at = unixepoch() - "#, + ", ) .bind(format!("{:#x}", table.id)) .bind(schema_json) @@ -236,11 +236,11 @@ impl IntrospectSqliteDb { async fn persist_alive_state(&self, table_id: Felt, alive: bool) -> SqliteDbResult<()> { let alive = i64::from(alive); sqlx::query( - r#" + r" UPDATE introspect_sink_schema_state SET alive = ?1, updated_at = unixepoch() WHERE table_id = ?2 - "#, + ", ) .bind(alive) .bind(format!("{table_id:#x}")) @@ -301,6 +301,18 @@ impl IntrospectSqliteDb { } } + pub async fn process_messages( + &self, + msgs: Vec<&IntrospectBody>, + ) -> SqliteDbResult>> { + let mut results = Vec::with_capacity(msgs.len()); + for body in msgs { + let (msg, metadata) = body.into(); + results.push(self.process_message(msg, metadata).await); + } + Ok(results) + } + async fn insert_fields( &self, event: &InsertsFields, diff --git a/crates/introspect-sqlite-sink/src/sink.rs b/crates/introspect-sqlite-sink/src/sink.rs index fee2cc19..53de7d7c 100644 --- a/crates/introspect-sqlite-sink/src/sink.rs +++ b/crates/introspect-sqlite-sink/src/sink.rs @@ -31,6 +31,7 @@ impl Sink for IntrospectSqliteDb { let mut inserts_fields = 0usize; let mut inserted_records = 0usize; let mut delete_records = 0usize; + let mut msgs = Vec::with_capacity(envelopes.len()); for envelope in envelopes { if envelope.type_id == INTROSPECT_TYPE { if let Some(body) = envelope.downcast_ref::() { @@ -47,10 +48,11 @@ impl Sink for IntrospectSqliteDb { _ => {} } processed += 1; - self.process_message(&body.msg, &body.metadata).await?; + msgs.push(body); } } } + self.process_messages(msgs).await?; if processed > 0 { tracing::info!( diff --git a/crates/torii-ecs-sink/src/grpc_service.rs b/crates/torii-ecs-sink/src/grpc_service.rs index 79cb728a..59014e16 100644 --- a/crates/torii-ecs-sink/src/grpc_service.rs +++ b/crates/torii-ecs-sink/src/grpc_service.rs @@ -8,7 +8,8 @@ use chrono::Utc; use introspect_types::serialize::ToCairoDeSeFrom; use introspect_types::serialize_def::CairoTypeSerialization; use introspect_types::{ - Attributes, CairoDeserializer, ColumnDef, PrimaryTypeDef, ResultDef, TupleDef, TypeDef, + Attributes, CairoDeserializer, ColumnDef, ColumnInfo, PrimaryTypeDef, ResultDef, TupleDef, + TypeDef, }; use serde::ser::SerializeMap; use serde::Serializer; @@ -1000,7 +1001,7 @@ impl EcsService { .connect_with(SqliteConnectOptions::from_str(&self.state.database_url)?) .await?; let store = SqliteStore(Arc::new(pool)); - Ok(store.load_tables(&[]).await?) + Ok(store.read_tables(&[]).await?) } DbBackend::Postgres => { let pool = PgPoolOptions::new() @@ -1008,7 +1009,7 @@ impl EcsService { .connect(&self.state.database_url) .await?; let store = PgStore(Arc::new(pool)); - Ok(store.load_tables(&[]).await?) + Ok(store.read_tables(&[]).await?) } } } @@ -1176,6 +1177,7 @@ impl EcsService { Ok((items, next_cursor)) } + #[allow(dead_code)] async fn load_entity_page_from_storage( &self, kind: TableKind, @@ -1353,7 +1355,7 @@ impl EcsService { direction: i32, limit: usize, ) -> Result> { - if table_ids.is_some_and(|table_ids| table_ids.is_empty()) { + if table_ids.is_some_and(<[String]>::is_empty) { return Ok(Vec::new()); } let limit = limit.clamp(1, 200); @@ -1369,11 +1371,11 @@ impl EcsService { "ASC" }; - let mut builder = QueryBuilder::::new(format!( + let mut builder = QueryBuilder::::new( "SELECT DISTINCT m.world_address, m.entity_id \ FROM torii_ecs_entity_meta m \ - WHERE m.kind = " - )); + WHERE m.kind = ", + ); builder.push_bind(kind.as_str()); if let Some(table_ids) = table_ids { builder.push(" AND m.table_id IN ("); @@ -1436,7 +1438,7 @@ impl EcsService { table_ids: Option<&[String]>, entity_keys: &[(String, String)], ) -> Result>> { - if table_ids.is_some_and(|table_ids| table_ids.is_empty()) || entity_keys.is_empty() { + if table_ids.is_some_and(<[String]>::is_empty) || entity_keys.is_empty() { return Ok(HashMap::new()); } @@ -1527,7 +1529,7 @@ impl EcsService { table_ids: Option<&[String]>, entity_keys: &[(String, String)], ) -> Result> { - if entity_keys.is_empty() || table_ids.is_some_and(|table_ids| table_ids.is_empty()) { + if entity_keys.is_empty() || table_ids.is_some_and(<[String]>::is_empty) { return Ok(Vec::new()); } @@ -1703,6 +1705,7 @@ impl EcsService { Ok(meta) } + #[allow(dead_code)] async fn load_entity_meta( &self, kind: TableKind, @@ -1742,6 +1745,7 @@ impl EcsService { Ok(meta) } + #[allow(dead_code)] async fn fetch_table_rows(&self, table: &ManagedTable) -> Result>> { match self.state.backend { DbBackend::Sqlite => { @@ -2269,8 +2273,18 @@ fn record_to_json_map( .get(column_id) .ok_or_else(|| anyhow!("column {column_id:#x} not found in table {}", table.name)) }) - .collect::>>()?; - let schema = torii_introspect::tables::RecordSchema::new(&schema_table.primary, schema_columns); + .collect::>>()? + .into_iter() + .cloned() + .map(|column| { + let (_, info): (Felt, ColumnInfo) = column.into(); + info + }) + .collect::>(); + let schema = torii_introspect::tables::RecordSchema::new( + &schema_table.primary, + schema_columns.iter().collect(), + ); let mut bytes = Vec::new(); let mut serializer = JsonSerializer::new(&mut bytes); @@ -2339,6 +2353,7 @@ fn build_entity_from_snapshot( Ok(Some(aggregate)) } +#[allow(dead_code)] fn any_row_to_json_map(row: &sqlx::any::AnyRow, table: &DojoTable) -> Result> { let mut map = Map::new(); let primary_name = &table.primary.name; @@ -2355,6 +2370,7 @@ fn any_row_to_json_map(row: &sqlx::any::AnyRow, table: &DojoTable) -> Result Result { if is_integer_type(type_def) { let value = row.try_get::, _>(name)?; @@ -2372,6 +2388,7 @@ fn read_row_value(row: &sqlx::any::AnyRow, name: &str, type_def: &TypeDef) -> Re } } +#[allow(dead_code)] fn is_integer_type(type_def: &TypeDef) -> bool { matches!( type_def, @@ -2386,6 +2403,7 @@ fn is_integer_type(type_def: &TypeDef) -> bool { ) } +#[allow(dead_code)] fn is_json_text_type(type_def: &TypeDef) -> bool { matches!( type_def, @@ -2631,6 +2649,7 @@ fn value_as_felt_bytes(value: &Value) -> Result> { Ok(felt_from_hex(trimmed)?.to_bytes_be().to_vec()) } +#[allow(dead_code)] fn value_to_primary_felt(value: &Value, type_def: &PrimaryTypeDef) -> Result { match type_def { PrimaryTypeDef::Bool => Ok(Felt::from(value.as_bool().unwrap_or_default() as u64)), diff --git a/crates/torii-ecs-sink/src/sink.rs b/crates/torii-ecs-sink/src/sink.rs index 047e6daa..314a46af 100644 --- a/crates/torii-ecs-sink/src/sink.rs +++ b/crates/torii-ecs-sink/src/sink.rs @@ -112,25 +112,25 @@ impl Sink for EcsSink { continue; }; let context = batch - .get_event_context(&body.transaction_hash, body.from_address) + .get_event_context(&body.metadata.transaction_hash, body.metadata.from_address) .unwrap_or_default(); match &body.msg { IntrospectMsg::CreateTable(table) => { self.service - .cache_created_table(body.from_address, table) + .cache_created_table(body.metadata.from_address, table) .await; self.service - .record_table_kind(body.from_address, table.id, TableKind::Entity) + .record_table_kind(body.metadata.from_address, table.id, TableKind::Entity) .await .ok(); } IntrospectMsg::UpdateTable(table) => { self.service - .cache_updated_table(body.from_address, table) + .cache_updated_table(body.metadata.from_address, table) .await; self.service - .record_table_kind(body.from_address, table.id, TableKind::Entity) + .record_table_kind(body.metadata.from_address, table.id, TableKind::Entity) .await .ok(); } @@ -146,14 +146,14 @@ impl Sink for EcsSink { TableKind::Entity }; self.service - .record_table_kind(body.from_address, insert.table, kind) + .record_table_kind(body.metadata.from_address, insert.table, kind) .await?; for record in &insert.records { let entity_id = Felt::from_bytes_be(&record.id); self.service .upsert_entity_meta( kind, - body.from_address, + body.metadata.from_address, insert.table, entity_id, context.block.timestamp, @@ -163,7 +163,7 @@ impl Sink for EcsSink { self.service .upsert_entity_model( kind, - body.from_address, + body.metadata.from_address, insert.table, &insert.columns, record, @@ -171,7 +171,7 @@ impl Sink for EcsSink { ) .await?; self.service - .publish_entity_update(kind, body.from_address, entity_id) + .publish_entity_update(kind, body.metadata.from_address, entity_id) .await?; } } @@ -182,7 +182,7 @@ impl Sink for EcsSink { self.service .upsert_entity_meta( kind, - body.from_address, + body.metadata.from_address, delete.table, entity_id, context.block.timestamp, @@ -190,10 +190,15 @@ impl Sink for EcsSink { ) .await?; self.service - .delete_entity_model(kind, body.from_address, delete.table, entity_id) + .delete_entity_model( + kind, + body.metadata.from_address, + delete.table, + entity_id, + ) .await?; self.service - .publish_entity_update(kind, body.from_address, entity_id) + .publish_entity_update(kind, body.metadata.from_address, entity_id) .await?; } } diff --git a/src/etl/decoder/context.rs b/src/etl/decoder/context.rs index d5f592ac..20c0a843 100644 --- a/src/etl/decoder/context.rs +++ b/src/etl/decoder/context.rs @@ -22,31 +22,10 @@ use super::{ContractFilter, Decoder, DecoderId}; use crate::etl::engine_db::EngineDb; use crate::etl::envelope::Envelope; -fn felt_to_utf8_lossy(felt: &Felt) -> String { - let bytes = felt.to_bytes_be(); - let start = bytes.iter().position(|b| *b != 0).unwrap_or(bytes.len()); - String::from_utf8_lossy(&bytes[start..]).into_owned() -} - fn event_preview(event: &EmittedEvent) -> String { - let key0 = event.keys.first(); - let data0 = event.data.first(); - let key0_hex = key0 - .map(|k| format!("{k:#x}")) - .unwrap_or_else(|| "".to_string()); - let key0_utf8 = key0 - .map(felt_to_utf8_lossy) - .unwrap_or_else(|| "".to_string()); - let data0_hex = data0 - .map(|d| format!("{d:#x}")) - .unwrap_or_else(|| "".to_string()); - let data0_utf8 = data0 - .map(felt_to_utf8_lossy) - .unwrap_or_else(|| "".to_string()); - format!( - "contract={:#x} tx={:#x} key0_hex={} key0_utf8={:?} data0_hex={} data0_utf8={:?}", - event.from_address, event.transaction_hash, key0_hex, key0_utf8, data0_hex, data0_utf8 + "contract={:#x} tx={:#x}", + event.from_address, event.transaction_hash ) } From 360d4c09273e6cf4a080081c8963b4b8cefb1556 Mon Sep 17 00:00:00 2001 From: Bengineer Date: Wed, 18 Mar 2026 10:44:48 +0000 Subject: [PATCH 09/10] feat: add Dojo introspect dependency and enhance error messages with hex formatting --- Cargo.lock | 4 ++++ bins/torii-introspect-bin/src/main.rs | 5 ++++- crates/dojo/src/error.rs | 12 ++++++------ crates/dojo/src/table.rs | 1 + 4 files changed, 15 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 70bb0b26..30cb16a1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -917,6 +917,7 @@ dependencies = [ [[package]] name = "dojo-introspect" version = "0.1.0" +source = "git+https://github.com/dojoengine/dojo-introspect?rev=aadc3c9#aadc3c980706596a4a083413813a0a3ab01fded7" dependencies = [ "async-trait", "introspect-rust-macros", @@ -1863,6 +1864,7 @@ dependencies = [ [[package]] name = "introspect-events" version = "0.1.2" +source = "git+https://github.com/cartridge-gg/introspect?rev=34e93c1#34e93c10c867c53c622cce03abb6431c9dae0ef5" dependencies = [ "cainome-cairo-serde", "introspect-types", @@ -1874,6 +1876,7 @@ dependencies = [ [[package]] name = "introspect-rust-macros" version = "0.1.0" +source = "git+https://github.com/cartridge-gg/introspect?rev=34e93c1#34e93c10c867c53c622cce03abb6431c9dae0ef5" dependencies = [ "paste", "proc-macro2", @@ -1885,6 +1888,7 @@ dependencies = [ [[package]] name = "introspect-types" version = "0.1.2" +source = "git+https://github.com/cartridge-gg/introspect?rev=34e93c1#34e93c10c867c53c622cce03abb6431c9dae0ef5" dependencies = [ "blake3", "convert_case", diff --git a/bins/torii-introspect-bin/src/main.rs b/bins/torii-introspect-bin/src/main.rs index 95c07861..03e36167 100644 --- a/bins/torii-introspect-bin/src/main.rs +++ b/bins/torii-introspect-bin/src/main.rs @@ -160,8 +160,11 @@ async fn run_with_postgres( ); let decoder = DojoDecoder::, _>::new(pool.clone(), provider); + let sink = IntrospectPgDb::new(pool.clone(), ()); decoder.store.initialize().await?; + sink.initialize_introspect_pg_sink().await?; decoder.load_tables(&[]).await?; + sink.load_tables_no_commit(decoder.get_tables()?)?; let decoder: Arc = Arc::new(decoder); @@ -170,7 +173,7 @@ async fn run_with_postgres( .engine_database_url(engine_database_url) .with_extractor(extractor) .add_decoder(decoder) - .add_sink_boxed(Box::new(IntrospectPgDb::new(pool.clone(), ()))); + .add_sink_boxed(Box::new(sink)); let decoder_id = DecoderId::new("dojo-introspect"); for contract in contracts { diff --git a/crates/dojo/src/error.rs b/crates/dojo/src/error.rs index 35b13dea..911f7550 100644 --- a/crates/dojo/src/error.rs +++ b/crates/dojo/src/error.rs @@ -8,21 +8,21 @@ use starknet_types_core::felt::Felt; #[derive(Debug, thiserror::Error)] pub enum DojoToriiError { - #[error("Unknown Dojo Event selector {0}")] + #[error("Unknown Dojo Event selector {0:#066x}")] UnknownDojoEventSelector(Felt), #[error("Missing event selector")] MissingEventSelector, - #[error("Column {0} not found in table {1}")] + #[error("Column {0:#066x} not found in table {1}")] ColumnNotFound(Felt, String), - #[error("Failed to parse field {0} in table {1}")] + #[error("Failed to parse field {0:#066x} in table {1}")] FieldParseError(Felt, String), - #[error("Too many values provided for field {0}")] + #[error("Too many values provided for field {0:#066x}")] TooManyFieldValues(Felt), #[error("Failed to parse values for table {0}")] ParseValuesError(String), - #[error("Cannot add {2} table already exists with id {0} and name {1}")] + #[error("Cannot add {2} table already exists with id {0:#066x} and name {1}")] TableAlreadyExists(Felt, String, String), - #[error("Table not found with id {0}")] + #[error("Table not found with id {0:#066x}")] TableNotFoundById(Felt), #[error("Failed to acquire lock: {0}")] LockError(String), diff --git a/crates/dojo/src/table.rs b/crates/dojo/src/table.rs index 4c8d24d3..05048a88 100644 --- a/crates/dojo/src/table.rs +++ b/crates/dojo/src/table.rs @@ -22,6 +22,7 @@ pub struct DojoTable { pub legacy: bool, } +#[derive(Debug, serde::Serialize, serde::Deserialize, Clone)] pub struct DojoTableInfo { pub name: String, pub attributes: Vec, From 490306f1809b24f602e9f697474697c0d7d55348 Mon Sep 17 00:00:00 2001 From: Bengineer Date: Wed, 18 Mar 2026 13:32:22 +0000 Subject: [PATCH 10/10] refactor(decoder): simplify event decoding by removing chunk processing --- src/etl/decoder/context.rs | 38 +++++--------------------------------- 1 file changed, 5 insertions(+), 33 deletions(-) diff --git a/src/etl/decoder/context.rs b/src/etl/decoder/context.rs index 20c0a843..a8068a32 100644 --- a/src/etl/decoder/context.rs +++ b/src/etl/decoder/context.rs @@ -12,7 +12,6 @@ //! - Deterministic ordering: decoders are always called in sorted DecoderId order use async_trait::async_trait; -use futures::stream::{self, StreamExt}; use starknet::core::types::{EmittedEvent, Felt}; use std::collections::HashMap; use std::sync::Arc; @@ -35,6 +34,8 @@ fn event_preview(event: &EmittedEvent) -> String { /// 1. Explicit mappings (highest priority, from ContractFilter) /// 2. Registry mappings (from ContractRegistry auto-identification) /// 3. All decoders (fallback when no registry is configured) +/// +#[allow(dead_code)] pub struct DecoderContext { /// Decoders indexed by their ID (hash of name) decoders: HashMap>, @@ -370,43 +371,14 @@ impl Decoder for DecoderContext { } async fn decode(&self, events: &[EmittedEvent]) -> anyhow::Result> { - const DECODE_CHUNK_SIZE: usize = 256; - - let chunk_count = events.len().div_ceil(DECODE_CHUNK_SIZE); - ::metrics::gauge!("torii_decode_parallelism").set(self.decode_parallelism as f64); - - let decode_start = std::time::Instant::now(); - let decode_tasks = events - .chunks(DECODE_CHUNK_SIZE) - .enumerate() - .map(|(chunk_index, chunk)| async move { - let mut chunk_envelopes = Vec::new(); - for event in chunk { - let envelopes = self.decode_event(event).await?; - chunk_envelopes.extend(envelopes); - } - - Ok::<_, anyhow::Error>((chunk_index, chunk_envelopes)) - }) - .collect::>(); - - let mut decoded_chunks = stream::iter(decode_tasks) - .buffer_unordered(self.decode_parallelism) - .collect::>() - .await - .into_iter() - .collect::>>()?; + let mut all_envelopes = Vec::new(); - decoded_chunks.sort_by_key(|(chunk_index, _)| *chunk_index); + for event in events { + let envelopes = self.decode_event(event).await?; - let mut all_envelopes = Vec::new(); - for (_, envelopes) in decoded_chunks { all_envelopes.extend(envelopes); } - ::metrics::histogram!("torii_decode_chunk_duration_seconds") - .record(decode_start.elapsed().as_secs_f64() / chunk_count.max(1) as f64); - tracing::debug!( target: "torii::etl::decoder_context", "Decoded {} events into {} envelopes across {} decoders",