From 70d7d5d9dd9caee766ecd4dd618534f0f3692eb2 Mon Sep 17 00:00:00 2001 From: Nasr Date: Tue, 17 Sep 2024 07:17:36 -0400 Subject: [PATCH 1/9] fix(torii-core): sqlite shared cache --- bin/torii/src/main.rs | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/bin/torii/src/main.rs b/bin/torii/src/main.rs index 374cce2829..aeaacb3e9a 100644 --- a/bin/torii/src/main.rs +++ b/bin/torii/src/main.rs @@ -154,8 +154,15 @@ async fn main() -> anyhow::Result<()> { .expect("Error setting Ctrl-C handler"); let database_url = format!("sqlite:{}", &args.database); - let options = - SqliteConnectOptions::from_str(&database_url)?.create_if_missing(true).with_regexp(); + let mut options = SqliteConnectOptions::from_str(&database_url)? + .create_if_missing(true) + .with_regexp(); + + // Set cache to shared if using in-memory database + if args.database == ":memory:" { + options = options.shared_cache(true); + } + let pool = SqlitePoolOptions::new() .min_connections(1) .max_connections(5) From b1ebc019a44309db335c97368ad9759c59aab12a Mon Sep 17 00:00:00 2001 From: Nasr Date: Tue, 17 Sep 2024 07:20:28 -0400 Subject: [PATCH 2/9] use options --- bin/torii/src/main.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/bin/torii/src/main.rs b/bin/torii/src/main.rs index aeaacb3e9a..5d6a5756d3 100644 --- a/bin/torii/src/main.rs +++ b/bin/torii/src/main.rs @@ -20,7 +20,7 @@ use clap::{ArgAction, Parser}; use dojo_metrics::{metrics_process, prometheus_exporter}; use dojo_utils::parse::{parse_socket_address, parse_url}; use dojo_world::contracts::world::WorldContractReader; -use sqlx::sqlite::{SqliteConnectOptions, SqlitePoolOptions}; +use sqlx::sqlite::{SqliteAutoVacuum, SqliteConnectOptions, SqliteJournalMode, SqlitePoolOptions, SqliteSynchronous}; use sqlx::SqlitePool; use starknet::core::types::Felt; use starknet::providers::jsonrpc::HttpTransport; @@ -163,17 +163,17 @@ async fn main() -> anyhow::Result<()> { options = options.shared_cache(true); } + // Performance settings + options = options.auto_vacuum(SqliteAutoVacuum::None); + options = options.journal_mode(SqliteJournalMode::Wal); + options = options.synchronous(SqliteSynchronous::Normal); + let pool = SqlitePoolOptions::new() .min_connections(1) .max_connections(5) .connect_with(options) .await?; - // Disable auto-vacuum - sqlx::query("PRAGMA auto_vacuum = NONE;").execute(&pool).await?; - sqlx::query("PRAGMA journal_mode = WAL;").execute(&pool).await?; - sqlx::query("PRAGMA synchronous = NORMAL;").execute(&pool).await?; - // Set the number of threads based on CPU count let cpu_count = std::thread::available_parallelism().unwrap().get(); let thread_count = cmp::min(cpu_count, 8); From 3ad736632a64fcb813459060bdf19f4a57d45249 Mon Sep 17 00:00:00 2001 From: Nasr Date: Tue, 17 Sep 2024 07:20:38 -0400 Subject: [PATCH 3/9] fmt --- bin/torii/src/main.rs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/bin/torii/src/main.rs b/bin/torii/src/main.rs index 5d6a5756d3..31a133d87b 100644 --- a/bin/torii/src/main.rs +++ b/bin/torii/src/main.rs @@ -20,7 +20,9 @@ use clap::{ArgAction, Parser}; use dojo_metrics::{metrics_process, prometheus_exporter}; use dojo_utils::parse::{parse_socket_address, parse_url}; use dojo_world::contracts::world::WorldContractReader; -use sqlx::sqlite::{SqliteAutoVacuum, SqliteConnectOptions, SqliteJournalMode, SqlitePoolOptions, SqliteSynchronous}; +use sqlx::sqlite::{ + SqliteAutoVacuum, SqliteConnectOptions, SqliteJournalMode, SqlitePoolOptions, SqliteSynchronous, +}; use sqlx::SqlitePool; use starknet::core::types::Felt; use starknet::providers::jsonrpc::HttpTransport; @@ -154,9 +156,8 @@ async fn main() -> anyhow::Result<()> { .expect("Error setting Ctrl-C handler"); let database_url = format!("sqlite:{}", &args.database); - let mut options = SqliteConnectOptions::from_str(&database_url)? - .create_if_missing(true) - .with_regexp(); + let mut options = + SqliteConnectOptions::from_str(&database_url)?.create_if_missing(true).with_regexp(); // Set cache to shared if using in-memory database if args.database == ":memory:" { From 7cf284c0992ebaf0f7d115dec93280a3bd1666d5 Mon Sep 17 00:00:00 2001 From: Nasr Date: Tue, 17 Sep 2024 07:25:06 -0400 Subject: [PATCH 4/9] database url --- bin/torii/src/main.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/bin/torii/src/main.rs b/bin/torii/src/main.rs index 31a133d87b..bb18dbe3ca 100644 --- a/bin/torii/src/main.rs +++ b/bin/torii/src/main.rs @@ -155,7 +155,11 @@ async fn main() -> anyhow::Result<()> { }) .expect("Error setting Ctrl-C handler"); - let database_url = format!("sqlite:{}", &args.database); + let database_url = if args.database == ":memory:" { + "sqlite:file:memorydb?mode=memory&cache=shared".to_string() + } else { + format!("sqlite:{}", &args.database) + }; let mut options = SqliteConnectOptions::from_str(&database_url)?.create_if_missing(true).with_regexp(); From 6425cbfd5df7c182ea901f1fba056cb94685a67c Mon Sep 17 00:00:00 2001 From: Nasr Date: Tue, 17 Sep 2024 07:37:24 -0400 Subject: [PATCH 5/9] chore --- bin/torii/src/main.rs | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/bin/torii/src/main.rs b/bin/torii/src/main.rs index bb18dbe3ca..818ebc29fb 100644 --- a/bin/torii/src/main.rs +++ b/bin/torii/src/main.rs @@ -155,19 +155,10 @@ async fn main() -> anyhow::Result<()> { }) .expect("Error setting Ctrl-C handler"); - let database_url = if args.database == ":memory:" { - "sqlite:file:memorydb?mode=memory&cache=shared".to_string() - } else { - format!("sqlite:{}", &args.database) - }; + let database_url = format!("sqlite:{}", &args.database); let mut options = SqliteConnectOptions::from_str(&database_url)?.create_if_missing(true).with_regexp(); - // Set cache to shared if using in-memory database - if args.database == ":memory:" { - options = options.shared_cache(true); - } - // Performance settings options = options.auto_vacuum(SqliteAutoVacuum::None); options = options.journal_mode(SqliteJournalMode::Wal); @@ -176,6 +167,8 @@ async fn main() -> anyhow::Result<()> { let pool = SqlitePoolOptions::new() .min_connections(1) .max_connections(5) + .max_lifetime(None) + .idle_timeout(None) .connect_with(options) .await?; From 80b5312116cb7459096f81447a7e21e55d1b48b7 Mon Sep 17 00:00:00 2001 From: Nasr Date: Tue, 17 Sep 2024 07:43:37 -0400 Subject: [PATCH 6/9] shared cache better --- bin/torii/src/main.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/bin/torii/src/main.rs b/bin/torii/src/main.rs index 818ebc29fb..768c46393e 100644 --- a/bin/torii/src/main.rs +++ b/bin/torii/src/main.rs @@ -155,7 +155,11 @@ async fn main() -> anyhow::Result<()> { }) .expect("Error setting Ctrl-C handler"); - let database_url = format!("sqlite:{}", &args.database); + let database_url = if args.database == ":memory:" { + "sqlite:file:memorydb?mode=memory&cache=shared".to_string() + } else { + format!("sqlite:{}", &args.database) + }; let mut options = SqliteConnectOptions::from_str(&database_url)?.create_if_missing(true).with_regexp(); @@ -167,8 +171,6 @@ async fn main() -> anyhow::Result<()> { let pool = SqlitePoolOptions::new() .min_connections(1) .max_connections(5) - .max_lifetime(None) - .idle_timeout(None) .connect_with(options) .await?; From b4dc62ac3171991fe181b97006ab59d7840f6d20 Mon Sep 17 00:00:00 2001 From: Nasr Date: Tue, 17 Sep 2024 09:01:23 -0400 Subject: [PATCH 7/9] removbe max connections --- bin/torii/src/main.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/bin/torii/src/main.rs b/bin/torii/src/main.rs index 768c46393e..1254a60b70 100644 --- a/bin/torii/src/main.rs +++ b/bin/torii/src/main.rs @@ -170,7 +170,6 @@ async fn main() -> anyhow::Result<()> { let pool = SqlitePoolOptions::new() .min_connections(1) - .max_connections(5) .connect_with(options) .await?; From 9067289793a5d79784e52b564c256de97c47c052 Mon Sep 17 00:00:00 2001 From: Nasr Date: Wed, 18 Sep 2024 12:34:27 -0400 Subject: [PATCH 8/9] shared memory is by default on --- bin/torii/src/main.rs | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/bin/torii/src/main.rs b/bin/torii/src/main.rs index 1254a60b70..c99fc8b3cc 100644 --- a/bin/torii/src/main.rs +++ b/bin/torii/src/main.rs @@ -155,23 +155,14 @@ async fn main() -> anyhow::Result<()> { }) .expect("Error setting Ctrl-C handler"); - let database_url = if args.database == ":memory:" { - "sqlite:file:memorydb?mode=memory&cache=shared".to_string() - } else { - format!("sqlite:{}", &args.database) - }; - let mut options = - SqliteConnectOptions::from_str(&database_url)?.create_if_missing(true).with_regexp(); + let mut options = SqliteConnectOptions::from_str(&args.database)?.create_if_missing(true).with_regexp(); // Performance settings options = options.auto_vacuum(SqliteAutoVacuum::None); options = options.journal_mode(SqliteJournalMode::Wal); options = options.synchronous(SqliteSynchronous::Normal); - let pool = SqlitePoolOptions::new() - .min_connections(1) - .connect_with(options) - .await?; + let pool = SqlitePoolOptions::new().min_connections(1).connect_with(options).await?; // Set the number of threads based on CPU count let cpu_count = std::thread::available_parallelism().unwrap().get(); From cd564a9cfbdeb3ccee6af910be122b11ac2c8925 Mon Sep 17 00:00:00 2001 From: Nasr Date: Wed, 18 Sep 2024 12:44:21 -0400 Subject: [PATCH 9/9] fmt --- bin/torii/src/main.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bin/torii/src/main.rs b/bin/torii/src/main.rs index c99fc8b3cc..b1447970de 100644 --- a/bin/torii/src/main.rs +++ b/bin/torii/src/main.rs @@ -155,7 +155,8 @@ async fn main() -> anyhow::Result<()> { }) .expect("Error setting Ctrl-C handler"); - let mut options = SqliteConnectOptions::from_str(&args.database)?.create_if_missing(true).with_regexp(); + let mut options = + SqliteConnectOptions::from_str(&args.database)?.create_if_missing(true).with_regexp(); // Performance settings options = options.auto_vacuum(SqliteAutoVacuum::None);