diff --git a/Cargo.lock b/Cargo.lock index b9cd4c99d2b..625ae5ddcf4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5709,7 +5709,7 @@ dependencies = [ [[package]] name = "nym-data-observatory" -version = "1.0.0" +version = "1.0.1" dependencies = [ "anyhow", "async-trait", diff --git a/common/nyxd-scraper-psql/src/storage/block_storage.rs b/common/nyxd-scraper-psql/src/storage/block_storage.rs index 49f2bab6220..186de491b29 100644 --- a/common/nyxd-scraper-psql/src/storage/block_storage.rs +++ b/common/nyxd-scraper-psql/src/storage/block_storage.rs @@ -13,7 +13,7 @@ use nyxd_scraper_shared::storage::helpers::log_db_operation_time; use nyxd_scraper_shared::storage::{NyxdScraperStorage, NyxdScraperStorageError}; use sqlx::types::time::{OffsetDateTime, PrimitiveDateTime}; use tokio::time::Instant; -use tracing::{debug, error, info, instrument, warn}; +use tracing::{debug, error, info, instrument}; #[derive(Clone)] pub struct PostgresScraperStorage { @@ -22,7 +22,10 @@ pub struct PostgresScraperStorage { impl PostgresScraperStorage { #[instrument] - pub async fn init(connection_string: &str) -> Result { + pub async fn init( + connection_string: &str, + run_migrations: &bool, + ) -> Result { debug!("initialising scraper database with '{connection_string}'",); let connection_pool = match sqlx::PgPool::connect(connection_string).await { @@ -33,12 +36,13 @@ impl PostgresScraperStorage { } }; - if let Err(err) = sqlx::migrate!("./sql_migrations") - .run(&connection_pool) - .await - { - warn!("Failed to initialize SQLx database: {err}"); - // return Err(err.into()); + if *run_migrations { + if let Err(err) = sqlx::migrate!("./sql_migrations") + .run(&connection_pool) + .await + { + return Err(err.into()); + } } info!("Database migration finished!"); @@ -192,8 +196,11 @@ impl PostgresScraperStorage { impl NyxdScraperStorage for PostgresScraperStorage { type StorageTransaction = PostgresStorageTransaction; - async fn initialise(storage: &str) -> Result { - PostgresScraperStorage::init(storage) + async fn initialise( + storage: &str, + run_migrations: &bool, + ) -> Result { + PostgresScraperStorage::init(storage, run_migrations) .await .map_err(NyxdScraperStorageError::from) } diff --git a/common/nyxd-scraper-shared/src/scraper/mod.rs b/common/nyxd-scraper-shared/src/scraper/mod.rs index df9224ae220..76b17a2a54c 100644 --- a/common/nyxd-scraper-shared/src/scraper/mod.rs +++ b/common/nyxd-scraper-shared/src/scraper/mod.rs @@ -48,6 +48,8 @@ pub struct Config { pub store_precommits: bool, pub start_block: StartingBlockOpts, + + pub run_migrations: bool, } pub struct NyxdScraperBuilder { @@ -161,7 +163,7 @@ where pub async fn new(config: Config) -> Result { config.pruning_options.validate()?; - let storage = S::initialise(&config.database_storage).await?; + let storage = S::initialise(&config.database_storage, &config.run_migrations).await?; let rpc_client = RpcClient::new(&config.rpc_url)?; Ok(NyxdScraper { diff --git a/common/nyxd-scraper-shared/src/storage/mod.rs b/common/nyxd-scraper-shared/src/storage/mod.rs index a847f2a6bf1..c0564e343fb 100644 --- a/common/nyxd-scraper-shared/src/storage/mod.rs +++ b/common/nyxd-scraper-shared/src/storage/mod.rs @@ -33,7 +33,10 @@ pub trait NyxdScraperStorage: Clone + Sized { type StorageTransaction: NyxdScraperTransaction; /// Either connection string (postgres) or storage path (sqlite) - async fn initialise(storage: &str) -> Result; + async fn initialise( + storage: &str, + run_migrations: &bool, + ) -> Result; async fn begin_processing_tx( &self, diff --git a/common/nyxd-scraper-sqlite/src/storage/block_storage.rs b/common/nyxd-scraper-sqlite/src/storage/block_storage.rs index 9a0a33d52ab..f6d81f6f00b 100644 --- a/common/nyxd-scraper-sqlite/src/storage/block_storage.rs +++ b/common/nyxd-scraper-sqlite/src/storage/block_storage.rs @@ -207,7 +207,10 @@ impl SqliteScraperStorage { impl NyxdScraperStorage for SqliteScraperStorage { type StorageTransaction = SqliteStorageTransaction; - async fn initialise(storage: &str) -> Result { + async fn initialise( + storage: &str, + _run_migrations: &bool, + ) -> Result { SqliteScraperStorage::init(storage) .await .map_err(NyxdScraperStorageError::from) diff --git a/nym-data-observatory/Cargo.toml b/nym-data-observatory/Cargo.toml index d23b40f20de..995c4fec7d4 100644 --- a/nym-data-observatory/Cargo.toml +++ b/nym-data-observatory/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "nym-data-observatory" -version = "1.0.0" +version = "1.0.1" authors.workspace = true repository.workspace = true homepage.workspace = true diff --git a/nym-data-observatory/src/chain_scraper/mod.rs b/nym-data-observatory/src/chain_scraper/mod.rs index a9cc9b9c216..49623a5ecd9 100644 --- a/nym-data-observatory/src/chain_scraper/mod.rs +++ b/nym-data-observatory/src/chain_scraper/mod.rs @@ -1,8 +1,7 @@ use crate::cli::commands::run::Args; use crate::db::DbPool; use nyxd_scraper_psql::{PostgresNyxdScraper, PruningOptions}; -use std::fs; -use tracing::{info, warn}; +use tracing::info; pub(crate) mod webhook; @@ -13,11 +12,6 @@ pub(crate) async fn run_chain_scraper( ) -> anyhow::Result { let use_best_effort_start_height = args.start_block_height.is_some(); - if args.nuke_db { - warn!("☢️☢️☢️ NUKING THE SCRAPER DATABASE"); - fs::remove_file(config.chain_scraper_connection_string())?; - } - let database_storage = config .chain_scraper_connection_string .clone() @@ -34,6 +28,7 @@ pub(crate) async fn run_chain_scraper( start_block_height: args.start_block_height, use_best_effort_start_height, }, + run_migrations: false, // ignore the base migrations }) .with_msg_module(crate::modules::wasm::WasmModule::new(connection_pool)) .with_tx_module(webhook::WebhookModule::new(config.clone())?); diff --git a/nym-data-observatory/src/cli/commands/run/args.rs b/nym-data-observatory/src/cli/commands/run/args.rs index 6d4f71b2723..bd5322988e4 100644 --- a/nym-data-observatory/src/cli/commands/run/args.rs +++ b/nym-data-observatory/src/cli/commands/run/args.rs @@ -15,9 +15,6 @@ pub(crate) struct Args { #[arg(long, env = NYXD_SCRAPER_START_HEIGHT)] pub(crate) start_block_height: Option, - #[arg(long, env = NYXD_SCRAPER_UNSAFE_NUKE_DB, default_value = "false")] - pub(crate) nuke_db: bool, - /// (Override) Postgres connection string for chain scraper history #[arg(long, env = NYM_DATA_OBSERVATORY_DB_URL, alias = "db_url")] pub(crate) db_connection_string: Option, diff --git a/nym-data-observatory/src/cli/commands/run/mod.rs b/nym-data-observatory/src/cli/commands/run/mod.rs index d6d3188cbf6..4f4927d0d2c 100644 --- a/nym-data-observatory/src/cli/commands/run/mod.rs +++ b/nym-data-observatory/src/cli/commands/run/mod.rs @@ -123,7 +123,6 @@ pub(crate) async fn execute(args: Args, http_port: u16) -> Result<(), NymDataObs w.watch_for_chain_message_types ); } - info!("nuke_db: {}", args.nuke_db); let storage = db::Storage::init(db_connection_string).await?; let watcher_pool = storage.pool_owned(); diff --git a/nym-data-observatory/src/db/mod.rs b/nym-data-observatory/src/db/mod.rs index 70c36a7b618..a8b7a7a9522 100644 --- a/nym-data-observatory/src/db/mod.rs +++ b/nym-data-observatory/src/db/mod.rs @@ -1,6 +1,7 @@ use anyhow::{Result, anyhow}; -use sqlx::{Postgres, postgres::PgConnectOptions}; +use sqlx::{Postgres, migrate::Migrator, postgres::PgConnectOptions}; use std::str::FromStr; +use tracing::info; pub(crate) mod models; pub(crate) mod queries { @@ -8,6 +9,8 @@ pub(crate) mod queries { pub mod wasm; } +static MIGRATOR: Migrator = sqlx::migrate!("./migrations"); + pub(crate) type DbPool = sqlx::Pool; pub(crate) struct Storage { @@ -22,6 +25,13 @@ impl Storage { .await .map_err(|err| anyhow!("Failed to connect to {}: {}", &connection_url, err))?; + MIGRATOR + .run(&pool) + .await + .map_err(|err| anyhow!("Failed to run migrations: {}", err))?; + + info!("✅ Successfully migrated the database"); + Ok(Storage { pool }) } diff --git a/nym-data-observatory/src/env.rs b/nym-data-observatory/src/env.rs index cff2212b57f..fe66aa492db 100644 --- a/nym-data-observatory/src/env.rs +++ b/nym-data-observatory/src/env.rs @@ -16,8 +16,6 @@ pub mod vars { pub const NYXD_SCRAPER_USE_BEST_EFFORT_START_HEIGHT: &str = "NYXD_SCRAPER_USE_BEST_EFFORT_START_HEIGHT"; - pub const NYXD_SCRAPER_UNSAFE_NUKE_DB: &str = "NYXD_SCRAPER_UNSAFE_NUKE_DB"; - pub const NYM_DATA_OBSERVATORY_ID_ARG: &str = "NYM_DATA_OBSERVATORY_ID"; pub const NYM_DATA_OBSERVATORY_OUTPUT_ARG: &str = "NYM_DATA_OBSERVATORY_OUTPUT"; diff --git a/nym-validator-rewarder/src/config/mod.rs b/nym-validator-rewarder/src/config/mod.rs index 146f971bada..524cc7df9ed 100644 --- a/nym-validator-rewarder/src/config/mod.rs +++ b/nym-validator-rewarder/src/config/mod.rs @@ -135,6 +135,7 @@ impl Config { start_block_height: None, use_best_effort_start_height: true, }, + run_migrations: true, }) } diff --git a/nyx-chain-watcher/src/chain_scraper/mod.rs b/nyx-chain-watcher/src/chain_scraper/mod.rs index c0db1b8e04d..fe0d2187ff5 100644 --- a/nyx-chain-watcher/src/chain_scraper/mod.rs +++ b/nyx-chain-watcher/src/chain_scraper/mod.rs @@ -60,6 +60,7 @@ pub(crate) async fn run_chain_scraper( start_block_height, use_best_effort_start_height, }, + run_migrations: true, }) .with_msg_module(BankScraperModule::new( db_pool,