From 6a6b53c466242ad56cc226c0398b8a4d074423cf Mon Sep 17 00:00:00 2001 From: ThouCheese Date: Tue, 21 Nov 2023 15:44:31 +0100 Subject: [PATCH 1/2] Modify todo example to use postgres and diesel_async --- examples/todo/Cargo.toml | 8 +++--- examples/todo/Rocket.toml | 6 ++-- examples/todo/db/DB_LIVES_HERE | 1 + examples/todo/src/main.rs | 52 ++++++++++++++++++++-------------- examples/todo/src/task.rs | 41 ++++++++++++--------------- examples/todo/src/tests.rs | 29 +++++++++++-------- 6 files changed, 74 insertions(+), 63 deletions(-) diff --git a/examples/todo/Cargo.toml b/examples/todo/Cargo.toml index 3aac187464..086b3a1a05 100644 --- a/examples/todo/Cargo.toml +++ b/examples/todo/Cargo.toml @@ -7,16 +7,16 @@ publish = false [dependencies] rocket = { path = "../../core/lib" } -diesel = { version = "2.0.0", features = ["sqlite", "r2d2"] } +diesel = { version = "2.0.0", features = ["postgres", "r2d2"] } diesel_migrations = "2.0.0" [dev-dependencies] parking_lot = "0.12" rand = "0.8" -[dependencies.rocket_sync_db_pools] -path = "../../contrib/sync_db_pools/lib/" -features = ["diesel_sqlite_pool"] +[dependencies.rocket_db_pools] +path = "../../contrib/db_pools/lib/" +features = ["diesel_postgres"] [dependencies.rocket_dyn_templates] path = "../../contrib/dyn_templates" diff --git a/examples/todo/Rocket.toml b/examples/todo/Rocket.toml index 48620aa16d..1a6dc1e002 100644 --- a/examples/todo/Rocket.toml +++ b/examples/todo/Rocket.toml @@ -1,5 +1,7 @@ [default] template_dir = "static" -[default.databases.sqlite_database] -url = "db/db.sqlite" +[default.databases.epic_todo_database] +url = "postgresql://postgres@localhost:5432/epic_todo_database" +max_connections = 1 +connect_timeout = 5 diff --git a/examples/todo/db/DB_LIVES_HERE b/examples/todo/db/DB_LIVES_HERE index e69de29bb2..1ab912506a 100644 --- a/examples/todo/db/DB_LIVES_HERE +++ b/examples/todo/db/DB_LIVES_HERE @@ -0,0 +1 @@ +db does not live here :( diff --git a/examples/todo/src/main.rs b/examples/todo/src/main.rs index 96763545aa..149ae28840 100644 --- a/examples/todo/src/main.rs +++ b/examples/todo/src/main.rs @@ -1,6 +1,4 @@ #[macro_use] extern crate rocket; -#[macro_use] extern crate rocket_sync_db_pools; -#[macro_use] extern crate diesel; #[cfg(test)] mod tests; @@ -13,13 +11,15 @@ use rocket::response::{Flash, Redirect}; use rocket::serde::Serialize; use rocket::form::Form; use rocket::fs::{FileServer, relative}; +use rocket_db_pools::{Connection, Database}; use rocket_dyn_templates::Template; use crate::task::{Task, Todo}; -#[database("sqlite_database")] -pub struct DbConn(diesel::SqliteConnection); +#[derive(Database)] +#[database("epic_todo_database")] +pub struct Db(rocket_db_pools::diesel::PgPool); #[derive(Debug, Serialize)] #[serde(crate = "rocket::serde")] @@ -29,14 +29,14 @@ struct Context { } impl Context { - pub async fn err(conn: &DbConn, msg: M) -> Context { + pub async fn err(conn: &mut Connection, msg: M) -> Context { Context { flash: Some(("error".into(), msg.to_string())), tasks: Task::all(conn).await.unwrap_or_default() } } - pub async fn raw(conn: &DbConn, flash: Option<(String, String)>) -> Context { + pub async fn raw(conn: &mut Connection, flash: Option<(String, String)>) -> Context { match Task::all(conn).await { Ok(tasks) => Context { flash, tasks }, Err(e) => { @@ -51,11 +51,11 @@ impl Context { } #[post("/", data = "")] -async fn new(todo_form: Form, conn: DbConn) -> Flash { +async fn new(todo_form: Form, mut conn: Connection) -> Flash { let todo = todo_form.into_inner(); if todo.description.is_empty() { Flash::error(Redirect::to("/"), "Description cannot be empty.") - } else if let Err(e) = Task::insert(todo, &conn).await { + } else if let Err(e) = Task::insert(todo, &mut conn).await { error_!("DB insertion error: {}", e); Flash::error(Redirect::to("/"), "Todo could not be inserted due an internal error.") } else { @@ -64,42 +64,50 @@ async fn new(todo_form: Form, conn: DbConn) -> Flash { } #[put("/")] -async fn toggle(id: i32, conn: DbConn) -> Result { - match Task::toggle_with_id(id, &conn).await { +async fn toggle(id: i32, mut conn: Connection) -> Result { + match Task::toggle_with_id(id, &mut conn).await { Ok(_) => Ok(Redirect::to("/")), Err(e) => { error_!("DB toggle({}) error: {}", id, e); - Err(Template::render("index", Context::err(&conn, "Failed to toggle task.").await)) + Err(Template::render("index", Context::err(&mut conn, "Failed to toggle task.").await)) } } } #[delete("/")] -async fn delete(id: i32, conn: DbConn) -> Result, Template> { - match Task::delete_with_id(id, &conn).await { +async fn delete(id: i32, mut conn: Connection) -> Result, Template> { + match Task::delete_with_id(id, &mut conn).await { Ok(_) => Ok(Flash::success(Redirect::to("/"), "Todo was deleted.")), Err(e) => { error_!("DB deletion({}) error: {}", id, e); - Err(Template::render("index", Context::err(&conn, "Failed to delete task.").await)) + Err(Template::render("index", Context::err(&mut conn, "Failed to delete task.").await)) } } } #[get("/")] -async fn index(flash: Option>, conn: DbConn) -> Template { +async fn index(flash: Option>, mut conn: Connection) -> Template { let flash = flash.map(FlashMessage::into_inner); - Template::render("index", Context::raw(&conn, flash).await) + Template::render("index", Context::raw(&mut conn, flash).await) } async fn run_migrations(rocket: Rocket) -> Rocket { + use diesel::Connection; use diesel_migrations::{embed_migrations, EmbeddedMigrations, MigrationHarness}; const MIGRATIONS: EmbeddedMigrations = embed_migrations!("migrations"); - - DbConn::get_one(&rocket).await - .expect("database connection") - .run(|conn| { conn.run_pending_migrations(MIGRATIONS).expect("diesel migrations"); }) - .await; + let config: rocket_db_pools::Config = rocket + .figment() + .extract_inner("databases.epic_todo_database") + .expect("Db not configured"); + + rocket::tokio::task::spawn_blocking(move || { + diesel::PgConnection::establish(&config.url) + .expect("No database") + .run_pending_migrations(MIGRATIONS) + .expect("Invalid migrations"); + }) + .await.expect("tokio doesn't work"); rocket } @@ -107,7 +115,7 @@ async fn run_migrations(rocket: Rocket) -> Rocket { #[launch] fn rocket() -> _ { rocket::build() - .attach(DbConn::fairing()) + .attach(Db::init()) .attach(Template::fairing()) .attach(AdHoc::on_ignite("Run Migrations", run_migrations)) .mount("/", FileServer::from(relative!("static"))) diff --git a/examples/todo/src/task.rs b/examples/todo/src/task.rs index eb93ffe74c..976063b9de 100644 --- a/examples/todo/src/task.rs +++ b/examples/todo/src/task.rs @@ -1,8 +1,9 @@ use rocket::serde::Serialize; use diesel::{self, result::QueryResult, prelude::*}; +use rocket_db_pools::diesel::RunQueryDsl; mod schema { - table! { + diesel::table! { tasks { id -> Nullable, description -> Text, @@ -13,7 +14,7 @@ mod schema { use self::schema::tasks; -use crate::DbConn; +type DbConn = rocket_db_pools::diesel::AsyncPgConnection; #[derive(Serialize, Queryable, Insertable, Debug, Clone)] #[serde(crate = "rocket::serde")] @@ -31,41 +32,35 @@ pub struct Todo { } impl Task { - pub async fn all(conn: &DbConn) -> QueryResult> { - conn.run(|c| { - tasks::table.order(tasks::id.desc()).load::(c) - }).await + pub async fn all(conn: &mut DbConn) -> QueryResult> { + tasks::table.order(tasks::id.desc()).load::(conn).await } /// Returns the number of affected rows: 1. - pub async fn insert(todo: Todo, conn: &DbConn) -> QueryResult { - conn.run(|c| { - let t = Task { id: None, description: todo.description, completed: false }; - diesel::insert_into(tasks::table).values(&t).execute(c) - }).await + pub async fn insert(todo: Todo, conn: &mut DbConn) -> QueryResult { + let t = Task { id: None, description: todo.description, completed: false }; + diesel::insert_into(tasks::table).values(&t).execute(conn).await } /// Returns the number of affected rows: 1. - pub async fn toggle_with_id(id: i32, conn: &DbConn) -> QueryResult { - conn.run(move |c| { - let task = tasks::table.filter(tasks::id.eq(id)).get_result::(c)?; - let new_status = !task.completed; - let updated_task = diesel::update(tasks::table.filter(tasks::id.eq(id))); - updated_task.set(tasks::completed.eq(new_status)).execute(c) - }).await + pub async fn toggle_with_id(id: i32, conn: &mut DbConn) -> QueryResult { + let task = tasks::table.filter(tasks::id.eq(id)).get_result::(conn).await?; + let new_status = !task.completed; + let updated_task = diesel::update(tasks::table.filter(tasks::id.eq(id))); + updated_task.set(tasks::completed.eq(new_status)).execute(conn).await } /// Returns the number of affected rows: 1. - pub async fn delete_with_id(id: i32, conn: &DbConn) -> QueryResult { - conn.run(move |c| diesel::delete(tasks::table) + pub async fn delete_with_id(id: i32, conn: &mut DbConn) -> QueryResult { + diesel::delete(tasks::table) .filter(tasks::id.eq(id)) - .execute(c)) + .execute(conn) .await } /// Returns the number of affected rows. #[cfg(test)] - pub async fn delete_all(conn: &DbConn) -> QueryResult { - conn.run(|c| diesel::delete(tasks::table).execute(c)).await + pub async fn delete_all(conn: &mut DbConn) -> QueryResult { + diesel::delete(tasks::table).execute(conn).await } } diff --git a/examples/todo/src/tests.rs b/examples/todo/src/tests.rs index 5c2eadd7bd..4f185c1b84 100644 --- a/examples/todo/src/tests.rs +++ b/examples/todo/src/tests.rs @@ -4,6 +4,7 @@ use rand::{Rng, thread_rng, distributions::Alphanumeric}; use rocket::local::asynchronous::Client; use rocket::http::{Status, ContentType}; +use rocket_db_pools::Database; // We use a lock to synchronize between tests so DB operations don't collide. // For now. In the future, we'll have a nice way to run each test in a DB @@ -15,10 +16,14 @@ macro_rules! run_test { let _lock = DB_LOCK.lock(); rocket::async_test(async move { - let $client = Client::tracked(super::rocket()).await.expect("Rocket client"); - let db = super::DbConn::get_one($client.rocket()).await; - let $conn = db.expect("failed to get database connection for testing"); - Task::delete_all(&$conn).await.expect("failed to delete all tasks for testing"); + let rocket = super::rocket(); + let mut $conn = super::Db::fetch(&rocket) + .expect("database") + .get() + .await + .expect("database connection"); + let $client = Client::tracked(rocket).await.expect("Rocket client"); + Task::delete_all(&mut $conn).await.expect("failed to delete all tasks for testing"); $block }) @@ -39,7 +44,7 @@ fn test_index() { fn test_insertion_deletion() { run_test!(|client, conn| { // Get the tasks before making changes. - let init_tasks = Task::all(&conn).await.unwrap(); + let init_tasks = Task::all(&mut conn).await.unwrap(); // Issue a request to insert a new task. client.post("/todo") @@ -49,7 +54,7 @@ fn test_insertion_deletion() { .await; // Ensure we have one more task in the database. - let new_tasks = Task::all(&conn).await.unwrap(); + let new_tasks = Task::all(&mut conn).await.unwrap(); assert_eq!(new_tasks.len(), init_tasks.len() + 1); // Ensure the task is what we expect. @@ -61,7 +66,7 @@ fn test_insertion_deletion() { client.delete(format!("/todo/{}", id)).dispatch().await; // Ensure it's gone. - let final_tasks = Task::all(&conn).await.unwrap(); + let final_tasks = Task::all(&mut conn).await.unwrap(); assert_eq!(final_tasks.len(), init_tasks.len()); if final_tasks.len() > 0 { assert_ne!(final_tasks[0].description, "My first task"); @@ -79,16 +84,16 @@ fn test_toggle() { .dispatch() .await; - let task = Task::all(&conn).await.unwrap()[0].clone(); + let task = Task::all(&mut conn).await.unwrap()[0].clone(); assert_eq!(task.completed, false); // Issue a request to toggle the task; ensure it is completed. client.put(format!("/todo/{}", task.id.unwrap())).dispatch().await; - assert_eq!(Task::all(&conn).await.unwrap()[0].completed, true); + assert_eq!(Task::all(&mut conn).await.unwrap()[0].completed, true); // Issue a request to toggle the task; ensure it's not completed again. client.put(format!("/todo/{}", task.id.unwrap())).dispatch().await; - assert_eq!(Task::all(&conn).await.unwrap()[0].completed, false); + assert_eq!(Task::all(&mut conn).await.unwrap()[0].completed, false); }) } @@ -98,7 +103,7 @@ fn test_many_insertions() { run_test!(|client, conn| { // Get the number of tasks initially. - let init_num = Task::all(&conn).await.unwrap().len(); + let init_num = Task::all(&mut conn).await.unwrap().len(); let mut descs = Vec::new(); for i in 0..ITER { @@ -119,7 +124,7 @@ fn test_many_insertions() { descs.insert(0, desc); // Ensure the task was inserted properly and all other tasks remain. - let tasks = Task::all(&conn).await.unwrap(); + let tasks = Task::all(&mut conn).await.unwrap(); assert_eq!(tasks.len(), init_num + i + 1); for j in 0..i { From a9843797d649e975fe04fdae1dbd12cccc461b27 Mon Sep 17 00:00:00 2001 From: Eric Dattore Date: Sat, 25 Nov 2023 19:37:27 -0500 Subject: [PATCH 2/2] Run postgres on CI - Start the preinstalled postgres service on the all executors - Note: This will work until macos-13 becomes the default since it's seemingly not installed on those runner images - Create `rocket_runner` user - Create the `epic_todo_database` DB - Update SQL migrations to match Postgres dialect (different from SQLite or MySQL) - Update user/pass configuration for Postgres user/db --- .github/workflows/ci.yml | 47 +++++++++++++++++-- examples/todo/Rocket.toml | 2 +- .../down.sql | 6 +++ .../up.sql | 36 ++++++++++++++ .../20160720150332_create_tasks_table/up.sql | 8 ++-- examples/todo/src/tests.rs | 4 +- 6 files changed, 92 insertions(+), 11 deletions(-) create mode 100644 examples/todo/migrations/00000000000000_diesel_initial_setup/down.sql create mode 100644 examples/todo/migrations/00000000000000_diesel_initial_setup/up.sql diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a7a0f5cd61..4f88d2b30d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,6 +4,7 @@ on: [push, pull_request] env: CARGO_TERM_COLOR: always + VCPKG_BINARY_SOURCES: "clear;x-gha,readwrite" jobs: test: @@ -63,15 +64,20 @@ jobs: brew install mysql-client libpq sqlite coreutils echo "/usr/local/opt/mysql-client/bin" >> "$GITHUB_PATH" + - name: Export GitHub Actions cache environment variables (Windows, vcpkg) + if: matrix.platform.name == 'Windows' + uses: actions/github-script@v6 + with: + script: | + core.exportVariable('ACTIONS_CACHE_URL', process.env.ACTIONS_CACHE_URL || ''); + core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || ''); + # vcpkg --triplet x64-windows install libmysql libpq sqlite3 openssl # + vcpkg/installed/vcpkg (in particular, the status file) - name: Install Native Dependencies (Windows) if: matrix.platform.name == 'Windows' run: | - curl -fsS -o vcpkg.7z https://rocket.rs/static/vcpkg-2019-07-05.7z - 7z x vcpkg.7z -y -bb0 - xcopy .\vcpkg $env:VCPKG_INSTALLATION_ROOT /s /e /h /y /q - vcpkg integrate install + vcpkg --triplet x64-windows install libmysql libpq sqlite3 openssl echo "VCPKGRS_DYNAMIC=1" >> "$env:GITHUB_ENV" echo "VCPKG_ROOT=$env:VCPKG_INSTALLATION_ROOT" >> "$env:GITHUB_ENV" echo "$env:VCPKG_INSTALLATION_ROOT\installed\x64-windows\lib" >> "$env:GITHUB_PATH" @@ -82,6 +88,39 @@ jobs: sudo apt-get update sudo apt-get install -y libmysqlclient-dev libpq-dev libsqlite3-dev + - name: Start Postgres (macOS) + if: matrix.platform.name == 'macOS' + run: | + brew services start postgresql@14 + RETRIES=5 + until pg_isready > /dev/null 2>&1 || [[ $RETRIES -eq 0 ]]; do + echo "waiting for Postgres to start, $((RETRIES--)) remaining attempts" + sleep 1 + done + psql postgres -c "CREATE ROLE rocket_runner PASSWORD 'password' SUPERUSER CREATEDB INHERIT LOGIN" + createdb -O rocket_runner epic_todo_database + + - name: Start Postgres (Linux) + if: matrix.platform.name == 'Linux' + run: | + sudo systemctl start postgresql.service + RETRIES=5 + until pg_isready > /dev/null 2>&1 || [[ $RETRIES -eq 0 ]]; do + echo "waiting for Postgres to start, $((RETRIES--)) remaining attempts" + sleep 1 + done + sudo -u postgres psql -U postgres -c "CREATE ROLE rocket_runner PASSWORD 'password' SUPERUSER CREATEDB INHERIT LOGIN" + sudo -u postgres createdb -O rocket_runner epic_todo_database + + - name: Start Postgres (Windows) + if: matrix.platform.name == 'Windows' + run: | + $pgService = Get-Service -Name postgresql* + Set-Service -InputObject $pgService -Status running -StartupType automatic + Start-Process -FilePath "$env:PGBIN\pg_isready" -Wait -PassThru + & $env:PGBIN\psql --command "CREATE ROLE rocket_runner PASSWORD 'password' SUPERUSER CREATEDB INHERIT LOGIN" + & $env:PGBIN\createdb -O rocket_runner epic_todo_database + - name: Install Rust uses: dtolnay/rust-toolchain@master id: toolchain diff --git a/examples/todo/Rocket.toml b/examples/todo/Rocket.toml index 1a6dc1e002..2a63a103e8 100644 --- a/examples/todo/Rocket.toml +++ b/examples/todo/Rocket.toml @@ -2,6 +2,6 @@ template_dir = "static" [default.databases.epic_todo_database] -url = "postgresql://postgres@localhost:5432/epic_todo_database" +url = "postgresql://rocket_runner:password@localhost:5432/epic_todo_database" max_connections = 1 connect_timeout = 5 diff --git a/examples/todo/migrations/00000000000000_diesel_initial_setup/down.sql b/examples/todo/migrations/00000000000000_diesel_initial_setup/down.sql new file mode 100644 index 0000000000..a9f5260911 --- /dev/null +++ b/examples/todo/migrations/00000000000000_diesel_initial_setup/down.sql @@ -0,0 +1,6 @@ +-- This file was automatically created by Diesel to setup helper functions +-- and other internal bookkeeping. This file is safe to edit, any future +-- changes will be added to existing projects as new migrations. + +DROP FUNCTION IF EXISTS diesel_manage_updated_at(_tbl regclass); +DROP FUNCTION IF EXISTS diesel_set_updated_at(); diff --git a/examples/todo/migrations/00000000000000_diesel_initial_setup/up.sql b/examples/todo/migrations/00000000000000_diesel_initial_setup/up.sql new file mode 100644 index 0000000000..d68895b1a7 --- /dev/null +++ b/examples/todo/migrations/00000000000000_diesel_initial_setup/up.sql @@ -0,0 +1,36 @@ +-- This file was automatically created by Diesel to setup helper functions +-- and other internal bookkeeping. This file is safe to edit, any future +-- changes will be added to existing projects as new migrations. + + + + +-- Sets up a trigger for the given table to automatically set a column called +-- `updated_at` whenever the row is modified (unless `updated_at` was included +-- in the modified columns) +-- +-- # Example +-- +-- ```sql +-- CREATE TABLE users (id SERIAL PRIMARY KEY, updated_at TIMESTAMP NOT NULL DEFAULT NOW()); +-- +-- SELECT diesel_manage_updated_at('users'); +-- ``` +CREATE OR REPLACE FUNCTION diesel_manage_updated_at(_tbl regclass) RETURNS VOID AS $$ +BEGIN + EXECUTE format('CREATE TRIGGER set_updated_at BEFORE UPDATE ON %s + FOR EACH ROW EXECUTE PROCEDURE diesel_set_updated_at()', _tbl); +END; +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION diesel_set_updated_at() RETURNS trigger AS $$ +BEGIN + IF ( + NEW IS DISTINCT FROM OLD AND + NEW.updated_at IS NOT DISTINCT FROM OLD.updated_at + ) THEN + NEW.updated_at := current_timestamp; + END IF; + RETURN NEW; +END; +$$ LANGUAGE plpgsql; diff --git a/examples/todo/migrations/20160720150332_create_tasks_table/up.sql b/examples/todo/migrations/20160720150332_create_tasks_table/up.sql index 65948c474b..fce35238f1 100644 --- a/examples/todo/migrations/20160720150332_create_tasks_table/up.sql +++ b/examples/todo/migrations/20160720150332_create_tasks_table/up.sql @@ -1,8 +1,8 @@ CREATE TABLE tasks ( - id INTEGER PRIMARY KEY AUTOINCREMENT, + id SERIAL PRIMARY KEY, description VARCHAR NOT NULL, - completed BOOLEAN NOT NULL DEFAULT 0 + completed BOOLEAN NOT NULL DEFAULT FALSE ); -INSERT INTO tasks (description) VALUES ("demo task"); -INSERT INTO tasks (description) VALUES ("demo task2"); +INSERT INTO tasks (description) VALUES ('demo task'); +INSERT INTO tasks (description) VALUES ('demo task2'); diff --git a/examples/todo/src/tests.rs b/examples/todo/src/tests.rs index 4f185c1b84..69c23544d8 100644 --- a/examples/todo/src/tests.rs +++ b/examples/todo/src/tests.rs @@ -16,13 +16,13 @@ macro_rules! run_test { let _lock = DB_LOCK.lock(); rocket::async_test(async move { - let rocket = super::rocket(); + let $client = Client::tracked(super::rocket()).await.expect("Rocket client"); + let rocket = $client.rocket(); let mut $conn = super::Db::fetch(&rocket) .expect("database") .get() .await .expect("database connection"); - let $client = Client::tracked(rocket).await.expect("Rocket client"); Task::delete_all(&mut $conn).await.expect("failed to delete all tasks for testing"); $block