From 49bb8cb0bc2a5e9164b2931b1ba799b4434c68a1 Mon Sep 17 00:00:00 2001 From: fdb-hiroshima <35889323+fdb-hiroshima@users.noreply.github.com> Date: Mon, 29 Apr 2019 16:30:20 +0200 Subject: [PATCH] import migrations and don't require diesel_cli for admins (#555) * import migrations via macro * panic on database not to the latest migration * add subcommand to plm * create migration that run tantivy index creation * remove diesel_cli from places it was * use our migration system for tests * create table __diesel_schema_migrations if needed --- .circleci/images/plume-buildenv/Dockerfile | 1 - Cargo.lock | 47 +++--- Cargo.toml | 2 +- Dockerfile | 2 - .../down.sql | 6 + .../up.sql | 10 ++ .../down.sql | 6 + .../up.sql | 10 ++ plume-cli/src/main.rs | 13 +- plume-cli/src/migration.rs | 59 ++++++++ plume-cli/src/search.rs | 16 +- plume-macro/Cargo.toml | 21 +++ plume-macro/src/lib.rs | 139 ++++++++++++++++++ plume-models/Cargo.toml | 8 +- plume-models/src/lib.rs | 23 +-- plume-models/src/migrations.rs | 122 +++++++++++++++ plume-models/src/search/searcher.rs | 12 ++ plume-models/tests/lib.rs | 17 +-- script/generate_artifact.sh | 1 - script/run_browser_test.sh | 5 +- script/run_unit_test.sh | 4 + src/main.rs | 17 +++ 22 files changed, 475 insertions(+), 66 deletions(-) create mode 100644 migrations/postgres/2019-04-28-201506_create_tantivy_index/down.sql create mode 100644 migrations/postgres/2019-04-28-201506_create_tantivy_index/up.sql create mode 100644 migrations/sqlite/2019-04-28-201506_create_tantivy_index/down.sql create mode 100644 migrations/sqlite/2019-04-28-201506_create_tantivy_index/up.sql create mode 100644 plume-cli/src/migration.rs create mode 100644 plume-macro/Cargo.toml create mode 100644 plume-macro/src/lib.rs create mode 100644 plume-models/src/migrations.rs diff --git a/.circleci/images/plume-buildenv/Dockerfile b/.circleci/images/plume-buildenv/Dockerfile index 90ab99e..4ab51d1 100644 --- a/.circleci/images/plume-buildenv/Dockerfile +++ b/.circleci/images/plume-buildenv/Dockerfile @@ -13,7 +13,6 @@ RUN curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain nightly-2019-0 #compile some deps RUN cargo install cargo-web &&\ - cargo install diesel_cli --no-default-features --features postgres,sqlite --version '=1.3.0' &&\ rm -fr ~/.cargo/registry #install coverage tools diff --git a/Cargo.lock b/Cargo.lock index 0d0c1f5..fd49b3d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -607,7 +607,7 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "devise_core 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", - "quote 0.6.11 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -617,7 +617,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", "proc-macro2 0.4.27 (registry+https://github.com/rust-lang/crates.io-index)", - "quote 0.6.11 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)", "syn 0.15.27 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -641,7 +641,7 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "proc-macro2 0.4.27 (registry+https://github.com/rust-lang/crates.io-index)", - "quote 0.6.11 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)", "syn 0.15.27 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -801,7 +801,7 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "proc-macro2 0.4.27 (registry+https://github.com/rust-lang/crates.io-index)", - "quote 0.6.11 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)", "syn 0.15.27 (registry+https://github.com/rust-lang/crates.io-index)", "synstructure 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1055,7 +1055,7 @@ dependencies = [ "mac 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", "markup5ever 0.7.5 (registry+https://github.com/rust-lang/crates.io-index)", "proc-macro2 0.4.27 (registry+https://github.com/rust-lang/crates.io-index)", - "quote 0.6.11 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)", "syn 0.15.27 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1716,7 +1716,7 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "proc-macro2 0.4.27 (registry+https://github.com/rust-lang/crates.io-index)", - "quote 0.6.11 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)", "syn 0.15.27 (registry+https://github.com/rust-lang/crates.io-index)", "version_check 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", "yansi 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1860,6 +1860,15 @@ dependencies = [ "stdweb 0.4.14 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "plume-macro" +version = "0.1.0" +dependencies = [ + "proc-macro2 0.4.27 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 0.11.11 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "plume-models" version = "0.3.0" @@ -1874,10 +1883,12 @@ dependencies = [ "guid-create 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", "heck 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", "itertools 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", - "lazy_static 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "migrations_internals 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "openssl 0.10.19 (registry+https://github.com/rust-lang/crates.io-index)", "plume-api 0.3.0", "plume-common 0.3.0", + "plume-macro 0.1.0", "reqwest 0.9.11 (registry+https://github.com/rust-lang/crates.io-index)", "rocket 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "rocket_i18n 0.4.0 (git+https://github.com/Plume-org/rocket_i18n?rev=e922afa7c366038b3433278c03b1456b346074f2)", @@ -1988,7 +1999,7 @@ dependencies = [ [[package]] name = "quote" -version = "0.6.11" +version = "0.6.12" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "proc-macro2 0.4.27 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2202,7 +2213,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "proc-macro2 0.4.27 (registry+https://github.com/rust-lang/crates.io-index)", - "quote 0.6.11 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)", "syn 0.15.27 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -2273,7 +2284,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "devise 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "indexmap 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", - "quote 0.6.11 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)", "rocket_http 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "version_check 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", "yansi 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2495,7 +2506,7 @@ version = "1.0.89" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "proc-macro2 0.4.27 (registry+https://github.com/rust-lang/crates.io-index)", - "quote 0.6.11 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)", "syn 0.15.27 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -2590,7 +2601,7 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "proc-macro2 0.4.27 (registry+https://github.com/rust-lang/crates.io-index)", - "quote 0.6.11 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.89 (registry+https://github.com/rust-lang/crates.io-index)", "serde_derive 1.0.89 (registry+https://github.com/rust-lang/crates.io-index)", "syn 0.15.27 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2603,7 +2614,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "base-x 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)", "proc-macro2 0.4.27 (registry+https://github.com/rust-lang/crates.io-index)", - "quote 0.6.11 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.89 (registry+https://github.com/rust-lang/crates.io-index)", "serde_derive 1.0.89 (registry+https://github.com/rust-lang/crates.io-index)", "serde_json 1.0.39 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2643,7 +2654,7 @@ dependencies = [ "phf_generator 0.7.24 (registry+https://github.com/rust-lang/crates.io-index)", "phf_shared 0.7.24 (registry+https://github.com/rust-lang/crates.io-index)", "proc-macro2 0.4.27 (registry+https://github.com/rust-lang/crates.io-index)", - "quote 0.6.11 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)", "string_cache_shared 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -2693,7 +2704,7 @@ version = "0.15.27" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "proc-macro2 0.4.27 (registry+https://github.com/rust-lang/crates.io-index)", - "quote 0.6.11 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)", "unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -2711,7 +2722,7 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "proc-macro2 0.4.27 (registry+https://github.com/rust-lang/crates.io-index)", - "quote 0.6.11 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)", "syn 0.15.27 (registry+https://github.com/rust-lang/crates.io-index)", "unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -3111,7 +3122,7 @@ dependencies = [ "if_chain 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "proc-macro2 0.4.27 (registry+https://github.com/rust-lang/crates.io-index)", - "quote 0.6.11 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)", "regex 1.1.2 (registry+https://github.com/rust-lang/crates.io-index)", "syn 0.15.27 (registry+https://github.com/rust-lang/crates.io-index)", "validator 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -3461,7 +3472,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum quote 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)" = "7a6e920b65c65f10b2ae65c831a81a073a89edd28c7cce89475bff467ab4167a" "checksum quote 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "1eca14c727ad12702eb4b6bfb5a232287dcf8385cb8ca83a3eeaf6519c44c408" "checksum quote 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "9949cfe66888ffe1d53e6ec9d9f3b70714083854be20fd5e271b232a017401e8" -"checksum quote 0.6.11 (registry+https://github.com/rust-lang/crates.io-index)" = "cdd8e04bd9c52e0342b406469d494fcb033be4bdbe5c606016defbb1681411e1" +"checksum quote 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)" = "faf4799c5d274f3868a4aae320a0a182cbd2baee377b378f080e16a23e9d80db" "checksum r2d2 0.8.3 (registry+https://github.com/rust-lang/crates.io-index)" = "5d746fc8a0dab19ccea7ff73ad535854e90ddb3b4b8cdce953dd5cd0b2e7bd22" "checksum rand 0.3.23 (registry+https://github.com/rust-lang/crates.io-index)" = "64ac302d8f83c0c1974bf758f6b041c6c8ada916fbb44a609158ca8b064cc76c" "checksum rand 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)" = "552840b97013b1a26992c11eac34bdd778e464601a4c2054b5f0bff7c6761293" diff --git a/Cargo.toml b/Cargo.toml index 98bd805..326a9f5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -77,4 +77,4 @@ debug-mailer = [] test = [] [workspace] -members = ["plume-api", "plume-cli", "plume-models", "plume-common", "plume-front"] +members = ["plume-api", "plume-cli", "plume-models", "plume-common", "plume-front", "plume-macro"] diff --git a/Dockerfile b/Dockerfile index 78bf9c4..db692b7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,7 +18,6 @@ RUN chmod a+x ./wasm-deps.sh && sleep 1 && ./wasm-deps.sh WORKDIR /app COPY Cargo.toml Cargo.lock rust-toolchain ./ -RUN cargo install diesel_cli --no-default-features --features postgres --version '=1.3.0' RUN cargo install cargo-web COPY . . @@ -40,7 +39,6 @@ WORKDIR /app COPY --from=builder /app /app COPY --from=builder /usr/local/cargo/bin/plm /bin/ COPY --from=builder /usr/local/cargo/bin/plume /bin/ -COPY --from=builder /usr/local/cargo/bin/diesel /bin/ CMD ["plume"] diff --git a/migrations/postgres/2019-04-28-201506_create_tantivy_index/down.sql b/migrations/postgres/2019-04-28-201506_create_tantivy_index/down.sql new file mode 100644 index 0000000..41b20eb --- /dev/null +++ b/migrations/postgres/2019-04-28-201506_create_tantivy_index/down.sql @@ -0,0 +1,6 @@ +-- This file should undo anything in `up.sql` +--#!|_conn, path: &Path| { +--#! let mut pb = path.to_path_buf(); +--#! pb.push("search_index"); +--#! std::fs::remove_dir_all(pb).map_err(Error::from) +--#!} diff --git a/migrations/postgres/2019-04-28-201506_create_tantivy_index/up.sql b/migrations/postgres/2019-04-28-201506_create_tantivy_index/up.sql new file mode 100644 index 0000000..2cdec21 --- /dev/null +++ b/migrations/postgres/2019-04-28-201506_create_tantivy_index/up.sql @@ -0,0 +1,10 @@ +-- Your SQL goes here +--#!|conn: &Connection, path: &Path| { +--#! let mut pb = path.to_path_buf(); +--#! pb.push("search_index"); +--#! let searcher = super::search::Searcher::create(&pb)?; +--#! searcher.fill(conn)?; +--#! searcher.commit(); +--#! Ok(()) +--#!} + diff --git a/migrations/sqlite/2019-04-28-201506_create_tantivy_index/down.sql b/migrations/sqlite/2019-04-28-201506_create_tantivy_index/down.sql new file mode 100644 index 0000000..41b20eb --- /dev/null +++ b/migrations/sqlite/2019-04-28-201506_create_tantivy_index/down.sql @@ -0,0 +1,6 @@ +-- This file should undo anything in `up.sql` +--#!|_conn, path: &Path| { +--#! let mut pb = path.to_path_buf(); +--#! pb.push("search_index"); +--#! std::fs::remove_dir_all(pb).map_err(Error::from) +--#!} diff --git a/migrations/sqlite/2019-04-28-201506_create_tantivy_index/up.sql b/migrations/sqlite/2019-04-28-201506_create_tantivy_index/up.sql new file mode 100644 index 0000000..2cdec21 --- /dev/null +++ b/migrations/sqlite/2019-04-28-201506_create_tantivy_index/up.sql @@ -0,0 +1,10 @@ +-- Your SQL goes here +--#!|conn: &Connection, path: &Path| { +--#! let mut pb = path.to_path_buf(); +--#! pb.push("search_index"); +--#! let searcher = super::search::Searcher::create(&pb)?; +--#! searcher.fill(conn)?; +--#! searcher.commit(); +--#! Ok(()) +--#!} + diff --git a/plume-cli/src/main.rs b/plume-cli/src/main.rs index 5aba00b..4a87224 100644 --- a/plume-cli/src/main.rs +++ b/plume-cli/src/main.rs @@ -10,6 +10,7 @@ use plume_models::{Connection as Conn, CONFIG}; use std::io::{self, prelude::*}; mod instance; +mod migration; mod search; mod users; @@ -19,8 +20,9 @@ fn main() { .version(env!("CARGO_PKG_VERSION")) .about("Collection of tools to manage your Plume instance.") .subcommand(instance::command()) - .subcommand(users::command()) - .subcommand(search::command()); + .subcommand(migration::command()) + .subcommand(search::command()) + .subcommand(users::command()); let matches = app.clone().get_matches(); dotenv::dotenv().ok(); @@ -30,12 +32,15 @@ fn main() { ("instance", Some(args)) => { instance::run(args, &conn.expect("Couldn't connect to the database.")) } - ("users", Some(args)) => { - users::run(args, &conn.expect("Couldn't connect to the database.")) + ("migration", Some(args)) => { + migration::run(args, &conn.expect("Couldn't connect to the database.")) } ("search", Some(args)) => { search::run(args, &conn.expect("Couldn't connect to the database.")) } + ("users", Some(args)) => { + users::run(args, &conn.expect("Couldn't connect to the database.")) + } _ => app.print_help().expect("Couldn't print help"), }; } diff --git a/plume-cli/src/migration.rs b/plume-cli/src/migration.rs new file mode 100644 index 0000000..3a147c6 --- /dev/null +++ b/plume-cli/src/migration.rs @@ -0,0 +1,59 @@ +use clap::{App, Arg, ArgMatches, SubCommand}; + +use plume_models::{migrations::IMPORTED_MIGRATIONS, Connection}; +use std::path::Path; + +pub fn command<'a, 'b>() -> App<'a, 'b> { + SubCommand::with_name("migration") + .about("Manage migrations") + .subcommand( + SubCommand::with_name("run") + .arg( + Arg::with_name("path") + .short("p") + .long("path") + .takes_value(true) + .required(false) + .help("Path to Plume's working directory"), + ) + .about("Run migrations"), + ) + .subcommand( + SubCommand::with_name("redo") + .arg( + Arg::with_name("path") + .short("p") + .long("path") + .takes_value(true) + .required(false) + .help("Path to Plume's working directory"), + ) + .about("Rerun latest migration"), + ) +} + +pub fn run<'a>(args: &ArgMatches<'a>, conn: &Connection) { + let conn = conn; + match args.subcommand() { + ("run", Some(x)) => run_(x, conn), + ("redo", Some(x)) => redo(x, conn), + ("", None) => command().print_help().unwrap(), + _ => println!("Unknown subcommand"), + } +} + +fn run_<'a>(args: &ArgMatches<'a>, conn: &Connection) { + let path = args.value_of("path").unwrap_or("."); + + IMPORTED_MIGRATIONS + .run_pending_migrations(conn, Path::new(path)) + .expect("Failed to run migrations") +} + +fn redo<'a>(args: &ArgMatches<'a>, conn: &Connection) { + let path = args.value_of("path").unwrap_or("."); + + IMPORTED_MIGRATIONS + .rerun_last_migration(conn, Path::new(path)) + .expect("Failed to rerun migrations") +} diff --git a/plume-cli/src/search.rs b/plume-cli/src/search.rs index 1e898d9..023522e 100644 --- a/plume-cli/src/search.rs +++ b/plume-cli/src/search.rs @@ -1,7 +1,6 @@ use clap::{App, Arg, ArgMatches, SubCommand}; -use diesel::{ExpressionMethods, QueryDsl, RunQueryDsl}; -use plume_models::{posts::Post, schema::posts, search::Searcher, Connection, CONFIG}; +use plume_models::{search::Searcher, Connection, CONFIG}; use std::fs::{read_dir, remove_file}; use std::io::ErrorKind; use std::path::Path; @@ -98,18 +97,7 @@ fn refill<'a>(args: &ArgMatches<'a>, conn: &Connection, searcher: Option(conn) - .expect("Post::get_recents: loading error"); - - let len = posts.len(); - for (i, post) in posts.iter().enumerate() { - println!("Importing {}/{} : {}", i + 1, len, post.title); - searcher - .update_document(conn, &post) - .expect("Couldn't import post"); - } + searcher.fill(conn).expect("Couldn't import post"); println!("Commiting result"); searcher.commit(); } diff --git a/plume-macro/Cargo.toml b/plume-macro/Cargo.toml new file mode 100644 index 0000000..b224db1 --- /dev/null +++ b/plume-macro/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "plume-macro" +version = "0.1.0" +authors = ["Trinity Pointard "] +edition = "2018" +description = "Plume procedural macros" +license = "AGPLv3" + +[lib] +proc-macro = true + +[dependencies] +proc-macro2 = "0.4" +quote = "0.6.12" +syn = "0.11.4" + + +[features] +default = [] +postgres = [] +sqlite = [] diff --git a/plume-macro/src/lib.rs b/plume-macro/src/lib.rs new file mode 100644 index 0000000..8ac082d --- /dev/null +++ b/plume-macro/src/lib.rs @@ -0,0 +1,139 @@ +#![recursion_limit = "128"] +extern crate proc_macro; +#[macro_use] +extern crate quote; +extern crate syn; + +use proc_macro::TokenStream; +use proc_macro2::TokenStream as TokenStream2; +use std::fs::{read_dir, File}; +use std::io::Read; +use std::str::FromStr; + +#[proc_macro] +pub fn import_migrations(input: TokenStream) -> TokenStream { + assert!(input.is_empty()); + let migration_dir = if cfg!(feature = "postgres") { + "migrations/postgres" + } else if cfg!(feature = "sqlite") { + "migrations/sqlite" + } else { + "migrations" + }; + let mut files = read_dir(migration_dir) + .unwrap() + .map(|dir| dir.unwrap()) + .filter(|dir| dir.file_type().unwrap().is_dir()) + .map(|dir| dir.path()) + .collect::>(); + files.sort_unstable(); + let migrations = files + .into_iter() + .map(|path| { + let mut up = path.clone(); + let mut down = path.clone(); + up.push("up.sql"); + down.push("down.sql"); + let mut up_sql = String::new(); + let mut down_sql = String::new(); + File::open(up).unwrap().read_to_string(&mut up_sql).unwrap(); + File::open(down) + .unwrap() + .read_to_string(&mut down_sql) + .unwrap(); + let name = path + .file_name() + .unwrap() + .to_str() + .unwrap() + .chars() + .filter(char::is_ascii_digit) + .take(14) + .collect::(); + (name, up_sql, down_sql) + }) + .collect::>(); + let migrations_name = migrations.iter().map(|m| &m.0).collect::>(); + let migrations_up = migrations + .iter() + .map(|m| m.1.as_str()) + .map(file_to_migration) + .collect::>(); + let migrations_down = migrations + .iter() + .map(|m| m.2.as_str()) + .map(file_to_migration) + .collect::>(); + + /* + enum Action { + Sql(&'static str), + Function(&'static Fn(&Connection, &Path) -> Result<()>) + }*/ + + quote!( + ImportedMigrations( + &[#(ComplexMigration{name: #migrations_name, up: #migrations_up, down: #migrations_down}),*] + ) + ).into() +} + +fn file_to_migration(file: &str) -> TokenStream2 { + let mut sql = true; + let mut acc = String::new(); + let mut actions = vec![]; + for line in file.lines() { + if sql { + if line.starts_with("--#!") { + if !acc.trim().is_empty() { + actions.push(quote!(Action::Sql(#acc))); + } + sql = false; + acc = line[4..].to_string(); + acc.push('\n'); + } else if line.starts_with("--") { + continue; + } else { + acc.push_str(line); + acc.push('\n'); + } + } else { + if line.starts_with("--#!") { + acc.push_str(&line[4..]); + acc.push('\n'); + } else if line.starts_with("--") { + continue; + } else { + let func: TokenStream2 = trampoline(TokenStream::from_str(&acc).unwrap().into()); + actions.push(quote!(Action::Function(&#func))); + sql = true; + acc = line.to_string(); + acc.push('\n'); + } + } + } + if !acc.trim().is_empty() { + if sql { + actions.push(quote!(Action::Sql(#acc))); + } else { + let func: TokenStream2 = trampoline(TokenStream::from_str(&acc).unwrap().into()); + actions.push(quote!(Action::Function(&#func))); + } + } + + quote!( + &[#(#actions),*] + ) +} + +/// Build a trampoline to allow reference to closure from const context +fn trampoline(closure: TokenStream2) -> TokenStream2 { + quote! { + { + fn trampoline<'a, 'b>(conn: &'a Connection, path: &'b Path) -> Result<()> { + (#closure)(conn, path) + } + trampoline + } + } +} diff --git a/plume-models/Cargo.toml b/plume-models/Cargo.toml index 4f96e1b..00fc8c8 100644 --- a/plume-models/Cargo.toml +++ b/plume-models/Cargo.toml @@ -12,6 +12,7 @@ guid-create = "0.1" heck = "0.3.0" itertools = "0.8.0" lazy_static = "*" +migrations_internals= "1.4.0" openssl = "0.10.15" rocket = "0.4.0" rocket_i18n = { git = "https://github.com/Plume-org/rocket_i18n", rev = "e922afa7c366038b3433278c03b1456b346074f2" } @@ -39,9 +40,12 @@ path = "../plume-api" [dependencies.plume-common] path = "../plume-common" +[dependencies.plume-macro] +path = "../plume-macro" + [dev-dependencies] diesel_migrations = "1.3.0" [features] -postgres = ["diesel/postgres"] -sqlite = ["diesel/sqlite"] +postgres = ["diesel/postgres", "plume-macro/postgres"] +sqlite = ["diesel/sqlite", "plume-macro/sqlite"] diff --git a/plume-models/src/lib.rs b/plume-models/src/lib.rs index 5dd1a86..fd46f61 100644 --- a/plume-models/src/lib.rs +++ b/plume-models/src/lib.rs @@ -1,6 +1,7 @@ #![feature(try_trait)] #![feature(never_type)] #![feature(custom_attribute)] +#![feature(proc_macro_hygiene)] extern crate activitypub; extern crate ammonia; @@ -14,9 +15,12 @@ extern crate heck; extern crate itertools; #[macro_use] extern crate lazy_static; +extern crate migrations_internals; extern crate openssl; extern crate plume_api; extern crate plume_common; +#[macro_use] +extern crate plume_macro; extern crate reqwest; extern crate rocket; extern crate rocket_i18n; @@ -32,10 +36,6 @@ extern crate url; extern crate webfinger; extern crate whatlang; -#[cfg(test)] -#[macro_use] -extern crate diesel_migrations; - use plume_common::activity_pub::inbox::InboxError; #[cfg(not(any(feature = "sqlite", feature = "postgres")))] @@ -302,18 +302,15 @@ mod tests { use diesel::r2d2::ConnectionManager; #[cfg(feature = "sqlite")] use diesel::{dsl::sql_query, RunQueryDsl}; + use migrations::IMPORTED_MIGRATIONS; + use plume_common::utils::random_hex; use scheduled_thread_pool::ScheduledThreadPool; use search; + use std::env::temp_dir; use std::sync::Arc; use Connection as Conn; use CONFIG; - #[cfg(feature = "sqlite")] - embed_migrations!("../migrations/sqlite"); - - #[cfg(feature = "postgres")] - embed_migrations!("../migrations/postgres"); - #[macro_export] macro_rules! part_eq { ( $x:expr, $y:expr, [$( $var:ident ),*] ) => { @@ -335,7 +332,10 @@ mod tests { .connection_customizer(Box::new(db_conn::PragmaForeignKey)) .build(ConnectionManager::::new(CONFIG.database_url.as_str())) .unwrap(); - embedded_migrations::run(&*pool.get().unwrap()).expect("Migrations error"); + let dir = temp_dir().join(format!("plume-test-{}", random_hex())); + IMPORTED_MIGRATIONS + .run_pending_migrations(&pool.get().unwrap(), &dir) + .expect("Migrations error"); pool }; } @@ -365,6 +365,7 @@ pub mod instance; pub mod likes; pub mod medias; pub mod mentions; +pub mod migrations; pub mod notifications; pub mod plume_rocket; pub mod post_authors; diff --git a/plume-models/src/migrations.rs b/plume-models/src/migrations.rs new file mode 100644 index 0000000..1d5a47f --- /dev/null +++ b/plume-models/src/migrations.rs @@ -0,0 +1,122 @@ +use Connection; +use Error; +use Result; + +use diesel::connection::{Connection as Conn, SimpleConnection}; +use migrations_internals::{setup_database, MigrationConnection}; + +use std::path::Path; + +#[allow(dead_code)] //variants might not be constructed if not required by current migrations +enum Action { + Sql(&'static str), + Function(&'static Fn(&Connection, &Path) -> Result<()>), +} + +impl Action { + fn run(&self, conn: &Connection, path: &Path) -> Result<()> { + match self { + Action::Sql(sql) => conn.batch_execute(sql).map_err(Error::from), + Action::Function(f) => f(conn, path), + } + } +} + +struct ComplexMigration { + name: &'static str, + up: &'static [Action], + down: &'static [Action], +} + +impl ComplexMigration { + fn run(&self, conn: &Connection, path: &Path) -> Result<()> { + println!("Running migration {}", self.name); + for step in self.up { + step.run(conn, path)? + } + Ok(()) + } + + fn revert(&self, conn: &Connection, path: &Path) -> Result<()> { + println!("Reverting migration {}", self.name); + for step in self.down { + step.run(conn, path)? + } + Ok(()) + } +} + +pub struct ImportedMigrations(&'static [ComplexMigration]); + +impl ImportedMigrations { + pub fn run_pending_migrations(&self, conn: &Connection, path: &Path) -> Result<()> { + use diesel::dsl::sql; + use diesel::sql_types::Bool; + use diesel::{select, RunQueryDsl}; + #[cfg(feature = "postgres")] + let schema_exists: bool = select(sql::( + "EXISTS \ + (SELECT 1 \ + FROM information_schema.tables \ + WHERE table_name = '__diesel_schema_migrations')", + )) + .get_result(conn)?; + #[cfg(feature = "sqlite")] + let schema_exists: bool = select(sql::( + "EXISTS \ + (SELECT 1 \ + FROM sqlite_master \ + WHERE type = 'table' \ + AND name = '__diesel_schema_migrations')", + )) + .get_result(conn)?; + + if !schema_exists { + setup_database(conn)?; + } + + let latest_migration = conn.latest_run_migration_version()?; + let latest_id = if let Some(migration) = latest_migration { + self.0 + .binary_search_by_key(&migration.as_str(), |mig| mig.name) + .map(|id| id + 1) + .map_err(|_| Error::NotFound)? + } else { + 0 + }; + + let to_run = &self.0[latest_id..]; + for migration in to_run { + conn.transaction(|| { + migration.run(conn, path)?; + conn.insert_new_migration(migration.name) + .map_err(Error::from) + })?; + } + Ok(()) + } + + pub fn is_pending(&self, conn: &Connection) -> Result { + let latest_migration = conn.latest_run_migration_version()?; + if let Some(migration) = latest_migration { + Ok(self.0.last().expect("no migrations found").name != migration) + } else { + Ok(true) + } + } + + pub fn rerun_last_migration(&self, conn: &Connection, path: &Path) -> Result<()> { + let latest_migration = conn.latest_run_migration_version()?; + let id = latest_migration + .and_then(|m| self.0.binary_search_by_key(&m.as_str(), |m| m.name).ok())?; + let migration = &self.0[id]; + conn.transaction(|| { + migration.revert(conn, path)?; + migration.run(conn, path) + }) + } +} + +pub const IMPORTED_MIGRATIONS: ImportedMigrations = { + import_migrations! {} +}; diff --git a/plume-models/src/search/searcher.rs b/plume-models/src/search/searcher.rs index 4433cb7..87ee6d7 100644 --- a/plume-models/src/search/searcher.rs +++ b/plume-models/src/search/searcher.rs @@ -1,9 +1,11 @@ use instance::Instance; use posts::Post; +use schema::posts; use tags::Tag; use Connection; use chrono::Datelike; +use diesel::{ExpressionMethods, QueryDsl, RunQueryDsl}; use itertools::Itertools; use std::{cmp, fs::create_dir_all, path::Path, sync::Mutex}; use tantivy::{ @@ -222,6 +224,16 @@ impl Searcher { .collect() } + pub fn fill(&self, conn: &Connection) -> Result<()> { + for post in posts::table + .filter(posts::published.eq(true)) + .load::(conn)? + { + self.update_document(conn, &post)? + } + Ok(()) + } + pub fn commit(&self) { let mut writer = self.writer.lock().unwrap(); writer.as_mut().unwrap().commit().unwrap(); diff --git a/plume-models/tests/lib.rs b/plume-models/tests/lib.rs index e58d7da..467be39 100644 --- a/plume-models/tests/lib.rs +++ b/plume-models/tests/lib.rs @@ -1,22 +1,21 @@ extern crate diesel; -#[macro_use] -extern crate diesel_migrations; - +extern crate plume_common; extern crate plume_models; use diesel::Connection; +use plume_common::utils::random_hex; +use plume_models::migrations::IMPORTED_MIGRATIONS; use plume_models::{Connection as Conn, CONFIG}; -#[cfg(feature = "sqlite")] -embed_migrations!("../migrations/sqlite"); - -#[cfg(feature = "postgres")] -embed_migrations!("../migrations/postgres"); +use std::env::temp_dir; fn db() -> Conn { let conn = Conn::establish(CONFIG.database_url.as_str()).expect("Couldn't connect to the database"); - embedded_migrations::run(&conn).expect("Couldn't run migrations"); + let dir = temp_dir().join(format!("plume-test-{}", random_hex())); + IMPORTED_MIGRATIONS + .run_pending_migrations(&conn, &dir) + .expect("Couldn't run migrations"); conn } diff --git a/script/generate_artifact.sh b/script/generate_artifact.sh index 05d190e..fabbd19 100755 --- a/script/generate_artifact.sh +++ b/script/generate_artifact.sh @@ -1,6 +1,5 @@ #!/bin/bash mkdir bin cp target/release/{plume,plm} bin -cp "$(which diesel)" bin strip -s bin/* tar -cvzf plume.tar.gz bin/ static/ migrations/$FEATURES diff --git a/script/run_browser_test.sh b/script/run_browser_test.sh index bbebc53..5e93fdd 100755 --- a/script/run_browser_test.sh +++ b/script/run_browser_test.sh @@ -7,11 +7,10 @@ mkdir -p "target/cov/plume" mkdir -p "target/cov/plm" plm='kcov --exclude-pattern=/.cargo,/usr/lib --verify target/cov/plm plm' -diesel migration run -diesel migration redo +$plm migration run +$plm migration redo $plm instance new -d plume-test.local -n plume-test $plm users new -n admin -N 'Admin' -e 'email@exemple.com' -p 'password' -$plm search init kcov --exclude-pattern=/.cargo,/usr/lib --verify target/cov/plume plume & caddy -conf /Caddyfile & diff --git a/script/run_unit_test.sh b/script/run_unit_test.sh index 5b02b10..e3ac665 100755 --- a/script/run_unit_test.sh +++ b/script/run_unit_test.sh @@ -4,6 +4,10 @@ for file in target/debug/*-*[^\.d]; do if [[ -x "$file" ]] then filename=$(basename $file) + if [[ $filename =~ ^plume_macro ]]; then + rm $file + continue + fi mkdir -p "target/cov/$filename" kcov --exclude-pattern=/.cargo,/usr/lib --verify "target/cov/$filename" "$file" rm $file diff --git a/src/main.rs b/src/main.rs index d34ff11..2711954 100644 --- a/src/main.rs +++ b/src/main.rs @@ -41,6 +41,7 @@ extern crate webfinger; use diesel::r2d2::ConnectionManager; use plume_models::{ db_conn::{DbPool, PragmaForeignKey}, + migrations::IMPORTED_MIGRATIONS, search::{Searcher as UnmanagedSearcher, SearcherError}, Connection, Error, CONFIG, }; @@ -83,6 +84,22 @@ fn init_pool() -> Option { fn main() { let dbpool = init_pool().expect("main: database pool initialization error"); + if IMPORTED_MIGRATIONS + .is_pending(&dbpool.get().unwrap()) + .unwrap_or(true) + { + panic!( + r#" +It appear your database migration does not run the migration required +by this version of Plume. To fix this, you can run migrations via +this command: + + plm migration run + +Then try to restart Plume. +"# + ) + } let workpool = ScheduledThreadPool::with_name("worker {}", num_cpus::get()); // we want a fast exit here, so #[allow(clippy::match_wild_err_arm)]