From 918bda14ececc7f72b5741896c2c7ca8fba114f9 Mon Sep 17 00:00:00 2001 From: fdb-hiroshima <35889323+fdb-hiroshima@users.noreply.github.com> Date: Sat, 4 May 2019 15:35:21 +0200 Subject: [PATCH] rewrite circleci config (#558) * rewrite circleci config with better syntax and try using grcov for coverage * attempt to use less memory and to rerun if memory lack * actually remove use of kcov * use the right var * try looping * run test serialized * use feature where we should and make macro path independant * cargo fmt and remove some '"' * add zip to dependancies * reduce number of used thread after first failure * push translations to crowdin on commit to master * don't cache result of coverage --- .circleci/config.yml | 509 ++++++++----------- .circleci/images/plume-buildenv/Dockerfile | 20 +- .circleci/images/plume-buildenv/cargo_config | 2 +- crowdin.yml | 2 + plume-macro/src/lib.rs | 8 +- script/run_browser_test.sh | 16 +- script/run_unit_test.sh | 15 - script/upload_coverage.sh | 3 - 8 files changed, 253 insertions(+), 322 deletions(-) delete mode 100755 script/run_unit_test.sh delete mode 100755 script/upload_coverage.sh diff --git a/.circleci/config.yml b/.circleci/config.yml index d229cfa..00b77c1 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,314 +1,253 @@ version: 2.1 -aliases: - - &plume-docker - image: plumeorg/plume-buildenv:v0.0.5 - - &defaults +executors: + default: + parameters: + postgres: + type: boolean + default: false + selenium: + type: boolean + default: false docker: - - *plume-docker + - image: plumeorg/plume-buildenv:v0.0.7 + - image: <<#parameters.postgres>>circleci/postgres:9.6-alpine<><<^parameters.postgres>>alpine:latest<> + environment: + POSTGRES_USER: postgres + POSTGRES_DB: plume + - image: <<#parameters.selenium>>elgalu/selenium:latest<><<^parameters.selenium>>alpine:latest<> working_directory: ~/projects/Plume - - &postgresql - docker: - - *plume-docker - - image: circleci/postgres:9.6-alpine - environment: - POSTGRES_USER: postgres - POSTGRES_DB: plume - working_directory: ~/projects/Plume - - &selenium - docker: - - *plume-docker - - image: elgalu/selenium:latest - working_directory: ~/projects/Plume - - &postgresql_selenium - docker: - - *plume-docker - - image: circleci/postgres:9.6-alpine - environment: - POSTGRES_USER: postgres - POSTGRES_DB: plume - - image: elgalu/selenium:latest - working_directory: ~/projects/Plume - - - &attach_workspace - attach_workspace: - at: ~/projects/Plume/ - - &persist_to_workspace - persist_to_workspace: - root: ~/projects/Plume/ - paths: - - ./ - - - &env_postgresql environment: - MIGRATION_DIRECTORY: migrations/postgres - FEATURES: postgres - DATABASE_URL: postgres://postgres@localhost/plume - RUST_TEST_THREADS: 1 - - &env_sqlite - environment: - MIGRATION_DIRECTORY: migrations/sqlite - FEATURES: sqlite - DATABASE_URL: plume.sqlite3 RUST_TEST_THREADS: 1 + FEATURES: <<#parameters.postgres>>postgres<><<^parameters.postgres>>sqlite<> + DATABASE_URL: <<#parameters.postgres>>postgres://postgres@localhost/plume<><<^parameters.postgres>>plume.sqlite<> - - &restore_cache - restore_cache: - keys: - - v2-plume-notest-{{ checksum "Cargo.lock" }} - - v2-plume-notest - - &save_cache - save_cache: - key: v2-plume-notest-{{ checksum "Cargo.lock" }} - paths: + +commands: + restore_env: + description: checkout and pull cache + parameters: + cache: + type: enum + default: none + enum: ["none", "clippy", "postgres", "sqlite", "release-postgres", "release-sqlite"] + steps: + - checkout + - run: git config --global --remove-section url."ssh://git@github.com" + - restore_cache: + keys: + - v0-<< parameters.cache >>-{{ checksum "Cargo.lock" }}-{{ .Branch }} + - v0-<< parameters.cache >>-{{ checksum "Cargo.lock" }}-master + + cache: + description: push cache + parameters: + cache: + type: enum + enum: ["clippy", "postgres", "sqlite", "release-postgres", "release-sqlite"] + steps: + - save_cache: + key: v0-<< parameters.cache >>-{{ checksum "Cargo.lock" }}-{{ .Branch }} + paths: - ~/.cargo/ - ./target - - &restore_cache_web - restore_cache: - keys: - - v2-plume-web-{{ checksum "Cargo.lock" }} - - v2-plume-web - - &save_cache_web - save_cache: - key: v2-plume-web-{{ checksum "Cargo.lock" }} - paths: - - ~/.cargo/ - - ./target - - - &restore_cache_cli - restore_cache: - keys: - - v2-plume-notest-{{ checksum "Cargo.lock" }}-{{ checksum "/FEATURES" }}-cli - - v2-plume-notest-{{ checksum "Cargo.lock" }}-{{ checksum "/FEATURES" }} - - v2-plume-notest-{{ checksum "Cargo.lock" }} - - v2-plume-notest - - &save_cache_cli - save_cache: - key: v2-plume-notest-{{ checksum "Cargo.lock" }}-{{ checksum "/FEATURES" }}-cli - paths: - - ~/.cargo/ - - target - - - &restore_cache_release - restore_cache: - keys: - - v2-plume-release-{{ checksum "Cargo.lock" }} - - v2-plume-release - - v2-plume-notest-{{ checksum "Cargo.lock" }} - - v2-plume-notest - - &save_cache_release - save_cache: - key: v2-plume-release-{{ checksum "Cargo.lock" }} - paths: - - ~/.cargo - - target/release - - target/wasm32-unknown-unknown/release - - - &restore_cache_plume_dead_code - restore_cache: - keys: - - v2-plume-test-{{ checksum "/FEATURES" }}-{{ checksum "Cargo.lock" }}-plume - - v2-plume-test-{{ checksum "/FEATURES" }}-{{ checksum "Cargo.lock" }} - - v2-plume-test-{{ checksum "/FEATURES" }} - - v2-plume-notest-{{ checksum "Cargo.lock" }}-{{ checksum "/FEATURES" }} - - v2-plume-notest-{{ checksum "Cargo.lock" }} - - v2-plume-notest - - &save_cache_plume_dead_code - save_cache: - key: v2-plume-test-{{ checksum "/FEATURES" }}-{{ checksum "Cargo.lock" }}-plume - paths: - - ~/.cargo/ - - target - - - &test_cli + clippy: + description: run cargo clippy + parameters: + package: + type: string + default: plume + no_feature: + type: boolean + default: false steps: - - *attach_workspace - - run: - name: Set cache key - command: echo "$FEATURES" > /FEATURES - - *restore_cache_cli - - run: - name: clippy - command: cargo clippy --no-default-features --features="${FEATURES}" --release -p plume-cli -- -D warnings - - *save_cache_cli - - &test_unit - steps: - - *attach_workspace - - run: - name: Set cache key - command: echo "$FEATURES" > /FEATURES - - *restore_cache_plume_dead_code - - run: - name: clippy - command: cargo clippy --no-default-features --features="${FEATURES}" --release -- -D warnings - - run: - name: compile test - command: cargo test --no-default-features --features="${FEATURES}" --all --exclude plume-front --no-run || cargo test --no-default-features --features="${FEATURES}" --all --exclude plume-front --no-run - - run: - name: run test - command: ./script/run_unit_test.sh - - run: - name: upload coverage - command: ./script/upload_coverage.sh unit - - *save_cache_plume_dead_code + - run: cargo clippy <<^parameters.no_feature>>--no-default-features --features="${FEATURES}"<> --release -p <> -- -D warnings - - &test_browser + run_with_coverage: + description: run command with environment for coverage + parameters: + cmd: + type: string steps: - - *attach_workspace - - run: - name: Set cache key - command: echo "$FEATURES" > /FEATURES - - *restore_cache_plume_dead_code - - run: - name: install server - command: cargo install --debug --no-default-features --features="${FEATURES}",test --path . --force || cargo install --debug --no-default-features --features="${FEATURES}",test --path . --force - - run: - name: install plm - command: cargo install --debug --no-default-features --features="${FEATURES}" --path plume-cli --force || cargo install --debug --no-default-features --features="${FEATURES}" --path plume-cli --force - - run: - name: run test - command: ./script/run_browser_test.sh - environment: - BROWSER: firefox - - run: - name: upload coverage - command: ./script/upload_coverage.sh integration - - *save_cache_plume_dead_code + - run: | + export RUSTFLAGS="-Zprofile -Zfewer-names -Ccodegen-units=1 -Cinline-threshold=0 -Clink-dead-code -Coverflow-checks=off -Zno-landing-pads -Clink-arg=-Xlinker -Clink-arg=--no-keep-memory -Clink-arg=-Xlinker -Clink-arg=--reduce-memory-overheads" + export CARGO_INCREMENTAL=0 + << parameters.cmd >> - - &release + upload_coverage: + description: merge coverage files and upload to codecov.io + parameters: + type: + type: string steps: - - *attach_workspace - - *restore_cache_release - - run: - name: build frontend - command: cargo web deploy -p plume-front --release - - run: - name: build server - command: cargo build --release --no-default-features --features="${FEATURES}" || cargo build --release --no-default-features --features="${FEATURES}" - - run: - name: build plm - command: cargo build --release --no-default-features --features="${FEATURES}" -p plume-cli || cargo build --release --no-default-features --features="${FEATURES}" -p plume-cli - - *save_cache_release - - run: script/generate_artifact.sh - - run: script/upload_test_environment.sh - - store_artifacts: - path: plume.tar.gz - destination: plume.tar.gz + - run: zip -0 ccov.zip `find . -name 'plume*.gc*' -o -name 'plm*.gc*'` + - run: grcov ccov.zip -s . -t lcov --llvm --branch --ignore-not-existing --ignore-dir '/*' -o lcov.info + - run: bash <(curl -s https://codecov.io/bash) -f lcov.info -F <> + - run: find . -name 'plume*.gc*' -delete -o -name 'plm*.gc*' -delete + - run: rm ccov.zip lcov.info + build: + description: build a package + parameters: + package: + type: string + default: plume + release: + type: boolean + default: false + steps: + - run: | + cmd="cargo build <<#parameters.release>>--release<> --no-default-features --features="${FEATURES}" -p <> -j" + for i in 36 4 2 1 1; do + $cmd $i && exit 0 + done + exit 1 jobs: - download_deps: - <<: *defaults + cargo fmt: + executor: + name: default steps: - - checkout - - *attach_workspace - - *restore_cache - - run: git config --global --remove-section url."ssh://git@github.com" - - run: cargo fetch - - *save_cache - - *persist_to_workspace + - restore_env + - run: cargo fmt --all -- --check - cargo_fmt: - <<: *defaults + clippy: + parameters: + postgres: + type: boolean + executor: + name: default + postgres: << parameters.postgres >> steps: - - *attach_workspace - - run: cargo fmt --all -- --check + - restore_env: + cache: clippy + - clippy + - clippy: + package: plume-cli + - clippy: + package: plume-front + no_feature: true + - cache: + cache: clippy - build_web: - <<: *defaults + unit: + parameters: + postgres: + type: boolean + executor: + name: default + postgres: << parameters.postgres >> steps: - - *attach_workspace - - *restore_cache_web - - run: cargo clippy -p plume-front -- -D warnings - - run: cargo web deploy -p plume-front - - *save_cache_web - - *persist_to_workspace + - restore_env: + cache: <<#parameters.postgres>>postgres<><<^parameters.postgres>>sqlite<> + - run_with_coverage: + cmd: | + cmd="cargo test --all --exclude plume-front --exclude plume-macro --no-run --no-default-features --features=${FEATURES} -j" + for i in 36 4 2 1 1; do + $cmd $i && break + done + cargo test --all --exclude plume-front --exclude plume-macro --no-default-features --features="${FEATURES}" -j1 -- --test-threads=1 + - upload_coverage: + type: unit + - cache: + cache: <<#parameters.postgres>>postgres<><<^parameters.postgres>>sqlite<> - test_cli_postgresql: - <<: *postgresql - <<: *env_postgresql - <<: *test_cli - - test_cli_sqlite: - <<: *defaults - <<: *env_sqlite - <<: *test_cli - - test_unit_postgresql: - <<: *postgresql - <<: *env_postgresql - <<: *test_unit - - test_unit_sqlite: - <<: *defaults - <<: *env_sqlite - <<: *test_unit - - test_browser_postgresql: - <<: *postgresql_selenium - <<: *env_postgresql - <<: *test_browser - - test_browser_sqlite: - <<: *selenium - <<: *env_sqlite - <<: *test_browser - all_ok: - docker: - - image: alpine:3.7 + integration: + parameters: + postgres: + type: boolean + executor: + name: default + postgres: << parameters.postgres >> + selenium: true steps: - - run: /bin/true - release_postgresql: - <<: *defaults - <<: *env_postgresql - <<: *release - release_sqlite: - <<: *defaults - <<: *env_sqlite - <<: *release + - restore_env: + cache: <<#parameters.postgres>>postgres<><<^parameters.postgres>>sqlite<> + - run: cargo web deploy -p plume-front + - run_with_coverage: + cmd: | + cmd="cargo install --debug --no-default-features --features="${FEATURES}",test --force --path . -j" + for i in 36 4 2 1 1; do + $cmd $i && exit 0 + done + exit 1 + - run_with_coverage: + cmd: | + cmd="cargo install --debug --no-default-features --features="${FEATURES}" --force --path plume-cli -j" + for i in 36 4 2 1 1; do + $cmd $i && exit 0 + done + exit 1 + - run: + name: run test + command: ./script/run_browser_test.sh + environment: + BROWSER: firefox + - upload_coverage: + type: integration + - cache: + cache: <<#parameters.postgres>>postgres<><<^parameters.postgres>>sqlite<> + + release: + parameters: + postgres: + type: boolean + executor: + name: default + postgres: << parameters.postgres >> + steps: + - restore_env: + cache: release-<<#parameters.postgres>>postgres<><<^parameters.postgres>>sqlite<> + - run: cargo web deploy -p plume-front --release + - build: + package: plume + release: true + - build: + package: plume-cli + release: true + - cache: + cache: release-<<#parameters.postgres>>postgres<><<^parameters.postgres>>sqlite<> + - run: ./script/generate_artifact.sh + - unless: + condition: << parameters.postgres >> + steps: + - run: ./script/upload_test_environment.sh + - store_artifacts: + path: plume.tar.gz + destination: plume.tar.gz + + push translations: + executor: + name: default + steps: + - restore_env: + cache: none + - run: crowdin upload workflows: version: 2 - build_and_test: + build and test: jobs: - - download_deps - - cargo_fmt: - requires: - - download_deps - - build_web: - requires: - - download_deps - - test_cli_postgresql: - requires: - - download_deps - - test_cli_sqlite: - requires: - - download_deps - - test_unit_postgresql: - requires: - - download_deps - - test_unit_sqlite: - requires: - - download_deps - - test_browser_postgresql: - requires: - - build_web - - test_browser_sqlite: - requires: - - build_web - - all_ok: - requires: - - cargo_fmt - - test_cli_postgresql - - test_cli_sqlite - - test_unit_postgresql - - test_unit_sqlite - - test_browser_postgresql - - test_browser_sqlite - - release_postgresql: - requires: - - all_ok - - release_sqlite: - requires: - - all_ok + - cargo fmt + - clippy: + postgres: false + - clippy: + postgres: true + - unit: + postgres: false + - unit: + postgres: true + - integration: + postgres: false + - integration: + postgres: true + - release: + postgres: false + - release: + postgres: true + - push translations: + filters: + branches: + only: + - /^master/ diff --git a/.circleci/images/plume-buildenv/Dockerfile b/.circleci/images/plume-buildenv/Dockerfile index 4ab51d1..f962e18 100644 --- a/.circleci/images/plume-buildenv/Dockerfile +++ b/.circleci/images/plume-buildenv/Dockerfile @@ -3,8 +3,9 @@ ENV PATH="/root/.cargo/bin:${PATH}" #install native/circleci/build dependancies RUN apt update &&\ - apt install -y git ssh tar gzip ca-certificates &&\ - apt install -y binutils-dev build-essential cmake curl gcc gettext git libcurl4-openssl-dev libdw-dev libelf-dev libiberty-dev libpq-dev libsqlite3-dev libssl-dev make openssl pkg-config postgresql postgresql-contrib python zlib1g-dev python3-pip + apt install -y --no-install-recommends git ssh tar gzip ca-certificates default-jre&&\ + apt install -y --no-install-recommends binutils-dev build-essential cmake curl gcc gettext git libcurl4-openssl-dev libdw-dev libelf-dev libiberty-dev libpq-dev libsqlite3-dev libssl-dev make openssl pkg-config postgresql postgresql-contrib python zlib1g-dev python3-pip zip unzip &&\ + rm -rf /var/lib/apt/lists/* #install and configure rust RUN curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain nightly-2019-03-23 -y &&\ @@ -13,13 +14,10 @@ RUN curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain nightly-2019-0 #compile some deps RUN cargo install cargo-web &&\ + cargo install grcov &&\ + strip /root/.cargo/bin/* &&\ rm -fr ~/.cargo/registry -#install coverage tools -RUN curl -L https://github.com/SimonKagstrom/kcov/archive/master.tar.gz | tar xz &&\ - mkdir -p kcov-master/build && cd kcov-master/build && cmake .. && make &&\ - make install && cd ../.. && rm -rf kcov-master - #set some compilation parametters COPY cargo_config /root/.cargo/config @@ -29,3 +27,11 @@ RUN pip3 install selenium #install and configure caddy RUN curl https://getcaddy.com | bash -s personal COPY Caddyfile /Caddyfile + +#install crowdin +RUN mkdir /crowdin && cd /crowdin &&\ + curl -O https://downloads.crowdin.com/cli/v2/crowdin-cli.zip &&\ + unzip crowdin-cli.zip && rm crowdin-cli.zip &&\ + cd * && mv crowdin-cli.jar /usr/local/bin && cd && rm -rf /crowdin &&\ + /bin/echo -e '#!/bin/sh\njava -jar /usr/local/bin/crowdin-cli.jar $@' > /usr/local/bin/crowdin &&\ + chmod +x /usr/local/bin/crowdin diff --git a/.circleci/images/plume-buildenv/cargo_config b/.circleci/images/plume-buildenv/cargo_config index 10d9285..889261b 100644 --- a/.circleci/images/plume-buildenv/cargo_config +++ b/.circleci/images/plume-buildenv/cargo_config @@ -1,3 +1,3 @@ [target.x86_64-unknown-linux-gnu] # link dead code for coverage, attempt to reduce linking memory usage to not get killed -rustflags = ["-Clink-dead-code", "-Clink-args=-Xlinker --no-keep-memory -Xlinker --reduce-memory-overheads"] +rustflags = ["-Clink-args=-Xlinker --no-keep-memory -Xlinker --reduce-memory-overheads"] diff --git a/crowdin.yml b/crowdin.yml index c1319cb..075babd 100644 --- a/crowdin.yml +++ b/crowdin.yml @@ -1,3 +1,5 @@ +"project_identifier": "plume" +"api_key_env": CROWDIN_API_KEY files: - source: /po/plume/plume.pot translation: /po/plume/%two_letters_code%.po diff --git a/plume-macro/src/lib.rs b/plume-macro/src/lib.rs index 8ac082d..a1966f4 100644 --- a/plume-macro/src/lib.rs +++ b/plume-macro/src/lib.rs @@ -8,6 +8,7 @@ use proc_macro::TokenStream; use proc_macro2::TokenStream as TokenStream2; use std::fs::{read_dir, File}; use std::io::Read; +use std::path::Path; use std::str::FromStr; #[proc_macro] @@ -20,7 +21,12 @@ pub fn import_migrations(input: TokenStream) -> TokenStream { } else { "migrations" }; - let mut files = read_dir(migration_dir) + let path = Path::new(env!("CARGO_MANIFEST_DIR")) + .ancestors() + .find(|path| path.join(migration_dir).is_dir() || path.join(".git").exists()) + .expect("migrations dir not found") + .join(migration_dir); + let mut files = read_dir(path) .unwrap() .map(|dir| dir.unwrap()) .filter(|dir| dir.file_type().unwrap().is_dir()) diff --git a/script/run_browser_test.sh b/script/run_browser_test.sh index 5e93fdd..b3df8ad 100755 --- a/script/run_browser_test.sh +++ b/script/run_browser_test.sh @@ -3,16 +3,12 @@ set -eo pipefail export ROCKET_SECRET_KEY="AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" -mkdir -p "target/cov/plume" -mkdir -p "target/cov/plm" -plm='kcov --exclude-pattern=/.cargo,/usr/lib --verify target/cov/plm plm' +plm migration run +plm migration redo +plm instance new -d plume-test.local -n plume-test +plm users new -n admin -N 'Admin' -e 'email@exemple.com' -p 'password' -$plm migration run -$plm migration redo -$plm instance new -d plume-test.local -n plume-test -$plm users new -n admin -N 'Admin' -e 'email@exemple.com' -p 'password' - -kcov --exclude-pattern=/.cargo,/usr/lib --verify target/cov/plume plume & +plume & caddy -conf /Caddyfile & until curl http://localhost:7878/test/health -f; do sleep 1; done 2>/dev/null >/dev/null @@ -22,4 +18,4 @@ python3 -m unittest *.py kill -SIGINT %1 kill -SIGKILL %2 -wait +sleep 15 diff --git a/script/run_unit_test.sh b/script/run_unit_test.sh deleted file mode 100755 index e3ac665..0000000 --- a/script/run_unit_test.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash -set -eo pipefail -for file in target/debug/*-*[^\.d]; do - if [[ -x "$file" ]] - then - filename=$(basename $file) - if [[ $filename =~ ^plume_macro ]]; then - rm $file - continue - fi - mkdir -p "target/cov/$filename" - kcov --exclude-pattern=/.cargo,/usr/lib --verify "target/cov/$filename" "$file" - rm $file - fi -done diff --git a/script/upload_coverage.sh b/script/upload_coverage.sh deleted file mode 100755 index b417529..0000000 --- a/script/upload_coverage.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash -set -eo pipefail -bash <(curl -s https://codecov.io/bash) -F $1