From 2bdd76abd084774652d5770147e57190a9a0d9be Mon Sep 17 00:00:00 2001 From: Felix Ableitner Date: Fri, 15 Dec 2023 11:57:31 +0100 Subject: [PATCH 01/20] Minor CI improvements (second attempt) --- .woodpecker.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.woodpecker.yml b/.woodpecker.yml index 0b8b6f8c88..86ae5e3000 100644 --- a/.woodpecker.yml +++ b/.woodpecker.yml @@ -83,7 +83,11 @@ steps: - IGNORED=$(git ls-files --cached -i --exclude-standard) - if [[ "$IGNORED" ]]; then echo "Ignored files present:\n$IGNORED\n"; exit 1; fi + # Restore build cache. This must run even when `*slow_check_paths` is not fulfilled, because otherwise + # rebuild-cache throws an error on missing files, and `when` conditions are always combined with *and*, + # so there is no way to rebuild cache only on main branch *and* slow_check_paths. restore-cache: + group: format image: meltwater/drone-cache:v1 pull: true settings: @@ -103,9 +107,9 @@ steps: - ".cargo_home" - "target" - "api_tests/node_modules" + exit_code: true secrets: [MINIO_ENDPOINT, MINIO_WRITE_USER, MINIO_WRITE_PASSWORD, MINIO_BUCKET] - when: *slow_check_paths # make sure api builds with default features (used by other crates relying on lemmy api) check_api_common_default_features: @@ -229,6 +233,7 @@ steps: - ".cargo_home" - "target" - "api_tests/node_modules" + exit_code: true secrets: [MINIO_ENDPOINT, MINIO_WRITE_USER, MINIO_WRITE_PASSWORD, MINIO_BUCKET] when: From 4cae7c98038a4265822fdef8051732a6cf71629d Mon Sep 17 00:00:00 2001 From: Felix Ableitner Date: Fri, 15 Dec 2023 11:58:54 +0100 Subject: [PATCH 02/20] test slow check condition --- scripts/compilation_benchmark.sh | 23 ----------------------- 1 file changed, 23 deletions(-) delete mode 100755 scripts/compilation_benchmark.sh diff --git a/scripts/compilation_benchmark.sh b/scripts/compilation_benchmark.sh deleted file mode 100755 index 93d1b6a23c..0000000000 --- a/scripts/compilation_benchmark.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env bash -set -e - -times=3 -duration=0 -for ((i=0; i < times; i++)) ; do - echo "Starting iteration $i" - echo "cargo clean" - # to benchmark incremental compilation time, do a full build with the same compiler version first, - # and use the following clean command: - cargo clean -p lemmy_utils - #cargo clean - echo "cargo build" - start=$(date +%s.%N) - RUSTC_WRAPPER='' cargo build -q - end=$(date +%s.%N) - echo "Finished iteration $i after $(bc <<< "scale=0; $end - $start") seconds" - duration=$(bc <<< "$duration + $end - $start") -done - -average=$(bc <<< "scale=0; $duration / $times") - -echo "Average compilation time over $times runs is $average seconds" From 077659c00b746ed6cdf6d4623da43118339e6f97 Mon Sep 17 00:00:00 2001 From: Felix Ableitner Date: Fri, 15 Dec 2023 12:10:32 +0100 Subject: [PATCH 03/20] remove rebuild cache condition for testing --- .woodpecker.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.woodpecker.yml b/.woodpecker.yml index 86ae5e3000..d78f99be8b 100644 --- a/.woodpecker.yml +++ b/.woodpecker.yml @@ -236,9 +236,9 @@ steps: exit_code: true secrets: [MINIO_ENDPOINT, MINIO_WRITE_USER, MINIO_WRITE_PASSWORD, MINIO_BUCKET] - when: - - event: push - branch: main + #when: + # - event: push + # branch: main publish_release_docker: image: woodpeckerci/plugin-docker-buildx From 263a77e53b2c3534e875b48140db610a3fb2e5c3 Mon Sep 17 00:00:00 2001 From: Felix Ableitner Date: Fri, 15 Dec 2023 12:11:25 +0100 Subject: [PATCH 04/20] trigger quick ci check --- scripts/compilation_benchmark.sh | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100755 scripts/compilation_benchmark.sh diff --git a/scripts/compilation_benchmark.sh b/scripts/compilation_benchmark.sh new file mode 100755 index 0000000000..93d1b6a23c --- /dev/null +++ b/scripts/compilation_benchmark.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +set -e + +times=3 +duration=0 +for ((i=0; i < times; i++)) ; do + echo "Starting iteration $i" + echo "cargo clean" + # to benchmark incremental compilation time, do a full build with the same compiler version first, + # and use the following clean command: + cargo clean -p lemmy_utils + #cargo clean + echo "cargo build" + start=$(date +%s.%N) + RUSTC_WRAPPER='' cargo build -q + end=$(date +%s.%N) + echo "Finished iteration $i after $(bc <<< "scale=0; $end - $start") seconds" + duration=$(bc <<< "$duration + $end - $start") +done + +average=$(bc <<< "scale=0; $duration / $times") + +echo "Average compilation time over $times runs is $average seconds" From 4189fcc582a573a968d8654d82a59548e58d3e17 Mon Sep 17 00:00:00 2001 From: Felix Ableitner Date: Tue, 19 Dec 2023 11:41:18 +0100 Subject: [PATCH 05/20] mkdir --- .woodpecker.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.woodpecker.yml b/.woodpecker.yml index d78f99be8b..147733eb52 100644 --- a/.woodpecker.yml +++ b/.woodpecker.yml @@ -211,6 +211,7 @@ steps: - cd api_tests/ - yarn - yarn api-test + - mkdir ".cargo_home" "target" "api_tests/node_modules" when: *slow_check_paths rebuild-cache: From ab518cc9195504d20bbbe6d9ea1745973c3746f8 Mon Sep 17 00:00:00 2001 From: Felix Ableitner Date: Wed, 20 Dec 2023 13:04:06 +0100 Subject: [PATCH 06/20] ls, remove steps --- .woodpecker.yml | 92 +-------------------------------------- crates/utils/translations | 2 +- 2 files changed, 2 insertions(+), 92 deletions(-) diff --git a/.woodpecker.yml b/.woodpecker.yml index 147733eb52..890f09169f 100644 --- a/.woodpecker.yml +++ b/.woodpecker.yml @@ -111,82 +111,6 @@ steps: secrets: [MINIO_ENDPOINT, MINIO_WRITE_USER, MINIO_WRITE_PASSWORD, MINIO_BUCKET] - # make sure api builds with default features (used by other crates relying on lemmy api) - check_api_common_default_features: - image: *rust_image - environment: - CARGO_HOME: .cargo_home - commands: - - cargo check --package lemmy_api_common - when: *slow_check_paths - - lemmy_api_common_doesnt_depend_on_diesel: - image: *rust_image - environment: - CARGO_HOME: .cargo_home - commands: - - "! cargo tree -p lemmy_api_common --no-default-features -i diesel" - when: *slow_check_paths - - lemmy_api_common_works_with_wasm: - image: *rust_image - environment: - CARGO_HOME: .cargo_home - commands: - - "rustup target add wasm32-unknown-unknown" - - "cargo check --target wasm32-unknown-unknown -p lemmy_api_common" - when: *slow_check_paths - - check_defaults_hjson_updated: - image: *rust_image - environment: - CARGO_HOME: .cargo_home - commands: - - export LEMMY_CONFIG_LOCATION=./config/config.hjson - - ./scripts/update_config_defaults.sh config/defaults_current.hjson - - diff config/defaults.hjson config/defaults_current.hjson - when: *slow_check_paths - - check_diesel_schema: - image: willsquire/diesel-cli - environment: - CARGO_HOME: .cargo_home - DATABASE_URL: postgres://lemmy:password@database:5432/lemmy - commands: - - diesel migration run - - diesel print-schema --config-file=diesel.toml > tmp.schema - - diff tmp.schema crates/db_schema/src/schema.rs - when: *slow_check_paths - - check_diesel_migration_revertable: - image: willsquire/diesel-cli - environment: - CARGO_HOME: .cargo_home - DATABASE_URL: postgres://lemmy:password@database:5432/lemmy - commands: - - diesel migration run - - diesel migration redo - when: *slow_check_paths - - cargo_clippy: - image: *rust_image - environment: - CARGO_HOME: .cargo_home - commands: - # when adding new clippy lints, make sure to also add them in scripts/lint.sh - - rustup component add clippy - - cargo clippy --workspace --tests --all-targets --features console -- -D warnings - when: *slow_check_paths - - cargo_build: - image: *rust_image - environment: - CARGO_HOME: .cargo_home - commands: - - cargo build - - mv target/debug/lemmy_server target/lemmy_server - when: *slow_check_paths - cargo_test: group: tests image: *rust_image @@ -197,21 +121,7 @@ steps: commands: - export LEMMY_CONFIG_LOCATION=../../config/config.hjson - cargo test --workspace --no-fail-fast - when: *slow_check_paths - - run_federation_tests: - group: tests - image: node:20-bookworm-slim - environment: - LEMMY_DATABASE_URL: postgres://lemmy:password@database:5432 - DO_WRITE_HOSTS_FILE: "1" - commands: - - apt update && apt install -y bash curl postgresql-client - - bash api_tests/prepare-drone-federation-test.sh - - cd api_tests/ - - yarn - - yarn api-test - - mkdir ".cargo_home" "target" "api_tests/node_modules" + - ls -la ".cargo_home" "target" "api_tests/node_modules" when: *slow_check_paths rebuild-cache: diff --git a/crates/utils/translations b/crates/utils/translations index 7a38baa734..15815aea74 160000 --- a/crates/utils/translations +++ b/crates/utils/translations @@ -1 +1 @@ -Subproject commit 7a38baa7341cfa9299df5a80ebe42fe298380e40 +Subproject commit 15815aea74fe97360afc03496b3ad62588649af0 From 5f8b90fa423bbc630fa86d4becedcdb8de669ec8 Mon Sep 17 00:00:00 2001 From: Nutomic Date: Thu, 21 Dec 2023 10:37:35 +0100 Subject: [PATCH 07/20] exclude cargo home from prettier --- .woodpecker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.woodpecker.yml b/.woodpecker.yml index 34da5a7c4b..f214536388 100644 --- a/.woodpecker.yml +++ b/.woodpecker.yml @@ -42,7 +42,7 @@ steps: group: format image: tmknom/prettier:3.0.0 commands: - - prettier -c . '!**/volumes' '!**/dist' '!target' '!**/translations' + - prettier -c . '!**/volumes' '!**/dist' '!target' '!**/translations' '!.cargo_home' toml_fmt: group: format From 603bc750ea7a5de9dc4174e6f7e9f24d697d5c49 Mon Sep 17 00:00:00 2001 From: Felix Ableitner Date: Thu, 21 Dec 2023 10:46:31 +0100 Subject: [PATCH 08/20] ci From ed36bfbc8812c4dcc17ef2cc218c42a0c96f4f09 Mon Sep 17 00:00:00 2001 From: Nutomic Date: Thu, 21 Dec 2023 12:10:01 +0100 Subject: [PATCH 09/20] increase timeout --- .woodpecker.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.woodpecker.yml b/.woodpecker.yml index f214536388..bda083b415 100644 --- a/.woodpecker.yml +++ b/.woodpecker.yml @@ -141,6 +141,7 @@ steps: cache_key: "rust-cache" region: us-east-1 path-style: true + backend_operation_timeout: 10m mount: - ".cargo_home" - "target" From 15707097b63015d9cb0c020ed755d0b87cd5362f Mon Sep 17 00:00:00 2001 From: Nutomic Date: Thu, 21 Dec 2023 12:32:31 +0100 Subject: [PATCH 10/20] more timeout --- .woodpecker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.woodpecker.yml b/.woodpecker.yml index bda083b415..f809f448c6 100644 --- a/.woodpecker.yml +++ b/.woodpecker.yml @@ -141,7 +141,7 @@ steps: cache_key: "rust-cache" region: us-east-1 path-style: true - backend_operation_timeout: 10m + backend_operation_timeout: 20m mount: - ".cargo_home" - "target" From 3016619ba4b55559eaa6036a99751777f7fd4519 Mon Sep 17 00:00:00 2001 From: Nutomic Date: Thu, 21 Dec 2023 12:32:55 +0100 Subject: [PATCH 11/20] even higher just for testing --- .woodpecker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.woodpecker.yml b/.woodpecker.yml index f809f448c6..df28b5f739 100644 --- a/.woodpecker.yml +++ b/.woodpecker.yml @@ -141,7 +141,7 @@ steps: cache_key: "rust-cache" region: us-east-1 path-style: true - backend_operation_timeout: 20m + backend_operation_timeout: 60m mount: - ".cargo_home" - "target" From 078a4bf2b4cfdf8240fc726caa530b823689366e Mon Sep 17 00:00:00 2001 From: Nutomic Date: Thu, 21 Dec 2023 13:28:42 +0100 Subject: [PATCH 12/20] disable compression --- .woodpecker.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.woodpecker.yml b/.woodpecker.yml index df28b5f739..d10c4c0112 100644 --- a/.woodpecker.yml +++ b/.woodpecker.yml @@ -142,6 +142,7 @@ steps: region: us-east-1 path-style: true backend_operation_timeout: 60m + compression_level: 0 mount: - ".cargo_home" - "target" From ca784919e7a8098b47651f33690276d176c12cd8 Mon Sep 17 00:00:00 2001 From: Nutomic Date: Thu, 21 Dec 2023 13:30:41 +0100 Subject: [PATCH 13/20] also increase restore timeout --- .woodpecker.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.woodpecker.yml b/.woodpecker.yml index d10c4c0112..8f7c2c00e2 100644 --- a/.woodpecker.yml +++ b/.woodpecker.yml @@ -104,6 +104,7 @@ steps: region: us-east-1 cache_key: "rust-cache" path-style: true + backend_operation_timeout: 60m mount: - ".cargo_home" - "target" From b212d5107f963fd410d37853b475e14a7c59528b Mon Sep 17 00:00:00 2001 From: Felix Ableitner Date: Thu, 21 Dec 2023 15:38:00 +0100 Subject: [PATCH 14/20] cleanup --- .woodpecker.yml | 109 +++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 98 insertions(+), 11 deletions(-) diff --git a/.woodpecker.yml b/.woodpecker.yml index 8f7c2c00e2..2d219164ed 100644 --- a/.woodpecker.yml +++ b/.woodpecker.yml @@ -42,7 +42,7 @@ steps: group: format image: tmknom/prettier:3.0.0 commands: - - prettier -c . '!**/volumes' '!**/dist' '!target' '!**/translations' '!.cargo_home' + - prettier -c . '!**/volumes' '!**/dist' '!target' '!**/translations' toml_fmt: group: format @@ -84,11 +84,8 @@ steps: - IGNORED=$(git ls-files --cached -i --exclude-standard) - if [[ "$IGNORED" ]]; then echo "Ignored files present:\n$IGNORED\n"; exit 1; fi - # Restore build cache. This must run even when `*slow_check_paths` is not fulfilled, because otherwise - # rebuild-cache throws an error on missing files, and `when` conditions are always combined with *and*, - # so there is no way to rebuild cache only on main branch *and* slow_check_paths. + # restore cache unconditionally to avoid failures in rebuild-cache restore-cache: - group: format image: meltwater/drone-cache:v1 pull: true settings: @@ -104,7 +101,7 @@ steps: region: us-east-1 cache_key: "rust-cache" path-style: true - backend_operation_timeout: 60m + backend_operation_timeout: 30m mount: - ".cargo_home" - "target" @@ -112,6 +109,83 @@ steps: exit_code: true secrets: [MINIO_ENDPOINT, MINIO_WRITE_USER, MINIO_WRITE_PASSWORD, MINIO_BUCKET] + when: *slow_check_paths + + # make sure api builds with default features (used by other crates relying on lemmy api) + check_api_common_default_features: + image: *rust_image + environment: + CARGO_HOME: .cargo_home + commands: + - cargo check --package lemmy_api_common + when: *slow_check_paths + + lemmy_api_common_doesnt_depend_on_diesel: + image: *rust_image + environment: + CARGO_HOME: .cargo_home + commands: + - "! cargo tree -p lemmy_api_common --no-default-features -i diesel" + when: *slow_check_paths + + lemmy_api_common_works_with_wasm: + image: *rust_image + environment: + CARGO_HOME: .cargo_home + commands: + - "rustup target add wasm32-unknown-unknown" + - "cargo check --target wasm32-unknown-unknown -p lemmy_api_common" + when: *slow_check_paths + + check_defaults_hjson_updated: + image: *rust_image + environment: + CARGO_HOME: .cargo_home + commands: + - export LEMMY_CONFIG_LOCATION=./config/config.hjson + - ./scripts/update_config_defaults.sh config/defaults_current.hjson + - diff config/defaults.hjson config/defaults_current.hjson + when: *slow_check_paths + + check_diesel_schema: + image: willsquire/diesel-cli + environment: + CARGO_HOME: .cargo_home + DATABASE_URL: postgres://lemmy:password@database:5432/lemmy + commands: + - diesel migration run + - diesel print-schema --config-file=diesel.toml > tmp.schema + - diff tmp.schema crates/db_schema/src/schema.rs + when: *slow_check_paths + + check_diesel_migration_revertable: + image: willsquire/diesel-cli + environment: + CARGO_HOME: .cargo_home + DATABASE_URL: postgres://lemmy:password@database:5432/lemmy + commands: + - diesel migration run + - diesel migration redo + when: *slow_check_paths + + cargo_clippy: + image: *rust_image + environment: + CARGO_HOME: .cargo_home + commands: + # when adding new clippy lints, make sure to also add them in scripts/lint.sh + - rustup component add clippy + - cargo clippy --workspace --tests --all-targets --features console -- -D warnings + when: *slow_check_paths + + cargo_build: + image: *rust_image + environment: + CARGO_HOME: .cargo_home + commands: + - cargo build + - mv target/debug/lemmy_server target/lemmy_server + when: *slow_check_paths cargo_test: group: tests @@ -123,7 +197,20 @@ steps: commands: - export LEMMY_CONFIG_LOCATION=../../config/config.hjson - cargo test --workspace --no-fail-fast - - ls -la ".cargo_home" "target" "api_tests/node_modules" + when: *slow_check_paths + + run_federation_tests: + group: tests + image: node:20-bookworm-slim + environment: + LEMMY_DATABASE_URL: postgres://lemmy:password@database:5432 + DO_WRITE_HOSTS_FILE: "1" + commands: + - apt update && apt install -y bash curl postgresql-client + - bash api_tests/prepare-drone-federation-test.sh + - cd api_tests/ + - yarn + - yarn api-test when: *slow_check_paths rebuild-cache: @@ -142,7 +229,7 @@ steps: cache_key: "rust-cache" region: us-east-1 path-style: true - backend_operation_timeout: 60m + backend_operation_timeout: 30m compression_level: 0 mount: - ".cargo_home" @@ -151,9 +238,9 @@ steps: exit_code: true secrets: [MINIO_ENDPOINT, MINIO_WRITE_USER, MINIO_WRITE_PASSWORD, MINIO_BUCKET] - #when: - # - event: push - # branch: main + when: + - event: push + branch: main publish_release_docker: image: woodpeckerci/plugin-docker-buildx From 887252a1c94b1feca84fd6afd26f266c1e241eb7 Mon Sep 17 00:00:00 2001 From: Nutomic Date: Thu, 21 Dec 2023 15:41:40 +0100 Subject: [PATCH 15/20] try again --- .woodpecker.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.woodpecker.yml b/.woodpecker.yml index 2d219164ed..b41f94296d 100644 --- a/.woodpecker.yml +++ b/.woodpecker.yml @@ -102,6 +102,7 @@ steps: cache_key: "rust-cache" path-style: true backend_operation_timeout: 30m + compression_level: 0 mount: - ".cargo_home" - "target" @@ -238,9 +239,9 @@ steps: exit_code: true secrets: [MINIO_ENDPOINT, MINIO_WRITE_USER, MINIO_WRITE_PASSWORD, MINIO_BUCKET] - when: - - event: push - branch: main + #when: + # - event: push + # branch: main publish_release_docker: image: woodpeckerci/plugin-docker-buildx From 51141974fa12f62fbf6bec0abf8e39fd4968cbc7 Mon Sep 17 00:00:00 2001 From: Nutomic Date: Thu, 21 Dec 2023 15:43:24 +0100 Subject: [PATCH 16/20] Update .woodpecker.yml --- .woodpecker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.woodpecker.yml b/.woodpecker.yml index b41f94296d..f28133b7a4 100644 --- a/.woodpecker.yml +++ b/.woodpecker.yml @@ -107,7 +107,7 @@ steps: - ".cargo_home" - "target" - "api_tests/node_modules" - exit_code: true + #exit_code: true secrets: [MINIO_ENDPOINT, MINIO_WRITE_USER, MINIO_WRITE_PASSWORD, MINIO_BUCKET] when: *slow_check_paths From 2d4cde9f5aa07a949dba7d8ac7abb0d3fc15ca03 Mon Sep 17 00:00:00 2001 From: Nutomic Date: Thu, 21 Dec 2023 16:44:39 +0100 Subject: [PATCH 17/20] Update .woodpecker.yml --- .woodpecker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.woodpecker.yml b/.woodpecker.yml index f28133b7a4..626bede124 100644 --- a/.woodpecker.yml +++ b/.woodpecker.yml @@ -230,7 +230,7 @@ steps: cache_key: "rust-cache" region: us-east-1 path-style: true - backend_operation_timeout: 30m + backend_operation_timeout: 60m compression_level: 0 mount: - ".cargo_home" From 5426c48b0da6c42a2d5d8d7ff572e7e61d04a0d2 Mon Sep 17 00:00:00 2001 From: Felix Ableitner Date: Fri, 22 Dec 2023 10:44:39 +0100 Subject: [PATCH 18/20] cleanup --- .woodpecker.yml | 60 ++++++++++++++++++++++++------------------------- 1 file changed, 29 insertions(+), 31 deletions(-) diff --git a/.woodpecker.yml b/.woodpecker.yml index 626bede124..12b9b236b4 100644 --- a/.woodpecker.yml +++ b/.woodpecker.yml @@ -84,7 +84,6 @@ steps: - IGNORED=$(git ls-files --cached -i --exclude-standard) - if [[ "$IGNORED" ]]; then echo "Ignored files present:\n$IGNORED\n"; exit 1; fi - # restore cache unconditionally to avoid failures in rebuild-cache restore-cache: image: meltwater/drone-cache:v1 pull: true @@ -107,7 +106,7 @@ steps: - ".cargo_home" - "target" - "api_tests/node_modules" - #exit_code: true + exit_code: true secrets: [MINIO_ENDPOINT, MINIO_WRITE_USER, MINIO_WRITE_PASSWORD, MINIO_BUCKET] when: *slow_check_paths @@ -214,35 +213,6 @@ steps: - yarn api-test when: *slow_check_paths - rebuild-cache: - image: meltwater/drone-cache:v1 - pull: true - settings: - rebuild: true - endpoint: - from_secret: MINIO_ENDPOINT - access-key: - from_secret: MINIO_WRITE_USER - secret-key: - from_secret: MINIO_WRITE_PASSWORD - bucket: - from_secret: MINIO_BUCKET - cache_key: "rust-cache" - region: us-east-1 - path-style: true - backend_operation_timeout: 60m - compression_level: 0 - mount: - - ".cargo_home" - - "target" - - "api_tests/node_modules" - exit_code: true - secrets: - [MINIO_ENDPOINT, MINIO_WRITE_USER, MINIO_WRITE_PASSWORD, MINIO_BUCKET] - #when: - # - event: push - # branch: main - publish_release_docker: image: woodpeckerci/plugin-docker-buildx secrets: [docker_username, docker_password] @@ -282,6 +252,34 @@ steps: when: event: tag + rebuild-cache: + image: meltwater/drone-cache:v1 + pull: true + settings: + rebuild: true + endpoint: + from_secret: MINIO_ENDPOINT + access-key: + from_secret: MINIO_WRITE_USER + secret-key: + from_secret: MINIO_WRITE_PASSWORD + bucket: + from_secret: MINIO_BUCKET + cache_key: "rust-cache" + region: us-east-1 + path-style: true + backend_operation_timeout: 60m + compression_level: 0 + mount: + - ".cargo_home" + - "target" + - "api_tests/node_modules" + exit_code: true + secrets: + [MINIO_ENDPOINT, MINIO_WRITE_USER, MINIO_WRITE_PASSWORD, MINIO_BUCKET] + when: + event: cron + notify_on_failure: image: alpine:3 commands: From d3c914675b2105500183eaff5b3421088843f3e1 Mon Sep 17 00:00:00 2001 From: Nutomic Date: Fri, 22 Dec 2023 11:24:46 +0100 Subject: [PATCH 19/20] rerun ci --- .woodpecker.yml | 97 +------------------------------------------------ 1 file changed, 2 insertions(+), 95 deletions(-) diff --git a/.woodpecker.yml b/.woodpecker.yml index 12b9b236b4..7db95803e5 100644 --- a/.woodpecker.yml +++ b/.woodpecker.yml @@ -111,73 +111,6 @@ steps: [MINIO_ENDPOINT, MINIO_WRITE_USER, MINIO_WRITE_PASSWORD, MINIO_BUCKET] when: *slow_check_paths - # make sure api builds with default features (used by other crates relying on lemmy api) - check_api_common_default_features: - image: *rust_image - environment: - CARGO_HOME: .cargo_home - commands: - - cargo check --package lemmy_api_common - when: *slow_check_paths - - lemmy_api_common_doesnt_depend_on_diesel: - image: *rust_image - environment: - CARGO_HOME: .cargo_home - commands: - - "! cargo tree -p lemmy_api_common --no-default-features -i diesel" - when: *slow_check_paths - - lemmy_api_common_works_with_wasm: - image: *rust_image - environment: - CARGO_HOME: .cargo_home - commands: - - "rustup target add wasm32-unknown-unknown" - - "cargo check --target wasm32-unknown-unknown -p lemmy_api_common" - when: *slow_check_paths - - check_defaults_hjson_updated: - image: *rust_image - environment: - CARGO_HOME: .cargo_home - commands: - - export LEMMY_CONFIG_LOCATION=./config/config.hjson - - ./scripts/update_config_defaults.sh config/defaults_current.hjson - - diff config/defaults.hjson config/defaults_current.hjson - when: *slow_check_paths - - check_diesel_schema: - image: willsquire/diesel-cli - environment: - CARGO_HOME: .cargo_home - DATABASE_URL: postgres://lemmy:password@database:5432/lemmy - commands: - - diesel migration run - - diesel print-schema --config-file=diesel.toml > tmp.schema - - diff tmp.schema crates/db_schema/src/schema.rs - when: *slow_check_paths - - check_diesel_migration_revertable: - image: willsquire/diesel-cli - environment: - CARGO_HOME: .cargo_home - DATABASE_URL: postgres://lemmy:password@database:5432/lemmy - commands: - - diesel migration run - - diesel migration redo - when: *slow_check_paths - - cargo_clippy: - image: *rust_image - environment: - CARGO_HOME: .cargo_home - commands: - # when adding new clippy lints, make sure to also add them in scripts/lint.sh - - rustup component add clippy - - cargo clippy --workspace --tests --all-targets --features console -- -D warnings - when: *slow_check_paths - cargo_build: image: *rust_image environment: @@ -187,32 +120,6 @@ steps: - mv target/debug/lemmy_server target/lemmy_server when: *slow_check_paths - cargo_test: - group: tests - image: *rust_image - environment: - LEMMY_DATABASE_URL: postgres://lemmy:password@database:5432/lemmy - RUST_BACKTRACE: "1" - CARGO_HOME: .cargo_home - commands: - - export LEMMY_CONFIG_LOCATION=../../config/config.hjson - - cargo test --workspace --no-fail-fast - when: *slow_check_paths - - run_federation_tests: - group: tests - image: node:20-bookworm-slim - environment: - LEMMY_DATABASE_URL: postgres://lemmy:password@database:5432 - DO_WRITE_HOSTS_FILE: "1" - commands: - - apt update && apt install -y bash curl postgresql-client - - bash api_tests/prepare-drone-federation-test.sh - - cd api_tests/ - - yarn - - yarn api-test - when: *slow_check_paths - publish_release_docker: image: woodpeckerci/plugin-docker-buildx secrets: [docker_username, docker_password] @@ -277,8 +184,8 @@ steps: exit_code: true secrets: [MINIO_ENDPOINT, MINIO_WRITE_USER, MINIO_WRITE_PASSWORD, MINIO_BUCKET] - when: - event: cron + #when: + # event: cron notify_on_failure: image: alpine:3 From 11f3842a5ef9318e1347f0e2554ee6514f952f63 Mon Sep 17 00:00:00 2001 From: Felix Ableitner Date: Fri, 22 Dec 2023 11:38:06 +0100 Subject: [PATCH 20/20] cleanup --- .woodpecker.yml | 154 ++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 124 insertions(+), 30 deletions(-) diff --git a/.woodpecker.yml b/.woodpecker.yml index 7db95803e5..d65c90a3aa 100644 --- a/.woodpecker.yml +++ b/.woodpecker.yml @@ -100,17 +100,84 @@ steps: region: us-east-1 cache_key: "rust-cache" path-style: true - backend_operation_timeout: 30m + backend_operation_timeout: 15m compression_level: 0 + exit_code: true mount: - ".cargo_home" - "target" - "api_tests/node_modules" - exit_code: true secrets: [MINIO_ENDPOINT, MINIO_WRITE_USER, MINIO_WRITE_PASSWORD, MINIO_BUCKET] when: *slow_check_paths + # make sure api builds with default features (used by other crates relying on lemmy api) + check_api_common_default_features: + image: *rust_image + environment: + CARGO_HOME: .cargo_home + commands: + - cargo check --package lemmy_api_common + when: *slow_check_paths + + lemmy_api_common_doesnt_depend_on_diesel: + image: *rust_image + environment: + CARGO_HOME: .cargo_home + commands: + - "! cargo tree -p lemmy_api_common --no-default-features -i diesel" + when: *slow_check_paths + + lemmy_api_common_works_with_wasm: + image: *rust_image + environment: + CARGO_HOME: .cargo_home + commands: + - "rustup target add wasm32-unknown-unknown" + - "cargo check --target wasm32-unknown-unknown -p lemmy_api_common" + when: *slow_check_paths + + check_defaults_hjson_updated: + image: *rust_image + environment: + CARGO_HOME: .cargo_home + commands: + - export LEMMY_CONFIG_LOCATION=./config/config.hjson + - ./scripts/update_config_defaults.sh config/defaults_current.hjson + - diff config/defaults.hjson config/defaults_current.hjson + when: *slow_check_paths + + check_diesel_schema: + image: willsquire/diesel-cli + environment: + CARGO_HOME: .cargo_home + DATABASE_URL: postgres://lemmy:password@database:5432/lemmy + commands: + - diesel migration run + - diesel print-schema --config-file=diesel.toml > tmp.schema + - diff tmp.schema crates/db_schema/src/schema.rs + when: *slow_check_paths + + check_diesel_migration_revertable: + image: willsquire/diesel-cli + environment: + CARGO_HOME: .cargo_home + DATABASE_URL: postgres://lemmy:password@database:5432/lemmy + commands: + - diesel migration run + - diesel migration redo + when: *slow_check_paths + + cargo_clippy: + image: *rust_image + environment: + CARGO_HOME: .cargo_home + commands: + # when adding new clippy lints, make sure to also add them in scripts/lint.sh + - rustup component add clippy + - cargo clippy --workspace --tests --all-targets --features console -- -D warnings + when: *slow_check_paths + cargo_build: image: *rust_image environment: @@ -120,6 +187,61 @@ steps: - mv target/debug/lemmy_server target/lemmy_server when: *slow_check_paths + cargo_test: + group: tests + image: *rust_image + environment: + LEMMY_DATABASE_URL: postgres://lemmy:password@database:5432/lemmy + RUST_BACKTRACE: "1" + CARGO_HOME: .cargo_home + commands: + - export LEMMY_CONFIG_LOCATION=../../config/config.hjson + - cargo test --workspace --no-fail-fast + when: *slow_check_paths + + run_federation_tests: + group: tests + image: node:20-bookworm-slim + environment: + LEMMY_DATABASE_URL: postgres://lemmy:password@database:5432 + DO_WRITE_HOSTS_FILE: "1" + commands: + - apt update && apt install -y bash curl postgresql-client + - bash api_tests/prepare-drone-federation-test.sh + - cd api_tests/ + - yarn + - yarn api-test + when: *slow_check_paths + + rebuild-cache: + image: meltwater/drone-cache:v1 + pull: true + settings: + rebuild: true + endpoint: + from_secret: MINIO_ENDPOINT + access-key: + from_secret: MINIO_WRITE_USER + secret-key: + from_secret: MINIO_WRITE_PASSWORD + bucket: + from_secret: MINIO_BUCKET + cache_key: "rust-cache" + region: us-east-1 + path-style: true + backend_operation_timeout: 60m + compression_level: 0 + exit_code: true + mount: + - ".cargo_home" + - "target" + - "api_tests/node_modules" + secrets: + [MINIO_ENDPOINT, MINIO_WRITE_USER, MINIO_WRITE_PASSWORD, MINIO_BUCKET] + when: + - event: push + branch: main + publish_release_docker: image: woodpeckerci/plugin-docker-buildx secrets: [docker_username, docker_password] @@ -159,34 +281,6 @@ steps: when: event: tag - rebuild-cache: - image: meltwater/drone-cache:v1 - pull: true - settings: - rebuild: true - endpoint: - from_secret: MINIO_ENDPOINT - access-key: - from_secret: MINIO_WRITE_USER - secret-key: - from_secret: MINIO_WRITE_PASSWORD - bucket: - from_secret: MINIO_BUCKET - cache_key: "rust-cache" - region: us-east-1 - path-style: true - backend_operation_timeout: 60m - compression_level: 0 - mount: - - ".cargo_home" - - "target" - - "api_tests/node_modules" - exit_code: true - secrets: - [MINIO_ENDPOINT, MINIO_WRITE_USER, MINIO_WRITE_PASSWORD, MINIO_BUCKET] - #when: - # event: cron - notify_on_failure: image: alpine:3 commands: