From b35d2d2e549aeb415813626c7e4a2cc293591f02 Mon Sep 17 00:00:00 2001 From: Oliver Tale-Yazdi Date: Mon, 26 Aug 2024 16:28:11 +0200 Subject: [PATCH 1/9] Maintenance Signed-off-by: Oliver Tale-Yazdi --- substrate/utils/frame/omni-bencher/Cargo.toml | 1 + substrate/utils/frame/omni-bencher/README.md | 59 +++++++++++++++++++ .../utils/frame/omni-bencher/src/command.rs | 14 +++-- .../utils/frame/omni-bencher/src/main.rs | 14 ++++- 4 files changed, 83 insertions(+), 5 deletions(-) create mode 100644 substrate/utils/frame/omni-bencher/README.md diff --git a/substrate/utils/frame/omni-bencher/Cargo.toml b/substrate/utils/frame/omni-bencher/Cargo.toml index 89ef2a48e01d..8d08b6417194 100644 --- a/substrate/utils/frame/omni-bencher/Cargo.toml +++ b/substrate/utils/frame/omni-bencher/Cargo.toml @@ -6,6 +6,7 @@ authors.workspace = true edition.workspace = true repository.workspace = true license.workspace = true +readme = "README.md" [lints] workspace = true diff --git a/substrate/utils/frame/omni-bencher/README.md b/substrate/utils/frame/omni-bencher/README.md new file mode 100644 index 000000000000..fb4caca23217 --- /dev/null +++ b/substrate/utils/frame/omni-bencher/README.md @@ -0,0 +1,59 @@ +# Polkadot Omni Benchmarking CLI + +The Polkadot Omni benchmarker allows to benchmark the extrinsics of any Polkadot runtime. It is +meant to replace the current manual integration of the `benchmark pallet` into every parachain node. +This reduces duplicate code and makes maintenance for builders easier. The CLI is currently only +able to benchmark extrinsics. In the future it is planned to extend this to some other areas. + +General FRAME runtimes could also be used with this benchmarker, as long as they don't utilize any +host functions that are not part of the Polkadot host specification. + +## Installation + +Directly via crates.io: + +```sh +cargo install frame-omni-bencher --profile=production +``` + +from GitHub: + +```sh +cargo install --git https://github.com/paritytech/polkadot-sdk frame-omni-bencher --profile=production +``` + +or locally from the sources: + +```sh +cargo install --path substrate/utils/frame/omni-bencher --profile=production +``` + +Check the installed version and print the docs: + +```sh +frame-omni-bencher --help +``` + +## Usage + +First we need to ensure that there is a runtime available. As example we will build the Westend +runtime: + +```sh +cargo build -p westend-runtime --profile production --features runtime-benchmarks +``` + +Now as an example, we benchmark the `balances` pallet: + +```sh +frame-omni-bencher v1 benchmark pallet \ +--runtime target/release/wbuild/westend-runtime/westend-runtime.compact.compressed.wasm \ +--pallet "pallet_balances" --extrinsic "" +``` + +The `--steps`, `--repeat`, `--heap-pages` and `--wasm-execution` arguments have sane defaults and do not need be passed explicitly anymore. + +## Backwards Compatibility + +The exposed pallet sub-command is identical as the node-integrated CLI. The only difference is that +it needs to be prefixed with a `v1` to ensure drop-in compatibility. diff --git a/substrate/utils/frame/omni-bencher/src/command.rs b/substrate/utils/frame/omni-bencher/src/command.rs index f0159f4307d6..19177ed549b7 100644 --- a/substrate/utils/frame/omni-bencher/src/command.rs +++ b/substrate/utils/frame/omni-bencher/src/command.rs @@ -36,13 +36,19 @@ use sp_runtime::traits::BlakeTwo256; /// Directly via crates.io: /// /// ```sh -/// cargo install --locked frame-omni-bencher +/// cargo install frame-omni-bencher --profile=production /// ``` /// -/// or when the sources are locally checked out: +/// from GitHub: /// /// ```sh -/// cargo install --locked --path substrate/utils/frame/omni-bencher --profile=production +/// cargo install --git https://github.com/paritytech/polkadot-sdk frame-omni-bencher --profile=production +/// ``` +/// +/// or locally from the sources: +/// +/// ```sh +/// cargo install --path substrate/utils/frame/omni-bencher --profile=production /// ``` /// /// Check the installed version and print the docs: @@ -60,7 +66,7 @@ use sp_runtime::traits::BlakeTwo256; /// cargo build -p westend-runtime --profile production --features runtime-benchmarks /// ``` /// -/// Now as example we benchmark `pallet_balances`: +/// Now as an example, we benchmark the `balances` pallet: /// /// ```sh /// frame-omni-bencher v1 benchmark pallet \ diff --git a/substrate/utils/frame/omni-bencher/src/main.rs b/substrate/utils/frame/omni-bencher/src/main.rs index a8893b5a79af..79afe9983493 100644 --- a/substrate/utils/frame/omni-bencher/src/main.rs +++ b/substrate/utils/frame/omni-bencher/src/main.rs @@ -19,10 +19,22 @@ mod command; use clap::Parser; use sc_cli::Result; +use tracing_subscriber::{fmt, prelude::*, EnvFilter}; fn main() -> Result<()> { - sp_tracing::try_init_simple(); + setup_logger(); + log::warn!("The FRAME omni-bencher is not yet battle tested - double check the results.",); command::Command::parse().run() } + +/// Setup logging with `info` as default level. Can be set via `RUST_LOG` env. +fn setup_logger() { + let env_filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info")); + + tracing_subscriber::fmt() + .with_env_filter(env_filter) + .with_writer(std::io::stderr) + .init(); +} From 176ca5aeff0eb2e2219deea502fc29176a4daa1a Mon Sep 17 00:00:00 2001 From: ggwpez Date: Mon, 26 Aug 2024 14:36:42 +0000 Subject: [PATCH 2/9] Add PrDoc (auto generated) --- prdoc/pr_5466.prdoc | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 prdoc/pr_5466.prdoc diff --git a/prdoc/pr_5466.prdoc b/prdoc/pr_5466.prdoc new file mode 100644 index 000000000000..a494721ae457 --- /dev/null +++ b/prdoc/pr_5466.prdoc @@ -0,0 +1,9 @@ +crates: +- bump: minor + name: frame-omni-bencher +doc: +- audience: Runtime Dev + description: "Changes:\r\n- Set default level to `Info` again. Seems like a dependency\ + \ update set it to something higher.\r\n- Fix docs to not use `--locked` since\ + \ we rely on dependency bumps via cargo.\r\n- Add README with rust docs." +title: frame-omni-bencher maintenance From f2b1ed9c19f3a6f9914b73b3d8dcc9b5ac56e105 Mon Sep 17 00:00:00 2001 From: Oliver Tale-Yazdi Date: Mon, 26 Aug 2024 16:52:40 +0200 Subject: [PATCH 3/9] Fix --heap-pages ignored on node Signed-off-by: Oliver Tale-Yazdi --- .../utils/frame/benchmarking-cli/src/pallet/command.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/substrate/utils/frame/benchmarking-cli/src/pallet/command.rs b/substrate/utils/frame/benchmarking-cli/src/pallet/command.rs index cfbbab4df7ca..471919815206 100644 --- a/substrate/utils/frame/benchmarking-cli/src/pallet/command.rs +++ b/substrate/utils/frame/benchmarking-cli/src/pallet/command.rs @@ -243,7 +243,7 @@ impl PalletCmd { let state = &state_without_tracking; let runtime = self.runtime_blob(&state_without_tracking)?; let runtime_code = runtime.code()?; - let alloc_strategy = Self::alloc_strategy(runtime_code.heap_pages); + let alloc_strategy = self.alloc_strategy(runtime_code.heap_pages); let executor = WasmExecutor::<( sp_io::SubstrateHostFunctions, @@ -753,9 +753,9 @@ impl PalletCmd { } /// Allocation strategy for pallet benchmarking. - fn alloc_strategy(heap_pages: Option) -> HeapAllocStrategy { - heap_pages.map_or(DEFAULT_HEAP_ALLOC_STRATEGY, |p| HeapAllocStrategy::Static { - extra_pages: p as _, + fn alloc_strategy(&self, runtime_heap_pages: Option) -> HeapAllocStrategy { + self.heap_pages.or(runtime_heap_pages).map_or(DEFAULT_HEAP_ALLOC_STRATEGY, |p| { + HeapAllocStrategy::Static { extra_pages: p as _ } }) } From 2e542aaf4dfbf180613b2050da5cdbbae895ce16 Mon Sep 17 00:00:00 2001 From: Oliver Tale-Yazdi Date: Mon, 26 Aug 2024 16:56:29 +0200 Subject: [PATCH 4/9] fix Signed-off-by: Oliver Tale-Yazdi --- substrate/utils/frame/omni-bencher/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/substrate/utils/frame/omni-bencher/Cargo.toml b/substrate/utils/frame/omni-bencher/Cargo.toml index 8d08b6417194..e2ffca8b4714 100644 --- a/substrate/utils/frame/omni-bencher/Cargo.toml +++ b/substrate/utils/frame/omni-bencher/Cargo.toml @@ -18,5 +18,5 @@ frame-benchmarking-cli = { workspace = true } sc-cli = { workspace = true, default-features = true } sp-runtime = { workspace = true, default-features = true } sp-statement-store = { workspace = true, default-features = true } -sp-tracing = { workspace = true } +tracing-subscriber = { workspace = true } log = { workspace = true } From 1a4a8df490fab7ef09dbf69dab68f6ca3bdd7e82 Mon Sep 17 00:00:00 2001 From: Oliver Tale-Yazdi Date: Mon, 26 Aug 2024 16:59:21 +0200 Subject: [PATCH 5/9] lockfile Signed-off-by: Oliver Tale-Yazdi --- Cargo.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 89cc7cfa8d49..35e46da1b473 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5991,7 +5991,7 @@ dependencies = [ "sc-cli", "sp-runtime", "sp-statement-store", - "sp-tracing 16.0.0", + "tracing-subscriber 0.3.18", ] [[package]] From 5f58b9540d03668a8304888a6efa24907e9e7aef Mon Sep 17 00:00:00 2001 From: Oliver Tale-Yazdi Date: Mon, 26 Aug 2024 17:26:47 +0200 Subject: [PATCH 6/9] Clippy Signed-off-by: Oliver Tale-Yazdi --- substrate/utils/frame/omni-bencher/src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/substrate/utils/frame/omni-bencher/src/main.rs b/substrate/utils/frame/omni-bencher/src/main.rs index 79afe9983493..ef3450add8e4 100644 --- a/substrate/utils/frame/omni-bencher/src/main.rs +++ b/substrate/utils/frame/omni-bencher/src/main.rs @@ -19,7 +19,7 @@ mod command; use clap::Parser; use sc_cli::Result; -use tracing_subscriber::{fmt, prelude::*, EnvFilter}; +use tracing_subscriber::EnvFilter; fn main() -> Result<()> { setup_logger(); From 813b048b4611ceca4509cf90ef4a91b2e3174c35 Mon Sep 17 00:00:00 2001 From: Oliver Tale-Yazdi Date: Mon, 26 Aug 2024 18:15:32 +0200 Subject: [PATCH 7/9] format prdoc Signed-off-by: Oliver Tale-Yazdi --- prdoc/pr_5466.prdoc | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/prdoc/pr_5466.prdoc b/prdoc/pr_5466.prdoc index a494721ae457..a76973114697 100644 --- a/prdoc/pr_5466.prdoc +++ b/prdoc/pr_5466.prdoc @@ -3,7 +3,10 @@ crates: name: frame-omni-bencher doc: - audience: Runtime Dev - description: "Changes:\r\n- Set default level to `Info` again. Seems like a dependency\ - \ update set it to something higher.\r\n- Fix docs to not use `--locked` since\ - \ we rely on dependency bumps via cargo.\r\n- Add README with rust docs." + description: | + Changes: + - Set default level to `Info` again. Seems like a dependency update set it to something higher. + - Fix docs to not use `--locked` since we rely on dependency bumps via cargo. + - Add README with rust docs. + - Fix bug where the node ignored `--heap-pages` argument. title: frame-omni-bencher maintenance From 0edba5035a8764ee5e9ffe90455ec144419345c4 Mon Sep 17 00:00:00 2001 From: Oliver Tale-Yazdi Date: Mon, 26 Aug 2024 19:55:50 +0200 Subject: [PATCH 8/9] Format markdown Signed-off-by: Oliver Tale-Yazdi --- substrate/utils/frame/omni-bencher/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/substrate/utils/frame/omni-bencher/README.md b/substrate/utils/frame/omni-bencher/README.md index fb4caca23217..29bfaeb6450b 100644 --- a/substrate/utils/frame/omni-bencher/README.md +++ b/substrate/utils/frame/omni-bencher/README.md @@ -51,7 +51,8 @@ frame-omni-bencher v1 benchmark pallet \ --pallet "pallet_balances" --extrinsic "" ``` -The `--steps`, `--repeat`, `--heap-pages` and `--wasm-execution` arguments have sane defaults and do not need be passed explicitly anymore. +The `--steps`, `--repeat`, `--heap-pages` and `--wasm-execution` arguments have sane defaults and do +not need be passed explicitly anymore. ## Backwards Compatibility From 4c05e624b2bf05ab78a3da7ed80b9dcfa9a5c007 Mon Sep 17 00:00:00 2001 From: Oliver Tale-Yazdi Date: Mon, 26 Aug 2024 21:45:16 +0200 Subject: [PATCH 9/9] prdoc Signed-off-by: Oliver Tale-Yazdi --- prdoc/pr_5466.prdoc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/prdoc/pr_5466.prdoc b/prdoc/pr_5466.prdoc index a76973114697..57f20b3585b4 100644 --- a/prdoc/pr_5466.prdoc +++ b/prdoc/pr_5466.prdoc @@ -1,6 +1,8 @@ crates: -- bump: minor +- bump: patch name: frame-omni-bencher +- bump: patch + name: frame-benchmarking-cli doc: - audience: Runtime Dev description: |