Merge branch 'main' into l1-random-walker

crates/subspace-farmer/Cargo.toml

@@ -25,6 +25,7 @@ criterion = { version = "0.5.1", default-features = false, features = ["rayon",
 derive_more = "0.99.17"
 event-listener-primitives = "2.0.1"
 fdlimit = "0.3.0"
+fs4 = "0.7.0"
 futures = "0.3.29"
 hex = { version = "0.4.3", features = ["serde"] }
 jsonrpsee = { version = "0.16.3", features = ["client"] }

crates/subspace-farmer/src/single_disk_farm.rs

@@ -88,6 +88,12 @@ impl SingleDiskFarmId {
     }
 }
 
+/// Exclusive lock for single disk farm info file, ensuring no concurrent edits by cooperating processes is done
+#[must_use = "Lock file must be kept around or as long as farm is used"]
+pub struct SingleDiskFarmInfoLock {
+    _file: File,
+}
+
 /// Important information about the contents of the `SingleDiskFarm`
 #[derive(Debug, Clone, Serialize, Deserialize)]
 #[serde(rename_all = "camelCase")]
@@ -162,6 +168,15 @@ impl SingleDiskFarmInfo {
         )
     }
 
+    /// Try to acquire exclusive lock on the single disk farm info file, ensuring no concurrent edits by cooperating
+    /// processes is done
+    pub fn try_lock(directory: &Path) -> io::Result<SingleDiskFarmInfoLock> {
+        let file = File::open(directory.join(Self::FILE_NAME))?;
+        fs4::FileExt::try_lock_exclusive(&file)?;
+
+        Ok(SingleDiskFarmInfoLock { _file: file })
+    }
+
     // ID of the farm
     pub fn id(&self) -> &SingleDiskFarmId {
         let Self::V0 { id, .. } = self;
@@ -287,6 +302,9 @@ pub enum SingleDiskFarmError {
     /// Failed to open or create identity
     #[error("Failed to open or create identity: {0}")]
     FailedToOpenIdentity(#[from] IdentityError),
+    /// Farm is likely already in use, make sure no other farmer is using it
+    #[error("Farm is likely already in use, make sure no other farmer is using it: {0}")]
+    LikelyAlreadyInUse(io::Error),
     // TODO: Make more variants out of this generic one
     /// I/O error occurred
     #[error("I/O error: {0}")]
@@ -378,6 +396,9 @@ pub enum SingleDiskFarmError {
 /// Errors happening during scrubbing
 #[derive(Debug, Error)]
 pub enum SingleDiskFarmScrubError {
+    /// Farm is likely already in use, make sure no other farmer is using it
+    #[error("Farm is likely already in use, make sure no other farmer is using it: {0}")]
+    LikelyAlreadyInUse(io::Error),
     /// Failed to determine file size
     #[error("Failed to file size of {file}: {error}")]
     FailedToDetermineFileSize {
@@ -559,6 +580,7 @@ pub struct SingleDiskFarm {
     start_sender: Option<broadcast::Sender<()>>,
     /// Sender that will be used to signal to background threads that they must stop
     stop_sender: Option<broadcast::Sender<()>>,
+    _single_disk_farm_info_lock: SingleDiskFarmInfoLock,
 }
 
 impl Drop for SingleDiskFarm {
@@ -684,6 +706,9 @@ impl SingleDiskFarm {
             }
         };
 
+        let single_disk_farm_info_lock = SingleDiskFarmInfo::try_lock(&directory)
+            .map_err(SingleDiskFarmError::LikelyAlreadyInUse)?;
+
         let pieces_in_sector = single_disk_farm_info.pieces_in_sector();
         let sector_size = sector_size(pieces_in_sector);
         let sector_metadata_size = SectorMetadataChecksummed::encoded_size();
@@ -745,7 +770,6 @@ impl SingleDiskFarm {
             }
         };
 
-        // TODO: Consider file locking to prevent other apps from modifying it
         let mut metadata_file = OpenOptions::new()
             .read(true)
             .write(true)
@@ -1195,6 +1219,7 @@ impl SingleDiskFarm {
             piece_reader,
             start_sender: Some(start_sender),
             stop_sender: Some(stop_sender),
+            _single_disk_farm_info_lock: single_disk_farm_info_lock,
         };
 
         Ok(farm)
@@ -1446,6 +1471,10 @@ impl SingleDiskFarm {
                 }
             }
         };
+
+        let _single_disk_farm_info_lock = SingleDiskFarmInfo::try_lock(directory)
+            .map_err(SingleDiskFarmScrubError::LikelyAlreadyInUse)?;
+
         let identity = {
             let file = directory.join(Identity::FILE_NAME);
             info!(path = %file.display(), "Checking identity file");

crates/subspace-malicious-operator/src/bin/subspace-malicious-operator.rs

@@ -34,9 +34,9 @@ use sp_core::traits::SpawnEssentialNamed;
 use sp_messenger::messages::ChainId;
 use subspace_malicious_operator::malicious_domain_instance_starter::DomainInstanceStarter;
 use subspace_node::domain::DomainCli;
-use subspace_node::{Cli, ExecutorDispatch};
+use subspace_node::Cli;
 use subspace_proof_of_space::chia::ChiaTable;
-use subspace_runtime::{Block, RuntimeApi};
+use subspace_runtime::{Block, ExecutorDispatch, RuntimeApi};
 use subspace_service::{DsnConfig, SubspaceConfiguration, SubspaceNetworking};
 
 type PosTable = ChiaTable;

crates/subspace-malicious-operator/src/malicious_domain_instance_starter.rs

@@ -5,6 +5,7 @@ use domain_eth_service::provider::EthProvider;
 use domain_eth_service::DefaultEthConfig;
 use domain_runtime_primitives::opaque::Block as DomainBlock;
 use domain_service::{FullBackend, FullClient};
+use evm_domain_runtime::ExecutorDispatch as EVMDomainExecutorDispatch;
 use futures::StreamExt;
 use sc_cli::CliConfiguration;
 use sc_consensus_subspace::block_import::BlockImportingNotification;
@@ -18,11 +19,8 @@ use sp_core::traits::SpawnEssentialNamed;
 use sp_domains::{DomainInstanceData, RuntimeType};
 use sp_keystore::KeystorePtr;
 use std::sync::Arc;
-use subspace_node::domain::{
-    create_configuration, evm_chain_spec, AccountId20, DomainCli, EVMDomainExecutorDispatch,
-};
-use subspace_node::ExecutorDispatch as CExecutorDispatch;
-use subspace_runtime::RuntimeApi as CRuntimeApi;
+use subspace_node::domain::{create_configuration, evm_chain_spec, AccountId20, DomainCli};
+use subspace_runtime::{ExecutorDispatch as CExecutorDispatch, RuntimeApi as CRuntimeApi};
 use subspace_runtime_primitives::opaque::Block as CBlock;
 use subspace_service::FullClient as CFullClient;
 

crates/subspace-node/src/bin/subspace-node.rs

@@ -22,6 +22,7 @@ static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc;
 use cross_domain_message_gossip::GossipWorkerBuilder;
 use domain_client_operator::Bootstrapper;
 use domain_runtime_primitives::opaque::Block as DomainBlock;
+use evm_domain_runtime::ExecutorDispatch as EVMDomainExecutorDispatch;
 use frame_benchmarking_cli::BenchmarkCmd;
 use futures::future::TryFutureExt;
 use log::warn;
@@ -37,12 +38,10 @@ use sp_core::traits::SpawnEssentialNamed;
 use sp_io::SubstrateHostFunctions;
 use sp_messenger::messages::ChainId;
 use sp_wasm_interface::ExtendedHostFunctions;
-use subspace_node::domain::{
-    DomainCli, DomainInstanceStarter, DomainSubcommand, EVMDomainExecutorDispatch,
-};
-use subspace_node::{Cli, ExecutorDispatch, Subcommand};
+use subspace_node::domain::{DomainCli, DomainInstanceStarter, DomainSubcommand};
+use subspace_node::{Cli, Subcommand};
 use subspace_proof_of_space::chia::ChiaTable;
-use subspace_runtime::{Block, RuntimeApi};
+use subspace_runtime::{Block, ExecutorDispatch, RuntimeApi};
 use subspace_service::{DsnConfig, SubspaceConfiguration, SubspaceNetworking};
 
 type PosTable = ChiaTable;

crates/subspace-node/src/domain.rs

@@ -21,22 +21,3 @@ pub mod evm_chain_spec;
 pub use self::cli::{DomainCli, Subcommand as DomainSubcommand};
 pub use self::domain_instance_starter::{create_configuration, DomainInstanceStarter};
 pub use evm_domain_runtime::AccountId as AccountId20;
-use sc_executor::NativeExecutionDispatch;
-
-/// EVM domain executor instance.
-pub struct EVMDomainExecutorDispatch;
-
-impl NativeExecutionDispatch for EVMDomainExecutorDispatch {
-    #[cfg(feature = "runtime-benchmarks")]
-    type ExtendHostFunctions = frame_benchmarking::benchmarking::HostFunctions;
-    #[cfg(not(feature = "runtime-benchmarks"))]
-    type ExtendHostFunctions = ();
-
-    fn dispatch(method: &str, data: &[u8]) -> Option<Vec<u8>> {
-        evm_domain_runtime::api::dispatch(method, data)
-    }
-
-    fn native_version() -> sc_executor::NativeVersion {
-        evm_domain_runtime::native_version()
-    }
-}

crates/subspace-node/src/domain/domain_instance_starter.rs

@@ -1,12 +1,12 @@
 use super::{evm_chain_spec, DomainCli};
-use crate::domain::{AccountId20, EVMDomainExecutorDispatch};
-use crate::ExecutorDispatch as CExecutorDispatch;
+use crate::domain::AccountId20;
 use cross_domain_message_gossip::{ChainTxPoolMsg, Message};
 use domain_client_operator::{BootstrapResult, OperatorStreams};
 use domain_eth_service::provider::EthProvider;
 use domain_eth_service::DefaultEthConfig;
 use domain_runtime_primitives::opaque::Block as DomainBlock;
 use domain_service::{FullBackend, FullClient};
+use evm_domain_runtime::ExecutorDispatch as EVMDomainExecutorDispatch;
 use futures::StreamExt;
 use sc_chain_spec::ChainSpec;
 use sc_cli::{CliConfiguration, Database, DefaultConfigurationValues, SubstrateCli};
@@ -19,7 +19,7 @@ use sc_transaction_pool_api::OffchainTransactionPoolFactory;
 use sc_utils::mpsc::{TracingUnboundedReceiver, TracingUnboundedSender};
 use sp_domains::{DomainInstanceData, RuntimeType};
 use std::sync::Arc;
-use subspace_runtime::RuntimeApi as CRuntimeApi;
+use subspace_runtime::{ExecutorDispatch as CExecutorDispatch, RuntimeApi as CRuntimeApi};
 use subspace_runtime_primitives::opaque::Block as CBlock;
 use subspace_service::FullClient as CFullClient;
 

crates/subspace-node/src/lib.rs

@@ -22,7 +22,6 @@ pub mod domain;
 
 use clap::Parser;
 use sc_cli::{RunCmd, SubstrateCli};
-use sc_executor::NativeExecutionDispatch;
 use sc_service::ChainSpec;
 use sc_storage_monitor::StorageMonitorParams;
 use sc_subspace_chain_specs::ConsensusChainSpec;
@@ -33,33 +32,6 @@ use std::io::Write;
 use std::{fs, io};
 use subspace_networking::libp2p::Multiaddr;
 
-/// Executor dispatch for subspace runtime
-pub struct ExecutorDispatch;
-
-impl NativeExecutionDispatch for ExecutorDispatch {
-    /// Only enable the benchmarking host functions when we actually want to benchmark.
-    #[cfg(feature = "runtime-benchmarks")]
-    type ExtendHostFunctions = (
-        frame_benchmarking::benchmarking::HostFunctions,
-        sp_consensus_subspace::consensus::HostFunctions,
-        sp_domains_fraud_proof::HostFunctions,
-    );
-    /// Otherwise we only use the default Substrate host functions.
-    #[cfg(not(feature = "runtime-benchmarks"))]
-    type ExtendHostFunctions = (
-        sp_consensus_subspace::consensus::HostFunctions,
-        sp_domains_fraud_proof::HostFunctions,
-    );
-
-    fn dispatch(method: &str, data: &[u8]) -> Option<Vec<u8>> {
-        subspace_runtime::api::dispatch(method, data)
-    }
-
-    fn native_version() -> sc_executor::NativeVersion {
-        subspace_runtime::native_version()
-    }
-}
-
 /// This `purge-chain` command used to remove both consensus chain and domain.
 #[derive(Debug, Clone, Parser)]
 #[group(skip)]

crates/subspace-runtime/Cargo.toml

@@ -40,6 +40,7 @@ pallet-transaction-payment-rpc-runtime-api = { version = "4.0.0-dev", default-fe
 pallet-transporter = { version = "0.1.0", path = "../../domains/pallets/transporter", default-features = false }
 pallet-utility = { version = "4.0.0-dev", default-features = false, git = "https://github.com/subspace/polkadot-sdk", rev = "0831dfc3c54b10ab46e82acf98603b4af1a47bd5" }
 scale-info = { version = "2.7.0", default-features = false, features = ["derive"] }
+sc-executor = { version = "0.10.0-dev", git = "https://github.com/subspace/polkadot-sdk", rev = "0831dfc3c54b10ab46e82acf98603b4af1a47bd5", optional = true }
 sp-api = { version = "4.0.0-dev", default-features = false, git = "https://github.com/subspace/polkadot-sdk", rev = "0831dfc3c54b10ab46e82acf98603b4af1a47bd5" }
 sp-block-builder = { git = "https://github.com/subspace/polkadot-sdk", rev = "0831dfc3c54b10ab46e82acf98603b4af1a47bd5", default-features = false, version = "4.0.0-dev" }
 sp-consensus-subspace = { version = "0.1.0", default-features = false, path = "../sp-consensus-subspace" }
@@ -95,6 +96,7 @@ std = [
     "pallet-transporter/std",
     "pallet-utility/std",
     "scale-info/std",
+    "sc-executor",
     "sp-api/std",
     "sp-block-builder/std",
     "sp-consensus-subspace/std",

crates/subspace-runtime/src/lib.rs

@@ -119,6 +119,35 @@ pub fn native_version() -> NativeVersion {
     }
 }
 
+/// Executor dispatch for subspace runtime
+#[cfg(feature = "std")]
+pub struct ExecutorDispatch;
+
+#[cfg(feature = "std")]
+impl sc_executor::NativeExecutionDispatch for ExecutorDispatch {
+    /// Only enable the benchmarking host functions when we actually want to benchmark.
+    #[cfg(feature = "runtime-benchmarks")]
+    type ExtendHostFunctions = (
+        frame_benchmarking::benchmarking::HostFunctions,
+        sp_consensus_subspace::consensus::HostFunctions,
+        sp_domains_fraud_proof::HostFunctions,
+    );
+    /// Otherwise we only use the default Substrate host functions.
+    #[cfg(not(feature = "runtime-benchmarks"))]
+    type ExtendHostFunctions = (
+        sp_consensus_subspace::consensus::HostFunctions,
+        sp_domains_fraud_proof::HostFunctions,
+    );
+
+    fn dispatch(method: &str, data: &[u8]) -> Option<Vec<u8>> {
+        api::dispatch(method, data)
+    }
+
+    fn native_version() -> sc_executor::NativeVersion {
+        native_version()
+    }
+}
+
 // TODO: Many of below constants should probably be updatable but currently they are not
 
 /// Since Subspace is probabilistic this is the average expected block time that

domains/runtime/evm/Cargo.toml

@@ -45,6 +45,7 @@ pallet-transaction-payment = { version = "4.0.0-dev", default-features = false,
 pallet-transaction-payment-rpc-runtime-api = { default-features = false, git = "https://github.com/subspace/polkadot-sdk", rev = "0831dfc3c54b10ab46e82acf98603b4af1a47bd5" }
 pallet-transporter = { version = "0.1.0", path = "../../pallets/transporter", default-features = false }
 scale-info = { version = "2.7.0", default-features = false, features = ["derive"] }
+sc-executor = { version = "0.10.0-dev", git = "https://github.com/subspace/polkadot-sdk", rev = "0831dfc3c54b10ab46e82acf98603b4af1a47bd5", optional = true }
 sp-api = { version = "4.0.0-dev", default-features = false, git = "https://github.com/subspace/polkadot-sdk", rev = "0831dfc3c54b10ab46e82acf98603b4af1a47bd5" }
 sp-block-builder = { version = "4.0.0-dev", default-features = false, git = "https://github.com/subspace/polkadot-sdk", rev = "0831dfc3c54b10ab46e82acf98603b4af1a47bd5" }
 sp-core = { version = "21.0.0", default-features = false, git = "https://github.com/subspace/polkadot-sdk", rev = "0831dfc3c54b10ab46e82acf98603b4af1a47bd5" }
@@ -98,6 +99,7 @@ std = [
     "pallet-transaction-payment/std",
     "pallet-transporter/std",
     "scale-info/std",
+    "sc-executor",
     "sp-api/std",
     "sp-block-builder/std",
     "sp-core/std",