Merge pull request #2095 from subspace/adjustable-plotting-concurrency

Adjustable plotting concurrency

crates/pallet-subspace/src/mock.rs

@@ -55,7 +55,9 @@ use subspace_core_primitives::{
 };
 use subspace_erasure_coding::ErasureCoding;
 use subspace_farmer_components::auditing::audit_sector;
-use subspace_farmer_components::plotting::{plot_sector, PieceGetterRetryPolicy};
+use subspace_farmer_components::plotting::{
+    plot_sector, PieceGetterRetryPolicy, PlotSectorOptions,
+};
 use subspace_farmer_components::FarmerProtocolInfo;
 use subspace_proof_of_space::shim::ShimTable;
 use subspace_proof_of_space::{Table, TableGenerator};
@@ -454,19 +456,21 @@ pub fn create_signed_vote(
         let mut plotted_sector_bytes = Vec::new();
         let mut plotted_sector_metadata_bytes = Vec::new();
 
-        let plotted_sector = block_on(plot_sector::<_, PosTable>(
-            &public_key,
+        let plotted_sector = block_on(plot_sector::<PosTable, _>(PlotSectorOptions {
+            public_key: &public_key,
             sector_index,
-            archived_history_segment,
-            PieceGetterRetryPolicy::default(),
-            &farmer_protocol_info,
+            piece_getter: archived_history_segment,
+            piece_getter_retry_policy: PieceGetterRetryPolicy::default(),
+            farmer_protocol_info: &farmer_protocol_info,
             kzg,
             erasure_coding,
             pieces_in_sector,
-            &mut plotted_sector_bytes,
-            &mut plotted_sector_metadata_bytes,
-            &mut table_generator,
-        ))
+            sector_output: &mut plotted_sector_bytes,
+            sector_metadata_output: &mut plotted_sector_metadata_bytes,
+            downloading_semaphore: None,
+            encoding_semaphore: None,
+            table_generator: &mut table_generator,
+        }))
         .unwrap();
 
         let global_challenge = proof_of_time

crates/sp-lightclient/src/tests.rs

@@ -175,7 +175,7 @@ fn valid_header(
         let mut plotted_sector_bytes = Vec::new();
         let mut plotted_sector_metadata_bytes = Vec::new();
 
-        let plotted_sector = block_on(plot_sector::<_, PosTable>(
+        let plotted_sector = block_on(plot_sector::<PosTable, _>(
             &public_key,
             sector_index,
             &archived_segment.pieces,

crates/subspace-farmer-components/benches/auditing.rs

@@ -16,7 +16,9 @@ use subspace_core_primitives::{
 use subspace_erasure_coding::ErasureCoding;
 use subspace_farmer_components::auditing::audit_sector;
 use subspace_farmer_components::file_ext::{FileExt, OpenOptionsExt};
-use subspace_farmer_components::plotting::{plot_sector, PieceGetterRetryPolicy, PlottedSector};
+use subspace_farmer_components::plotting::{
+    plot_sector, PieceGetterRetryPolicy, PlotSectorOptions, PlottedSector,
+};
 use subspace_farmer_components::sector::{
     sector_size, SectorContentsMap, SectorMetadata, SectorMetadataChecksummed,
 };
@@ -116,19 +118,21 @@ pub fn criterion_benchmark(c: &mut Criterion) {
         let mut plotted_sector_bytes = Vec::new();
         let mut plotted_sector_metadata_bytes = Vec::new();
 
-        let plotted_sector = block_on(plot_sector::<_, PosTable>(
-            &public_key,
+        let plotted_sector = block_on(plot_sector::<PosTable, _>(PlotSectorOptions {
+            public_key: &public_key,
             sector_index,
-            &archived_history_segment,
-            PieceGetterRetryPolicy::default(),
-            &farmer_protocol_info,
-            &kzg,
-            &erasure_coding,
+            piece_getter: &archived_history_segment,
+            piece_getter_retry_policy: PieceGetterRetryPolicy::default(),
+            farmer_protocol_info: &farmer_protocol_info,
+            kzg: &kzg,
+            erasure_coding: &erasure_coding,
             pieces_in_sector,
-            &mut plotted_sector_bytes,
-            &mut plotted_sector_metadata_bytes,
-            &mut table_generator,
-        ))
+            sector_output: &mut plotted_sector_bytes,
+            sector_metadata_output: &mut plotted_sector_metadata_bytes,
+            downloading_semaphore: black_box(None),
+            encoding_semaphore: black_box(None),
+            table_generator: &mut table_generator,
+        }))
         .unwrap();
 
         (plotted_sector, plotted_sector_bytes)

crates/subspace-farmer-components/benches/plotting.rs

@@ -8,7 +8,9 @@ use subspace_core_primitives::crypto::kzg;
 use subspace_core_primitives::crypto::kzg::Kzg;
 use subspace_core_primitives::{HistorySize, PublicKey, Record, RecordedHistorySegment};
 use subspace_erasure_coding::ErasureCoding;
-use subspace_farmer_components::plotting::{plot_sector, PieceGetterRetryPolicy};
+use subspace_farmer_components::plotting::{
+    plot_sector, PieceGetterRetryPolicy, PlotSectorOptions,
+};
 use subspace_farmer_components::sector::sector_size;
 use subspace_farmer_components::FarmerProtocolInfo;
 use subspace_proof_of_space::chia::ChiaTable;
@@ -65,19 +67,21 @@ fn criterion_benchmark(c: &mut Criterion) {
     group.throughput(Throughput::Bytes(sector_size as u64));
     group.bench_function("in-memory", |b| {
         b.iter(|| {
-            block_on(plot_sector::<_, PosTable>(
-                black_box(&public_key),
-                black_box(sector_index),
-                black_box(&archived_history_segment),
-                black_box(PieceGetterRetryPolicy::default()),
-                black_box(&farmer_protocol_info),
-                black_box(&kzg),
-                black_box(&erasure_coding),
-                black_box(pieces_in_sector),
-                black_box(&mut sector_bytes),
-                black_box(&mut sector_metadata_bytes),
-                black_box(&mut table_generator),
-            ))
+            block_on(plot_sector::<PosTable, _>(PlotSectorOptions {
+                public_key: black_box(&public_key),
+                sector_index: black_box(sector_index),
+                piece_getter: black_box(&archived_history_segment),
+                piece_getter_retry_policy: black_box(PieceGetterRetryPolicy::default()),
+                farmer_protocol_info: black_box(&farmer_protocol_info),
+                kzg: black_box(&kzg),
+                erasure_coding: black_box(&erasure_coding),
+                pieces_in_sector: black_box(pieces_in_sector),
+                sector_output: black_box(&mut sector_bytes),
+                sector_metadata_output: black_box(&mut sector_metadata_bytes),
+                downloading_semaphore: black_box(None),
+                encoding_semaphore: black_box(None),
+                table_generator: black_box(&mut table_generator),
+            }))
             .unwrap();
         })
     });

crates/subspace-farmer-components/benches/proving.rs

@@ -17,7 +17,9 @@ use subspace_core_primitives::{
 use subspace_erasure_coding::ErasureCoding;
 use subspace_farmer_components::auditing::audit_sector;
 use subspace_farmer_components::file_ext::{FileExt, OpenOptionsExt};
-use subspace_farmer_components::plotting::{plot_sector, PieceGetterRetryPolicy, PlottedSector};
+use subspace_farmer_components::plotting::{
+    plot_sector, PieceGetterRetryPolicy, PlotSectorOptions, PlottedSector,
+};
 use subspace_farmer_components::sector::{
     sector_size, SectorContentsMap, SectorMetadata, SectorMetadataChecksummed,
 };
@@ -119,19 +121,21 @@ pub fn criterion_benchmark(c: &mut Criterion) {
         let mut plotted_sector_bytes = Vec::new();
         let mut plotted_sector_metadata_bytes = Vec::new();
 
-        let plotted_sector = block_on(plot_sector::<_, PosTable>(
-            &public_key,
+        let plotted_sector = block_on(plot_sector::<PosTable, _>(PlotSectorOptions {
+            public_key: &public_key,
             sector_index,
-            &archived_history_segment,
-            PieceGetterRetryPolicy::default(),
-            &farmer_protocol_info,
-            &kzg,
-            &erasure_coding,
+            piece_getter: &archived_history_segment,
+            piece_getter_retry_policy: PieceGetterRetryPolicy::default(),
+            farmer_protocol_info: &farmer_protocol_info,
+            kzg: &kzg,
+            erasure_coding: &erasure_coding,
             pieces_in_sector,
-            &mut plotted_sector_bytes,
-            &mut plotted_sector_metadata_bytes,
-            &mut table_generator,
-        ))
+            sector_output: &mut plotted_sector_bytes,
+            sector_metadata_output: &mut plotted_sector_metadata_bytes,
+            downloading_semaphore: black_box(None),
+            encoding_semaphore: black_box(None),
+            table_generator: &mut table_generator,
+        }))
         .unwrap();
 
         (plotted_sector, plotted_sector_bytes)

crates/subspace-farmer-components/benches/reading.rs

@@ -14,7 +14,9 @@ use subspace_core_primitives::{
 };
 use subspace_erasure_coding::ErasureCoding;
 use subspace_farmer_components::file_ext::{FileExt, OpenOptionsExt};
-use subspace_farmer_components::plotting::{plot_sector, PieceGetterRetryPolicy, PlottedSector};
+use subspace_farmer_components::plotting::{
+    plot_sector, PieceGetterRetryPolicy, PlotSectorOptions, PlottedSector,
+};
 use subspace_farmer_components::reading::read_piece;
 use subspace_farmer_components::sector::{
     sector_size, SectorContentsMap, SectorMetadata, SectorMetadataChecksummed,
@@ -113,19 +115,21 @@ pub fn criterion_benchmark(c: &mut Criterion) {
         let mut plotted_sector_bytes = Vec::new();
         let mut plotted_sector_metadata_bytes = Vec::new();
 
-        let plotted_sector = block_on(plot_sector::<_, PosTable>(
-            &public_key,
+        let plotted_sector = block_on(plot_sector::<PosTable, _>(PlotSectorOptions {
+            public_key: &public_key,
             sector_index,
-            &archived_history_segment,
-            PieceGetterRetryPolicy::default(),
-            &farmer_protocol_info,
-            &kzg,
-            &erasure_coding,
+            piece_getter: &archived_history_segment,
+            piece_getter_retry_policy: PieceGetterRetryPolicy::default(),
+            farmer_protocol_info: &farmer_protocol_info,
+            kzg: &kzg,
+            erasure_coding: &erasure_coding,
             pieces_in_sector,
-            &mut plotted_sector_bytes,
-            &mut plotted_sector_metadata_bytes,
-            &mut table_generator,
-        ))
+            sector_output: &mut plotted_sector_bytes,
+            sector_metadata_output: &mut plotted_sector_metadata_bytes,
+            downloading_semaphore: black_box(None),
+            encoding_semaphore: black_box(None),
+            table_generator: &mut table_generator,
+        }))
         .unwrap();
 
         (plotted_sector, plotted_sector_bytes)

crates/subspace-farmer-components/src/plotting.rs

@@ -151,34 +151,74 @@ pub enum PlottingError {
     },
 }
 
-/// Plot a single sector, where `sector` and `sector_metadata` must be positioned correctly at the
-/// beginning of the sector (seek to desired offset before calling this function and seek back
-/// afterwards if necessary).
+/// Options for plotting a sector.
 ///
 /// Sector output and sector metadata output should be either empty (in which case they'll be
 /// resized to correct size automatically) or correctly sized from the beginning or else error will
 /// be returned.
+pub struct PlotSectorOptions<'a, PosTable, PG>
+where
+    PosTable: Table,
+{
+    /// Public key corresponding to sector
+    pub public_key: &'a PublicKey,
+    /// Sector index
+    pub sector_index: SectorIndex,
+    /// Getter for pieces of archival history
+    pub piece_getter: &'a PG,
+    /// Retry policy for piece getter
+    pub piece_getter_retry_policy: PieceGetterRetryPolicy,
+    /// Farmer protocol info
+    pub farmer_protocol_info: &'a FarmerProtocolInfo,
+    /// KZG instance
+    pub kzg: &'a Kzg,
+    /// Erasure coding instance
+    pub erasure_coding: &'a ErasureCoding,
+    /// How many pieces should sector contain
+    pub pieces_in_sector: u16,
+    /// Where plotted sector should be written, vector must either be empty (in which case it'll be
+    /// resized to correct size automatically) or correctly sized from the beginning
+    pub sector_output: &'a mut Vec<u8>,
+    /// Where plotted sector metadata should be written, vector must either be empty (in which case
+    /// it'll be resized to correct size automatically) or correctly sized from the beginning
+    pub sector_metadata_output: &'a mut Vec<u8>,
+    /// Semaphore for part of the plotting when farmer downloads new sector, allows to limit memory
+    /// usage of the plotting process, permit will be held until the end of the plotting process
+    pub downloading_semaphore: Option<&'a Semaphore>,
+    /// Semaphore for part of the plotting when farmer encodes downloaded sector, should typically
+    /// allow one permit at a time for efficient CPU utilization
+    pub encoding_semaphore: Option<&'a Semaphore>,
+    /// Proof of space table generator
+    pub table_generator: &'a mut PosTable::Generator,
+}
+
+/// Plot a single sector.
 ///
 /// NOTE: Even though this function is async, it has blocking code inside and must be running in a
 /// separate thread in order to prevent blocking an executor.
-#[allow(clippy::too_many_arguments)]
-pub async fn plot_sector<PG, PosTable>(
-    public_key: &PublicKey,
-    sector_index: SectorIndex,
-    piece_getter: &PG,
-    piece_getter_retry_policy: PieceGetterRetryPolicy,
-    farmer_protocol_info: &FarmerProtocolInfo,
-    kzg: &Kzg,
-    erasure_coding: &ErasureCoding,
-    pieces_in_sector: u16,
-    sector_output: &mut Vec<u8>,
-    sector_metadata_output: &mut Vec<u8>,
-    table_generator: &mut PosTable::Generator,
+pub async fn plot_sector<PosTable, PG>(
+    options: PlotSectorOptions<'_, PosTable, PG>,
 ) -> Result<PlottedSector, PlottingError>
 where
-    PG: PieceGetter,
     PosTable: Table,
+    PG: PieceGetter,
 {
+    let PlotSectorOptions {
+        public_key,
+        sector_index,
+        piece_getter,
+        piece_getter_retry_policy,
+        farmer_protocol_info,
+        kzg,
+        erasure_coding,
+        pieces_in_sector,
+        sector_output,
+        sector_metadata_output,
+        downloading_semaphore,
+        encoding_semaphore,
+        table_generator,
+    } = options;
+
     if erasure_coding.max_shards() < Record::NUM_S_BUCKETS {
         return Err(PlottingError::InvalidErasureCodingInstance);
     }
@@ -201,6 +241,11 @@ where
         });
     }
 
+    let _downloading_permit = match downloading_semaphore {
+        Some(downloading_semaphore) => Some(downloading_semaphore.acquire().await),
+        None => None,
+    };
+
     let sector_id = SectorId::new(public_key.hash(), sector_index);
 
     let piece_indexes: Vec<PieceIndex> = (PieceOffset::ZERO..)
@@ -263,6 +308,11 @@ where
 
     let mut raw_sector = raw_sector.into_inner();
 
+    let _encoding_permit = match encoding_semaphore {
+        Some(encoding_semaphore) => Some(encoding_semaphore.acquire().await),
+        None => None,
+    };
+
     let mut sector_contents_map = SectorContentsMap::new(pieces_in_sector);
 
     (PieceOffset::ZERO..)

crates/subspace-farmer/Cargo.toml

@@ -38,7 +38,6 @@ schnorrkel = "0.9.1"
 serde = { version = "1.0.183", features = ["derive"] }
 serde_json = "1.0.106"
 static_assertions = "1.1.0"
-std-semaphore = "0.1.0"
 ss58-registry = "1.43.0"
 subspace-archiving = { version = "0.1.0", path = "../subspace-archiving" }
 subspace-erasure-coding = { version = "0.1.0", path = "../subspace-erasure-coding" }