Deploying to gh-pages from @ b1b4f9e 🚀

src-files.js

@@ -43,7 +43,7 @@ var srcIndex = JSON.parse('{\
 "subspace_archiving":["",[["archiver",[],["incremental_record_commitments.rs"]]],["archiver.rs","lib.rs","piece_reconstructor.rs","reconstructor.rs"]],\
 "subspace_core_primitives":["",[["crypto",[],["kzg.rs"]],["pieces",[],["serde.rs"]]],["checksum.rs","crypto.rs","lib.rs","objects.rs","pieces.rs","segments.rs","serde.rs"]],\
 "subspace_erasure_coding":["",[],["lib.rs"]],\
-"subspace_farmer":["",[["node_client",[],["node_rpc_client.rs"]],["single_disk_farm",[["farming",[],["monoio.rs","sync_fallback.rs"]]],["farming.rs","piece_cache.rs","piece_reader.rs","plotting.rs"]],["utils",[],["farmer_piece_getter.rs","piece_validator.rs","readers_and_pieces.rs","ss58.rs"]]],["identity.rs","lib.rs","node_client.rs","piece_cache.rs","reward_signing.rs","single_disk_farm.rs","utils.rs"]],\
+"subspace_farmer":["",[["node_client",[],["node_rpc_client.rs"]],["single_disk_farm",[["farming",[],["monoio.rs","rayon_files.rs","sync_fallback.rs"]]],["farming.rs","piece_cache.rs","piece_reader.rs","plotting.rs"]],["utils",[],["farmer_piece_getter.rs","piece_validator.rs","readers_and_pieces.rs","ss58.rs"]]],["identity.rs","lib.rs","node_client.rs","piece_cache.rs","reward_signing.rs","single_disk_farm.rs","utils.rs"]],\
 "subspace_farmer_components":["",[],["auditing.rs","file_ext.rs","lib.rs","plotting.rs","proving.rs","reading.rs","sector.rs","segment_reconstruction.rs"]],\
 "subspace_metrics":["",[],["lib.rs"]],\
 "subspace_networking":["",[["behavior",[],["persistent_parameters.rs"]],["constructor",[],["temporary_bans.rs","transport.rs"]],["protocols",[["connected_peers",[],["handler.rs"]],["peer_info",[],["handler.rs","protocol.rs"]],["request_response",[["handlers",[],["generic_request_handler.rs","piece_by_index.rs","segment_header.rs"]]],["handlers.rs","request_response_factory.rs"]],["reserved_peers",[],["handler.rs"]]],["connected_peers.rs","peer_info.rs","request_response.rs","reserved_peers.rs"]],["utils",[["rate_limiter",[],["resizable_semaphore.rs"]]],["multihash.rs","piece_provider.rs","rate_limiter.rs","unique_record_binary_heap.rs"]]],["behavior.rs","constructor.rs","lib.rs","node.rs","node_runner.rs","protocols.rs","shared.rs","utils.rs"]],\

src/subspace_farmer/single_disk_farm.rs.html

@@ -1898,15 +1898,6 @@
 <a href="#1898" id="1898">1898</a>
 <a href="#1899" id="1899">1899</a>
 <a href="#1900" id="1900">1900</a>
-<a href="#1901" id="1901">1901</a>
-<a href="#1902" id="1902">1902</a>
-<a href="#1903" id="1903">1903</a>
-<a href="#1904" id="1904">1904</a>
-<a href="#1905" id="1905">1905</a>
-<a href="#1906" id="1906">1906</a>
-<a href="#1907" id="1907">1907</a>
-<a href="#1908" id="1908">1908</a>
-<a href="#1909" id="1909">1909</a>
 </pre></div><pre class="rust"><code><span class="kw">pub mod </span>farming;
 <span class="kw">pub mod </span>piece_cache;
 <span class="kw">pub mod </span>piece_reader;
@@ -1915,6 +1906,7 @@
 <span class="kw">use </span><span class="kw">crate</span>::identity::{Identity, IdentityError};
 <span class="kw">use </span><span class="kw">crate</span>::node_client::NodeClient;
 <span class="kw">use </span><span class="kw">crate</span>::reward_signing::reward_signing;
+<span class="kw">use </span><span class="kw">crate</span>::single_disk_farm::farming::rayon_files::RayonFiles;
 <span class="kw">use </span><span class="kw">crate</span>::single_disk_farm::farming::sync_fallback::SyncPlotAudit;
 <span class="kw">pub use </span><span class="kw">crate</span>::single_disk_farm::farming::FarmingError;
 <span class="kw">use </span><span class="kw">crate</span>::single_disk_farm::farming::{farming, slot_notification_forwarder, FarmingOptions};
@@ -2880,7 +2872,6 @@
         <span class="kw">let </span>farming_join_handle = thread::Builder::new()
             .name(<span class="macro">format!</span>(<span class="string">&quot;farming-{disk_farm_index}&quot;</span>))
             .spawn({
-                <span class="kw">let </span>plot_file = Arc::clone(<span class="kw-2">&amp;</span>plot_file);
                 <span class="kw">let </span>handle = handle.clone();
                 <span class="kw">let </span>erasure_coding = erasure_coding.clone();
                 <span class="kw">let </span>handlers = Arc::clone(<span class="kw-2">&amp;</span>handlers);
@@ -2933,17 +2924,8 @@
                                 }
                             }
 
-                            <span class="attr">#[cfg(not(windows))]
-                            </span><span class="kw">let </span>plot_audit = <span class="kw-2">&amp;</span>SyncPlotAudit::new(<span class="kw-2">&amp;*</span>plot_file);
-                            <span class="attr">#[cfg(windows)]
-                            </span><span class="kw">let </span>plot_mmap = <span class="kw">unsafe </span>{
-                                memmap2::Mmap::map(<span class="kw-2">&amp;*</span>plot_file).map_err(FarmingError::from)<span class="question-mark">?
-                            </span>};
-                            <span class="comment">// On Windows random read is horrible in terms of performance, memory-mapped I/O helps
-                            // TODO: Remove this once https://internals.rust-lang.org/t/introduce-write-all-at-read-exact-at-on-windows/19649
-                            //  or similar exists in standard library
-                            </span><span class="attr">#[cfg(windows)]
-                            </span><span class="kw">let </span>plot_audit = <span class="kw-2">&amp;</span>SyncPlotAudit::new(<span class="kw-2">&amp;*</span>plot_mmap);
+                            <span class="kw">let </span>plot = RayonFiles::open(<span class="kw-2">&amp;</span>directory.join(<span class="self">Self</span>::PLOT_FILE))<span class="question-mark">?</span>;
+                            <span class="kw">let </span>plot_audit = <span class="kw-2">&amp;</span>SyncPlotAudit::new(<span class="kw-2">&amp;</span>plot);
 
                             <span class="kw">let </span>farming_options = FarmingOptions {
                                 public_key,

src/subspace_farmer/single_disk_farm/farming.rs.html

@@ -249,8 +249,10 @@
 <a href="#249" id="249">249</a>
 <a href="#250" id="250">250</a>
 <a href="#251" id="251">251</a>
+<a href="#252" id="252">252</a>
 </pre></div><pre class="rust"><code><span class="attr">#[cfg(any(target_os = <span class="string">&quot;linux&quot;</span>, target_os = <span class="string">&quot;macos&quot;</span>))]
 </span><span class="kw">pub mod </span>monoio;
+<span class="kw">pub mod </span>rayon_files;
 <span class="kw">pub mod </span>sync_fallback;
 
 <span class="kw">use </span><span class="kw">crate</span>::node_client;

src/subspace_farmer/single_disk_farm/farming/rayon_files.rs.html

@@ -0,0 +1,107 @@
+<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><meta name="generator" content="rustdoc"><meta name="description" content="Source of the Rust file `crates/subspace-farmer/src/single_disk_farm/farming/rayon_files.rs`."><title>rayon_files.rs - source</title><link rel="preload" as="font" type="font/woff2" crossorigin href="../../../../static.files/SourceSerif4-Regular-46f98efaafac5295.ttf.woff2"><link rel="preload" as="font" type="font/woff2" crossorigin href="../../../../static.files/FiraSans-Regular-018c141bf0843ffd.woff2"><link rel="preload" as="font" type="font/woff2" crossorigin href="../../../../static.files/FiraSans-Medium-8f9a781e4970d388.woff2"><link rel="preload" as="font" type="font/woff2" crossorigin href="../../../../static.files/SourceCodePro-Regular-562dcc5011b6de7d.ttf.woff2"><link rel="preload" as="font" type="font/woff2" crossorigin href="../../../../static.files/SourceSerif4-Bold-a2c9cd1067f8b328.ttf.woff2"><link rel="preload" as="font" type="font/woff2" crossorigin href="../../../../static.files/SourceCodePro-Semibold-d899c5a5c4aeb14a.ttf.woff2"><link rel="stylesheet" href="../../../../static.files/normalize-76eba96aa4d2e634.css"><link rel="stylesheet" href="../../../../static.files/rustdoc-deb46770fd761b69.css"><meta name="rustdoc-vars" data-root-path="../../../../" data-static-root-path="../../../../static.files/" data-current-crate="subspace_farmer" data-themes="" data-resource-suffix="" data-rustdoc-version="1.75.0-nightly (42b1224e9 2023-10-15)" data-channel="nightly" data-search-js="search-8fbf244ebcf71464.js" data-settings-js="settings-74424d7eec62a23e.js" ><script src="../../../../static.files/storage-fec3eaa3851e447d.js"></script><script defer src="../../../../static.files/src-script-3280b574d94e47b4.js"></script><script defer src="../../../../src-files.js"></script><script defer src="../../../../static.files/main-5f34af1a0ee6bacd.js"></script><noscript><link rel="stylesheet" href="../../../../static.files/noscript-5d8b3c7633ad77ba.css"></noscript><link rel="alternate icon" type="image/png" href="../../../../static.files/favicon-16x16-8b506e7a72182f1c.png"><link rel="alternate icon" type="image/png" href="../../../../static.files/favicon-32x32-422f7d1d52889060.png"><link rel="icon" type="image/svg+xml" href="../../../../static.files/favicon-2c020d218678b618.svg"></head><body class="rustdoc src"><!--[if lte IE 11]><div class="warning">This old browser is unsupported and will most likely display funky things.</div><![endif]--><nav class="sidebar"></nav><main><nav class="sub"><form class="search-form"><span></span><input class="search-input" name="search" aria-label="Run search in the documentation" autocomplete="off" spellcheck="false" placeholder="Click or press ‘S’ to search, ‘?’ for more options…" type="search"><div id="help-button" title="help" tabindex="-1"><a href="../../../../help.html">?</a></div><div id="settings-menu" tabindex="-1"><a href="../../../../settings.html" title="settings"><img width="22" height="22" alt="Change settings" src="../../../../static.files/wheel-7b819b6101059cd0.svg"></a></div></form></nav><section id="main-content" class="content"><div class="example-wrap"><div data-nosnippet><pre class="src-line-numbers"><a href="#1" id="1">1</a>
+<a href="#2" id="2">2</a>
+<a href="#3" id="3">3</a>
+<a href="#4" id="4">4</a>
+<a href="#5" id="5">5</a>
+<a href="#6" id="6">6</a>
+<a href="#7" id="7">7</a>
+<a href="#8" id="8">8</a>
+<a href="#9" id="9">9</a>
+<a href="#10" id="10">10</a>
+<a href="#11" id="11">11</a>
+<a href="#12" id="12">12</a>
+<a href="#13" id="13">13</a>
+<a href="#14" id="14">14</a>
+<a href="#15" id="15">15</a>
+<a href="#16" id="16">16</a>
+<a href="#17" id="17">17</a>
+<a href="#18" id="18">18</a>
+<a href="#19" id="19">19</a>
+<a href="#20" id="20">20</a>
+<a href="#21" id="21">21</a>
+<a href="#22" id="22">22</a>
+<a href="#23" id="23">23</a>
+<a href="#24" id="24">24</a>
+<a href="#25" id="25">25</a>
+<a href="#26" id="26">26</a>
+<a href="#27" id="27">27</a>
+<a href="#28" id="28">28</a>
+<a href="#29" id="29">29</a>
+<a href="#30" id="30">30</a>
+<a href="#31" id="31">31</a>
+<a href="#32" id="32">32</a>
+<a href="#33" id="33">33</a>
+<a href="#34" id="34">34</a>
+<a href="#35" id="35">35</a>
+<a href="#36" id="36">36</a>
+<a href="#37" id="37">37</a>
+<a href="#38" id="38">38</a>
+<a href="#39" id="39">39</a>
+<a href="#40" id="40">40</a>
+<a href="#41" id="41">41</a>
+<a href="#42" id="42">42</a>
+<a href="#43" id="43">43</a>
+<a href="#44" id="44">44</a>
+<a href="#45" id="45">45</a>
+<a href="#46" id="46">46</a>
+<a href="#47" id="47">47</a>
+<a href="#48" id="48">48</a>
+<a href="#49" id="49">49</a>
+<a href="#50" id="50">50</a>
+<a href="#51" id="51">51</a>
+<a href="#52" id="52">52</a>
+<a href="#53" id="53">53</a>
+</pre></div><pre class="rust"><code><span class="kw">use </span>std::fs::{File, OpenOptions};
+<span class="kw">use </span>std::io;
+<span class="kw">use </span>std::path::Path;
+<span class="kw">use </span>subspace_farmer_components::file_ext::{FileExt, OpenOptionsExt};
+<span class="kw">use </span>subspace_farmer_components::ReadAtSync;
+
+<span class="doccomment">/// Wrapper data structure for multiple files to be used with [`rayon`] thread pool, where the same
+/// file is opened multiple times, once for each thread.
+</span><span class="kw">pub struct </span>RayonFiles {
+    files: Vec&lt;File&gt;,
+}
+
+<span class="kw">impl </span>ReadAtSync <span class="kw">for </span>RayonFiles {
+    <span class="kw">fn </span>read_at(<span class="kw-2">&amp;</span><span class="self">self</span>, buf: <span class="kw-2">&amp;mut </span>[u8], offset: usize) -&gt; io::Result&lt;()&gt; {
+        <span class="kw">let </span>thread_index = rayon::current_thread_index().ok_or_else(|| {
+            io::Error::new(
+                io::ErrorKind::Other,
+                <span class="string">&quot;Reads must be called from rayon worker thread&quot;</span>,
+            )
+        })<span class="question-mark">?</span>;
+        <span class="kw">let </span>file = <span class="self">self</span>.files.get(thread_index).ok_or_else(|| {
+            io::Error::new(io::ErrorKind::Other, <span class="string">&quot;No files entry for this rayon thread&quot;</span>)
+        })<span class="question-mark">?</span>;
+
+        file.read_at(buf, offset)
+    }
+}
+
+<span class="kw">impl </span>ReadAtSync <span class="kw">for </span><span class="kw-2">&amp;</span>RayonFiles {
+    <span class="kw">fn </span>read_at(<span class="kw-2">&amp;</span><span class="self">self</span>, buf: <span class="kw-2">&amp;mut </span>[u8], offset: usize) -&gt; io::Result&lt;()&gt; {
+        (<span class="kw-2">*</span><span class="self">self</span>).read_at(buf, offset)
+    }
+}
+
+<span class="kw">impl </span>RayonFiles {
+    <span class="doccomment">/// Open file at specified as many times as there is number of threads in current [`rayon`]
+    /// thread pool.
+    </span><span class="kw">pub fn </span>open(path: <span class="kw-2">&amp;</span>Path) -&gt; io::Result&lt;<span class="self">Self</span>&gt; {
+        <span class="kw">let </span>files = (<span class="number">0</span>..rayon::current_num_threads())
+            .map(|<span class="kw">_</span>| {
+                <span class="kw">let </span>file = OpenOptions::new()
+                    .read(<span class="bool-val">true</span>)
+                    .advise_random_access()
+                    .open(path)<span class="question-mark">?</span>;
+                file.advise_random_access()<span class="question-mark">?</span>;
+
+                <span class="prelude-val">Ok</span>::&lt;<span class="kw">_</span>, io::Error&gt;(file)
+            })
+            .collect::&lt;<span class="prelude-ty">Result</span>&lt;Vec&lt;<span class="kw">_</span>&gt;, <span class="kw">_</span>&gt;&gt;()<span class="question-mark">?</span>;
+
+        <span class="prelude-val">Ok</span>(<span class="self">Self </span>{ files })
+    }
+}
+</code></pre></div></section></main></body></html>