From 34cb7151713c0c7ec8fbb0da5928c66b49cd0897 Mon Sep 17 00:00:00 2001 From: Josh Wilson Date: Thu, 26 Sep 2024 11:09:32 +0900 Subject: [PATCH 1/3] feat: allow compile time override of MAX_CHUNK_SIZE --- Cargo.toml | 47 +++++++------- src/lib.rs | 37 ++++++----- tests/lib.rs | 171 +++++++++++++++++++++++++-------------------------- 3 files changed, 132 insertions(+), 123 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 1a0c23401..56cbf8eeb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,5 @@ [package] -authors = [ "MaidSafe Developers " ] +authors = ["MaidSafe Developers "] description = "Self encrypting files (convergent encryption plus obfuscation)" documentation = "https://docs.rs/self_encryption" edition = "2018" @@ -14,6 +14,7 @@ version = "0.29.2" aes = "~0.8.1" bincode = "~1.3.3" hex = "~0.4.3" +lazy_static = "1.4.0" rand = "~0.8.5" rand_chacha = "~0.3.1" rayon = "1.5.1" @@ -23,38 +24,38 @@ itertools = "~0.10.0" tempfile = "3.6.0" xor_name = "5.0.0" - [dependencies.brotli] - version = "~3.3.0" - default-features = false - features = [ "std" ] +[dependencies.brotli] +version = "~3.3.0" +default-features = false +features = ["std"] - [dependencies.cbc] - version = "~0.1.1" - features = [ "alloc", "block-padding" ] +[dependencies.cbc] +version = "~0.1.1" +features = ["alloc", "block-padding"] - [dependencies.bytes] - version = "1.1.0" - features = [ "serde" ] +[dependencies.bytes] +version = "1.1.0" +features = ["serde"] - [dependencies.serde] - version = "1.0.136" - features = [ "derive" ] +[dependencies.serde] +version = "1.0.136" +features = ["derive"] - [dependencies.tiny-keccak] - version = "2.0.2" - features = [ "sha3" ] +[dependencies.tiny-keccak] +version = "2.0.2" +features = ["sha3"] - [dependencies.tokio] - version = "1.34.0" - features = [ "rt" ] +[dependencies.tokio] +version = "1.34.0" +features = ["rt"] [dev-dependencies] criterion = "~0.3" docopt = "~0.9.0" - [dev-dependencies.tokio] - version = "1.34.0" - features = [ "rt-multi-thread", "macros" ] +[dev-dependencies.tokio] +version = "1.34.0" +features = ["rt-multi-thread", "macros"] [[example]] bench = false diff --git a/src/lib.rs b/src/lib.rs index 5e566d589..4d8d4cf7e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -80,7 +80,6 @@ unused_results )] #![allow( - box_pointers, missing_copy_implementations, missing_debug_implementations, variant_size_differences, @@ -110,6 +109,7 @@ use chunk::batch_positions; use decrypt::decrypt_chunk; use encrypt::encrypt_chunk; use itertools::Itertools; +use lazy_static::lazy_static; use std::{ collections::BTreeMap, fs::{self, File, OpenOptions}, @@ -126,8 +126,17 @@ pub use xor_name; /// The minimum size (before compression) of data to be self-encrypted, defined as 3B. pub const MIN_ENCRYPTABLE_BYTES: usize = 3 * MIN_CHUNK_SIZE; -/// The maximum size (before compression) of an individual chunk of a file, defined as 500kiB. -pub const MAX_CHUNK_SIZE: usize = 512 * 1024; +/// The default maximum size (before compression) of an individual chunk of a file, defaulting as 1MiB. +const DEFAULT_MAX_CHUNK_SIZE: usize = 1024 * 1024; + +lazy_static! { + /// The maximum size (before compression) of an individual chunk of a file, defaulting as 1MiB. + pub static ref MAX_CHUNK_SIZE: usize = std::option_env!("MAX_CHUNK_SIZE") + .unwrap_or("1048576") + .parse::() + .unwrap_or(DEFAULT_MAX_CHUNK_SIZE); +} + /// The minimum size (before compression) of an individual chunk of a file, defined as 1B. pub const MIN_CHUNK_SIZE: usize = 1; /// Controls the compression-speed vs compression-density tradeoffs. The higher the quality, the @@ -487,7 +496,7 @@ pub struct SeekInfo { pub fn seek_info(file_size: usize, pos: usize, len: usize) -> SeekInfo { let (start_index, end_index) = overlapped_chunks(file_size, pos, len); - let relative_pos = if start_index == 2 && file_size < 3 * MAX_CHUNK_SIZE { + let relative_pos = if start_index == 2 && file_size < 3 * *MAX_CHUNK_SIZE { pos - (2 * get_chunk_size(file_size, 0)) } else { pos % get_chunk_size(file_size, start_index) @@ -569,13 +578,13 @@ fn get_num_chunks(file_size: usize) -> usize { if file_size < (3 * MIN_CHUNK_SIZE) { return 0; } - if file_size < (3 * MAX_CHUNK_SIZE) { + if file_size < (3 * *MAX_CHUNK_SIZE) { return 3; } - if file_size % MAX_CHUNK_SIZE == 0 { - file_size / MAX_CHUNK_SIZE + if file_size % *MAX_CHUNK_SIZE == 0 { + file_size / *MAX_CHUNK_SIZE } else { - (file_size / MAX_CHUNK_SIZE) + 1 + (file_size / *MAX_CHUNK_SIZE) + 1 } } @@ -584,7 +593,7 @@ fn get_chunk_size(file_size: usize, chunk_index: usize) -> usize { if file_size < 3 * MIN_CHUNK_SIZE { return 0; } - if file_size < 3 * MAX_CHUNK_SIZE { + if file_size < 3 * *MAX_CHUNK_SIZE { if chunk_index < 2 { return file_size / 3; } else { @@ -594,21 +603,21 @@ fn get_chunk_size(file_size: usize, chunk_index: usize) -> usize { } let total_chunks = get_num_chunks(file_size); if chunk_index < total_chunks - 2 { - return MAX_CHUNK_SIZE; + return *MAX_CHUNK_SIZE; } - let remainder = file_size % MAX_CHUNK_SIZE; + let remainder = file_size % *MAX_CHUNK_SIZE; let penultimate = (total_chunks - 2) == chunk_index; if remainder == 0 { - return MAX_CHUNK_SIZE; + return *MAX_CHUNK_SIZE; } if remainder < MIN_CHUNK_SIZE { if penultimate { - MAX_CHUNK_SIZE - MIN_CHUNK_SIZE + *MAX_CHUNK_SIZE - MIN_CHUNK_SIZE } else { MIN_CHUNK_SIZE + remainder } } else if penultimate { - MAX_CHUNK_SIZE + *MAX_CHUNK_SIZE } else { remainder } diff --git a/tests/lib.rs b/tests/lib.rs index fca253f97..62402e6ba 100644 --- a/tests/lib.rs +++ b/tests/lib.rs @@ -43,7 +43,6 @@ unused_results )] #![allow( - box_pointers, missing_copy_implementations, missing_debug_implementations, variant_size_differences @@ -55,7 +54,7 @@ use xor_name::XorName; #[tokio::test] async fn cross_platform_check() -> Result<()> { - let content_size: usize = 20 * MAX_CHUNK_SIZE + 100; + let content_size: usize = 20 * *MAX_CHUNK_SIZE + 100; let mut content = vec![0u8; content_size]; for (i, c) in content.iter_mut().enumerate().take(content_size) { *c = (i % 17) as u8; @@ -67,252 +66,252 @@ async fn cross_platform_check() -> Result<()> { let ref_data_map = vec![ ChunkInfo { src_hash: XorName([ - 248, 242, 229, 119, 92, 211, 180, 222, 177, 34, 82, 94, 51, 178, 62, 12, 185, 77, - 145, 206, 168, 75, 176, 141, 46, 197, 1, 83, 199, 165, 37, 28, + 219, 177, 84, 234, 189, 172, 82, 64, 169, 100, 5, 56, 3, 43, 142, 126, 51, 235, + 194, 243, 30, 130, 132, 197, 137, 36, 170, 62, 46, 44, 176, 201, ]), dst_hash: XorName([ - 160, 57, 64, 193, 147, 235, 173, 54, 53, 206, 248, 12, 40, 147, 119, 107, 154, 21, - 50, 57, 151, 18, 151, 0, 95, 157, 103, 220, 160, 79, 248, 85, + 248, 155, 46, 153, 173, 52, 226, 212, 133, 172, 107, 200, 72, 150, 41, 50, 116, 77, + 85, 92, 67, 168, 25, 56, 93, 61, 209, 194, 65, 172, 227, 130, ]), index: 0, src_size: 0, }, ChunkInfo { src_hash: XorName([ - 68, 137, 10, 147, 116, 198, 83, 144, 156, 198, 68, 195, 94, 96, 125, 162, 213, 218, - 179, 255, 177, 143, 232, 48, 99, 204, 118, 246, 67, 243, 190, 96, + 65, 81, 63, 82, 119, 126, 216, 9, 44, 18, 160, 174, 225, 8, 202, 32, 245, 140, 14, + 169, 252, 209, 97, 96, 134, 165, 102, 106, 250, 196, 27, 70, ]), dst_hash: XorName([ - 30, 212, 77, 155, 165, 236, 65, 212, 88, 181, 48, 138, 226, 135, 144, 227, 132, - 195, 223, 199, 172, 235, 51, 146, 109, 209, 54, 63, 34, 169, 91, 55, + 42, 62, 224, 152, 136, 214, 91, 160, 125, 249, 229, 115, 81, 220, 213, 34, 29, 173, + 235, 99, 67, 210, 234, 160, 79, 254, 208, 174, 117, 127, 205, 36, ]), index: 0, src_size: 0, }, ChunkInfo { src_hash: XorName([ - 227, 224, 98, 89, 131, 120, 169, 214, 165, 171, 189, 187, 15, 7, 80, 133, 16, 63, - 74, 197, 17, 127, 22, 137, 171, 117, 34, 195, 186, 185, 51, 2, + 80, 237, 26, 5, 69, 59, 53, 210, 44, 236, 191, 69, 92, 39, 113, 124, 206, 169, 5, + 126, 189, 2, 146, 80, 68, 186, 142, 219, 37, 170, 135, 61, ]), dst_hash: XorName([ - 166, 232, 206, 232, 6, 23, 232, 20, 105, 230, 249, 86, 35, 117, 181, 65, 192, 245, - 65, 130, 238, 50, 188, 82, 193, 115, 172, 113, 237, 33, 248, 102, + 200, 203, 81, 29, 131, 156, 60, 140, 166, 254, 103, 60, 212, 223, 22, 41, 85, 192, + 140, 154, 33, 34, 188, 94, 84, 101, 62, 254, 164, 81, 209, 154, ]), index: 0, src_size: 0, }, ChunkInfo { src_hash: XorName([ - 48, 157, 209, 23, 176, 114, 223, 155, 203, 103, 11, 52, 211, 111, 167, 33, 13, 77, - 71, 6, 188, 152, 179, 76, 155, 59, 4, 92, 3, 9, 67, 227, + 168, 223, 46, 4, 138, 115, 226, 112, 179, 67, 36, 186, 170, 199, 21, 195, 41, 17, + 99, 227, 30, 226, 46, 42, 78, 210, 189, 107, 185, 167, 32, 74, ]), dst_hash: XorName([ - 156, 144, 25, 237, 84, 230, 81, 90, 205, 79, 203, 161, 113, 141, 59, 138, 117, 157, - 50, 9, 46, 76, 68, 64, 254, 250, 59, 11, 27, 134, 114, 175, + 42, 138, 132, 73, 12, 78, 47, 136, 153, 177, 25, 247, 202, 227, 145, 31, 193, 9, + 33, 63, 89, 160, 240, 51, 189, 72, 94, 193, 75, 144, 58, 233, ]), index: 0, src_size: 0, }, ChunkInfo { src_hash: XorName([ - 53, 238, 190, 32, 115, 7, 143, 124, 163, 186, 189, 137, 50, 118, 2, 232, 57, 223, - 124, 10, 239, 109, 31, 4, 77, 67, 150, 92, 207, 26, 53, 0, + 41, 137, 66, 160, 103, 223, 72, 133, 180, 83, 8, 139, 180, 108, 20, 196, 106, 59, + 73, 6, 160, 187, 8, 16, 93, 157, 142, 155, 85, 118, 239, 192, ]), dst_hash: XorName([ - 146, 0, 118, 252, 165, 0, 60, 204, 12, 126, 121, 68, 193, 237, 32, 58, 78, 125, - 110, 49, 215, 140, 37, 90, 141, 80, 8, 205, 206, 94, 115, 91, + 220, 162, 48, 182, 212, 178, 139, 207, 231, 191, 209, 53, 187, 22, 66, 221, 242, + 66, 220, 19, 96, 201, 137, 25, 101, 184, 1, 178, 80, 204, 253, 179, ]), index: 0, src_size: 0, }, ChunkInfo { src_hash: XorName([ - 208, 239, 194, 163, 28, 94, 172, 182, 163, 69, 43, 242, 76, 157, 70, 10, 49, 228, - 153, 45, 154, 149, 111, 131, 132, 48, 67, 149, 198, 188, 147, 187, + 48, 226, 1, 203, 69, 49, 140, 152, 90, 232, 209, 42, 178, 241, 60, 11, 24, 2, 196, + 26, 14, 229, 127, 68, 119, 116, 135, 195, 248, 217, 227, 78, ]), dst_hash: XorName([ - 138, 105, 198, 150, 73, 205, 0, 204, 67, 235, 102, 199, 152, 47, 215, 34, 230, 6, - 211, 6, 72, 38, 102, 74, 161, 22, 201, 229, 73, 179, 241, 183, + 168, 232, 79, 142, 149, 51, 198, 62, 224, 177, 45, 203, 243, 51, 12, 23, 104, 80, + 174, 5, 246, 234, 54, 70, 58, 11, 100, 117, 60, 67, 65, 64, ]), index: 0, src_size: 0, }, ChunkInfo { src_hash: XorName([ - 127, 180, 215, 240, 32, 8, 203, 232, 31, 47, 232, 156, 181, 145, 96, 189, 228, 127, - 8, 243, 144, 169, 251, 212, 128, 243, 90, 159, 209, 101, 22, 26, + 92, 201, 208, 153, 241, 202, 111, 28, 118, 47, 47, 32, 121, 48, 203, 48, 230, 107, + 102, 195, 184, 106, 245, 173, 157, 171, 139, 50, 28, 56, 80, 225, ]), dst_hash: XorName([ - 163, 215, 111, 245, 3, 80, 107, 218, 200, 254, 69, 43, 230, 168, 85, 162, 65, 230, - 46, 203, 49, 1, 99, 25, 102, 218, 105, 129, 215, 124, 132, 104, + 199, 114, 193, 185, 26, 6, 140, 71, 142, 73, 45, 198, 110, 126, 232, 182, 226, 85, + 137, 210, 69, 24, 139, 163, 236, 47, 155, 130, 43, 229, 148, 172, ]), index: 0, src_size: 0, }, ChunkInfo { src_hash: XorName([ - 79, 17, 134, 221, 54, 248, 197, 215, 92, 180, 23, 186, 143, 71, 41, 138, 151, 174, - 241, 128, 212, 7, 63, 136, 61, 132, 177, 198, 129, 20, 168, 87, + 50, 8, 67, 204, 158, 4, 255, 227, 50, 18, 176, 150, 249, 233, 188, 72, 86, 217, 61, + 100, 161, 131, 124, 26, 245, 166, 44, 16, 125, 230, 153, 190, ]), dst_hash: XorName([ - 207, 109, 164, 0, 68, 241, 197, 210, 209, 143, 239, 76, 198, 12, 225, 162, 159, 37, - 175, 0, 159, 239, 160, 178, 18, 75, 206, 126, 208, 0, 142, 213, + 151, 255, 185, 86, 239, 216, 199, 233, 149, 16, 247, 122, 156, 66, 178, 95, 32, + 219, 218, 228, 63, 23, 34, 207, 140, 20, 75, 2, 225, 3, 243, 193, ]), index: 0, src_size: 0, }, ChunkInfo { src_hash: XorName([ - 119, 172, 206, 200, 245, 153, 32, 24, 14, 70, 123, 251, 75, 66, 0, 50, 44, 145, - 126, 243, 42, 39, 232, 208, 117, 190, 105, 120, 169, 193, 192, 228, + 132, 6, 224, 90, 168, 59, 66, 114, 199, 67, 140, 171, 226, 213, 141, 21, 32, 143, + 4, 192, 143, 64, 253, 216, 200, 76, 162, 121, 130, 169, 89, 229, ]), dst_hash: XorName([ - 243, 107, 119, 61, 216, 70, 121, 241, 109, 84, 231, 232, 220, 177, 230, 158, 168, - 204, 215, 19, 185, 45, 178, 225, 103, 198, 119, 238, 144, 175, 38, 147, + 126, 221, 146, 123, 252, 37, 250, 160, 75, 182, 9, 39, 80, 87, 93, 229, 173, 203, + 31, 203, 208, 190, 226, 111, 87, 78, 246, 141, 85, 237, 82, 87, ]), index: 0, src_size: 0, }, ChunkInfo { src_hash: XorName([ - 52, 10, 82, 208, 199, 46, 246, 175, 107, 245, 168, 201, 212, 133, 79, 187, 24, 226, - 10, 241, 43, 148, 84, 103, 153, 32, 66, 36, 146, 87, 60, 37, + 238, 37, 229, 233, 96, 228, 150, 41, 89, 130, 145, 198, 50, 165, 207, 108, 15, 167, + 122, 116, 209, 223, 68, 203, 24, 169, 74, 93, 44, 170, 24, 233, ]), dst_hash: XorName([ - 77, 167, 37, 235, 4, 230, 211, 221, 27, 211, 207, 32, 23, 202, 118, 100, 8, 199, - 67, 28, 195, 87, 141, 11, 24, 138, 34, 233, 63, 68, 123, 236, + 109, 123, 118, 55, 228, 175, 144, 231, 103, 223, 51, 185, 146, 37, 47, 46, 185, + 208, 140, 202, 231, 18, 70, 47, 48, 245, 254, 93, 185, 120, 17, 143, ]), index: 0, src_size: 0, }, ChunkInfo { src_hash: XorName([ - 234, 15, 144, 252, 29, 7, 78, 150, 66, 143, 174, 179, 66, 68, 42, 120, 8, 164, 46, - 52, 160, 207, 208, 231, 27, 130, 21, 85, 37, 208, 47, 244, + 70, 131, 32, 243, 131, 152, 215, 108, 51, 231, 184, 113, 117, 8, 164, 174, 151, + 152, 232, 29, 11, 58, 104, 46, 55, 81, 249, 207, 213, 77, 151, 237, ]), dst_hash: XorName([ - 207, 20, 30, 153, 250, 15, 151, 131, 100, 211, 67, 43, 61, 243, 191, 134, 242, 134, - 57, 183, 213, 94, 7, 240, 252, 121, 250, 158, 97, 246, 149, 112, + 85, 8, 26, 126, 9, 32, 28, 70, 112, 134, 226, 170, 46, 25, 115, 222, 131, 175, 117, + 141, 96, 45, 201, 108, 148, 142, 12, 27, 184, 109, 44, 70, ]), index: 0, src_size: 0, }, ChunkInfo { src_hash: XorName([ - 19, 159, 237, 131, 18, 84, 26, 161, 106, 99, 30, 134, 241, 30, 186, 36, 119, 74, - 59, 254, 246, 37, 96, 24, 200, 211, 236, 79, 53, 174, 252, 32, + 50, 175, 184, 213, 76, 189, 138, 227, 190, 200, 141, 26, 235, 78, 173, 171, 137, + 95, 43, 119, 8, 145, 253, 102, 189, 117, 247, 89, 246, 214, 129, 182, ]), dst_hash: XorName([ - 97, 110, 47, 182, 255, 22, 193, 218, 28, 21, 118, 43, 163, 189, 60, 14, 48, 88, - 197, 236, 146, 105, 40, 25, 53, 0, 90, 168, 159, 115, 143, 168, + 240, 135, 94, 165, 73, 209, 176, 218, 159, 232, 76, 254, 32, 84, 238, 245, 226, 2, + 227, 194, 95, 48, 125, 227, 42, 118, 85, 160, 39, 83, 2, 124, ]), index: 0, src_size: 0, }, ChunkInfo { src_hash: XorName([ - 185, 120, 111, 228, 41, 75, 228, 6, 222, 23, 163, 157, 32, 254, 96, 15, 210, 204, - 1, 147, 238, 121, 11, 33, 57, 5, 45, 54, 79, 237, 135, 139, + 160, 175, 104, 136, 24, 18, 192, 185, 147, 31, 227, 81, 212, 143, 214, 63, 52, 62, + 218, 48, 35, 220, 0, 184, 62, 137, 152, 35, 144, 149, 229, 86, ]), dst_hash: XorName([ - 52, 40, 33, 121, 186, 17, 252, 107, 128, 67, 227, 187, 86, 57, 142, 200, 119, 201, - 141, 120, 246, 70, 169, 99, 84, 208, 167, 233, 13, 125, 224, 168, + 198, 136, 45, 128, 93, 197, 174, 93, 27, 19, 218, 211, 184, 14, 214, 97, 182, 149, + 36, 161, 66, 19, 118, 105, 240, 100, 104, 1, 192, 87, 236, 132, ]), index: 0, src_size: 0, }, ChunkInfo { src_hash: XorName([ - 183, 193, 139, 225, 128, 162, 132, 138, 184, 75, 153, 229, 203, 147, 49, 174, 96, - 73, 135, 218, 79, 235, 79, 135, 162, 223, 248, 58, 82, 35, 196, 153, + 158, 201, 252, 234, 200, 107, 72, 126, 69, 234, 165, 203, 122, 90, 36, 46, 82, 183, + 61, 84, 128, 62, 118, 112, 222, 74, 164, 198, 20, 217, 96, 143, ]), dst_hash: XorName([ - 129, 161, 112, 120, 153, 202, 222, 238, 92, 86, 180, 251, 231, 79, 103, 59, 158, - 156, 53, 126, 49, 0, 223, 72, 66, 83, 34, 154, 249, 74, 147, 147, + 187, 81, 209, 66, 106, 200, 142, 130, 197, 102, 170, 211, 120, 197, 65, 210, 229, + 57, 27, 231, 120, 217, 180, 231, 34, 155, 32, 41, 78, 74, 193, 115, ]), index: 0, src_size: 0, }, ChunkInfo { src_hash: XorName([ - 53, 1, 114, 234, 6, 112, 255, 8, 148, 43, 130, 202, 155, 114, 99, 246, 81, 204, 77, - 60, 119, 237, 100, 198, 159, 144, 203, 60, 157, 246, 205, 22, + 208, 35, 197, 158, 225, 12, 21, 130, 132, 59, 227, 65, 238, 178, 232, 169, 186, 48, + 27, 106, 153, 46, 168, 196, 199, 70, 105, 236, 161, 167, 109, 43, ]), dst_hash: XorName([ - 235, 170, 170, 154, 173, 162, 71, 155, 236, 208, 97, 41, 167, 62, 209, 5, 255, 65, - 75, 239, 235, 133, 161, 30, 152, 3, 221, 99, 140, 207, 31, 64, + 145, 170, 97, 191, 204, 99, 185, 85, 4, 199, 204, 34, 104, 219, 97, 0, 184, 167, + 32, 173, 83, 249, 254, 42, 251, 10, 168, 231, 211, 67, 70, 120, ]), index: 0, src_size: 0, }, ChunkInfo { src_hash: XorName([ - 24, 147, 188, 118, 102, 72, 207, 163, 202, 63, 40, 237, 169, 100, 8, 190, 23, 67, - 243, 179, 196, 232, 214, 36, 76, 83, 220, 76, 241, 238, 107, 23, + 191, 47, 52, 224, 196, 196, 113, 118, 243, 7, 35, 213, 174, 114, 228, 229, 165, + 182, 217, 102, 55, 16, 174, 159, 197, 166, 75, 192, 182, 186, 173, 1, ]), dst_hash: XorName([ - 115, 143, 30, 6, 239, 108, 101, 10, 213, 216, 75, 254, 13, 110, 10, 245, 50, 189, - 83, 39, 63, 72, 11, 160, 107, 139, 123, 181, 64, 233, 190, 200, + 130, 233, 29, 245, 160, 80, 144, 117, 139, 251, 91, 240, 232, 173, 233, 168, 61, + 138, 88, 0, 92, 133, 16, 118, 29, 118, 131, 218, 42, 197, 132, 54, ]), index: 0, src_size: 0, }, ChunkInfo { src_hash: XorName([ - 100, 94, 19, 195, 150, 133, 161, 134, 150, 106, 44, 152, 201, 113, 171, 176, 147, - 244, 165, 93, 46, 227, 247, 118, 188, 29, 130, 19, 130, 137, 244, 15, + 116, 242, 114, 183, 140, 120, 52, 135, 104, 100, 112, 208, 10, 8, 99, 108, 78, 75, + 84, 111, 100, 57, 241, 143, 117, 172, 80, 19, 43, 142, 225, 227, ]), dst_hash: XorName([ - 120, 86, 200, 233, 111, 96, 122, 72, 234, 77, 181, 205, 248, 56, 175, 55, 124, 174, - 152, 163, 125, 67, 25, 33, 90, 151, 57, 103, 27, 123, 100, 148, + 0, 52, 220, 168, 128, 29, 228, 70, 0, 29, 73, 244, 83, 7, 171, 237, 31, 236, 231, + 24, 148, 14, 100, 16, 117, 82, 41, 11, 216, 126, 209, 127, ]), index: 0, src_size: 0, }, ChunkInfo { src_hash: XorName([ - 248, 242, 229, 119, 92, 211, 180, 222, 177, 34, 82, 94, 51, 178, 62, 12, 185, 77, - 145, 206, 168, 75, 176, 141, 46, 197, 1, 83, 199, 165, 37, 28, + 219, 177, 84, 234, 189, 172, 82, 64, 169, 100, 5, 56, 3, 43, 142, 126, 51, 235, + 194, 243, 30, 130, 132, 197, 137, 36, 170, 62, 46, 44, 176, 201, ]), dst_hash: XorName([ - 148, 17, 25, 147, 128, 108, 212, 70, 12, 32, 68, 96, 192, 215, 241, 123, 162, 224, - 223, 52, 230, 27, 100, 122, 97, 85, 148, 53, 103, 230, 21, 11, + 77, 246, 174, 53, 36, 156, 19, 157, 46, 142, 60, 60, 122, 133, 52, 118, 73, 80, 40, + 205, 174, 231, 211, 110, 38, 8, 189, 206, 102, 252, 166, 34, ]), index: 0, src_size: 0, }, ChunkInfo { src_hash: XorName([ - 68, 137, 10, 147, 116, 198, 83, 144, 156, 198, 68, 195, 94, 96, 125, 162, 213, 218, - 179, 255, 177, 143, 232, 48, 99, 204, 118, 246, 67, 243, 190, 96, + 65, 81, 63, 82, 119, 126, 216, 9, 44, 18, 160, 174, 225, 8, 202, 32, 245, 140, 14, + 169, 252, 209, 97, 96, 134, 165, 102, 106, 250, 196, 27, 70, ]), dst_hash: XorName([ - 30, 212, 77, 155, 165, 236, 65, 212, 88, 181, 48, 138, 226, 135, 144, 227, 132, - 195, 223, 199, 172, 235, 51, 146, 109, 209, 54, 63, 34, 169, 91, 55, + 42, 62, 224, 152, 136, 214, 91, 160, 125, 249, 229, 115, 81, 220, 213, 34, 29, 173, + 235, 99, 67, 210, 234, 160, 79, 254, 208, 174, 117, 127, 205, 36, ]), index: 0, src_size: 0, }, ChunkInfo { src_hash: XorName([ - 227, 224, 98, 89, 131, 120, 169, 214, 165, 171, 189, 187, 15, 7, 80, 133, 16, 63, - 74, 197, 17, 127, 22, 137, 171, 117, 34, 195, 186, 185, 51, 2, + 80, 237, 26, 5, 69, 59, 53, 210, 44, 236, 191, 69, 92, 39, 113, 124, 206, 169, 5, + 126, 189, 2, 146, 80, 68, 186, 142, 219, 37, 170, 135, 61, ]), dst_hash: XorName([ - 166, 232, 206, 232, 6, 23, 232, 20, 105, 230, 249, 86, 35, 117, 181, 65, 192, 245, - 65, 130, 238, 50, 188, 82, 193, 115, 172, 113, 237, 33, 248, 102, + 200, 203, 81, 29, 131, 156, 60, 140, 166, 254, 103, 60, 212, 223, 22, 41, 85, 192, + 140, 154, 33, 34, 188, 94, 84, 101, 62, 254, 164, 81, 209, 154, ]), index: 0, src_size: 0, }, ChunkInfo { src_hash: XorName([ - 199, 77, 9, 166, 29, 63, 254, 6, 165, 71, 110, 151, 121, 199, 60, 144, 197, 6, 92, - 182, 237, 202, 223, 171, 20, 80, 193, 237, 148, 96, 190, 70, + 176, 37, 236, 132, 229, 46, 239, 66, 127, 19, 235, 251, 254, 140, 231, 120, 170, + 173, 169, 2, 98, 159, 72, 160, 215, 103, 243, 7, 179, 63, 61, 173, ]), dst_hash: XorName([ - 221, 131, 122, 148, 84, 180, 72, 155, 240, 84, 4, 189, 156, 65, 164, 204, 215, 198, - 118, 227, 41, 95, 185, 117, 152, 128, 119, 205, 173, 180, 155, 86, + 160, 38, 187, 68, 9, 245, 147, 175, 244, 167, 195, 133, 79, 231, 89, 53, 165, 222, + 24, 162, 83, 158, 227, 193, 103, 232, 230, 209, 244, 58, 44, 208, ]), index: 0, src_size: 0, From 917d07e2a56ff3b4d113826a20a5eb2bcd8e9ab2 Mon Sep 17 00:00:00 2001 From: Josh Wilson Date: Thu, 26 Sep 2024 11:16:45 +0900 Subject: [PATCH 2/3] chore: clippy warnings fixed --- benches/lib.rs | 6 +----- src/chunk.rs | 1 - src/lib.rs | 6 +++--- src/tests.rs | 2 +- 4 files changed, 5 insertions(+), 10 deletions(-) diff --git a/benches/lib.rs b/benches/lib.rs index a7e9c9330..e3f67a5af 100644 --- a/benches/lib.rs +++ b/benches/lib.rs @@ -41,11 +41,7 @@ unused_qualifications, variant_size_differences )] -#![allow( - box_pointers, - missing_copy_implementations, - missing_debug_implementations -)] +#![allow(missing_copy_implementations, missing_debug_implementations)] use criterion::{BatchSize, Bencher, Criterion}; use self_encryption::{decrypt_full_set, encrypt, test_helpers::random_bytes}; diff --git a/src/chunk.rs b/src/chunk.rs index 603353438..ab65225c0 100644 --- a/src/chunk.rs +++ b/src/chunk.rs @@ -11,7 +11,6 @@ use bytes::Bytes; use rayon::prelude::*; use xor_name::XorName; -/// #[derive(Clone)] pub(crate) struct EncryptionBatch { pub(crate) raw_chunks: Vec, diff --git a/src/lib.rs b/src/lib.rs index 4d8d4cf7e..c57ba94ab 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -346,7 +346,7 @@ impl StreamSelfDecryptor { // Drain any in-order chunks due to the recent filled in piece. fn drain_unprocessed(&mut self) -> Result<()> { while let Some(chunk_name) = self.encrypted_chunks.get(&self.chunk_index) { - let file_path = self.temp_dir.path().join(&hex::encode(chunk_name)); + let file_path = self.temp_dir.path().join(hex::encode(chunk_name)); let mut chunk_file = File::open(file_path)?; let mut chunk_data = Vec::new(); let _ = chunk_file.read_to_end(&mut chunk_data)?; @@ -376,7 +376,7 @@ pub fn encrypt_from_file(file_path: &Path, output_dir: &Path) -> Result<(DataMap let chunk_name = XorName::from_content(&chunk.content); chunk_names.push(chunk_name); - let file_path = output_dir.join(&hex::encode(chunk_name)); + let file_path = output_dir.join(hex::encode(chunk_name)); let mut output_file = File::create(file_path)?; output_file.write_all(&chunk.content)?; } @@ -394,7 +394,7 @@ pub fn decrypt_from_chunk_files( let mut encrypted_chunks = Vec::new(); for chunk_info in data_map.infos() { let chunk_name = chunk_info.dst_hash; - let file_path = chunk_dir.join(&hex::encode(chunk_name)); + let file_path = chunk_dir.join(hex::encode(chunk_name)); let mut chunk_file = File::open(file_path)?; let mut chunk_data = Vec::new(); let _ = chunk_file.read_to_end(&mut chunk_data)?; diff --git a/src/tests.rs b/src/tests.rs index 73bfe2d21..eb7d50f81 100644 --- a/src/tests.rs +++ b/src/tests.rs @@ -82,7 +82,7 @@ fn test_stream_self_encryptor() -> Result<(), Error> { // Use the flushed encrypted chunks to recover the file and verify with the original data let mut flushed_encrypted_chunks = Vec::new(); for chunk_info in data_map.infos() { - let file_path = chunk_path.join(&hex::encode(chunk_info.dst_hash)); + let file_path = chunk_path.join(hex::encode(chunk_info.dst_hash)); let mut chunk_file = File::open(file_path)?; let mut chunk_data = Vec::new(); let _ = chunk_file.read_to_end(&mut chunk_data)?; From 653b34cab72ab550db6d113554ba1cbf54ed91c4 Mon Sep 17 00:00:00 2001 From: Josh Wilson Date: Thu, 26 Sep 2024 11:31:09 +0900 Subject: [PATCH 3/3] chore: update error derivation crates --- Cargo.toml | 2 +- src/error.rs | 32 ++++++++++++++++---------------- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 56cbf8eeb..c73136c8a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,7 +18,7 @@ lazy_static = "1.4.0" rand = "~0.8.5" rand_chacha = "~0.3.1" rayon = "1.5.1" -err-derive = "~0.3.1" +thiserror = "1.0" num_cpus = "1.13.0" itertools = "~0.10.0" tempfile = "3.6.0" diff --git a/src/error.rs b/src/error.rs index ea143b18f..c5e4278ce 100644 --- a/src/error.rs +++ b/src/error.rs @@ -7,8 +7,8 @@ // permissions and limitations relating to use of the SAFE Network Software. use bincode::ErrorKind; -use err_derive::Error; use std::io::Error as IoError; +use thiserror::Error; /// Specialisation of `std::Result` for crate. pub type Result = std::result::Result; @@ -17,26 +17,26 @@ pub type Result = std::result::Result; #[derive(Debug, Error)] #[allow(missing_docs)] pub enum Error { - #[error(display = "An error during compression or decompression.")] + #[error("An error during compression or decompression.")] Compression, - #[error(display = "An error during initializing CBC-AES cipher instance.")] + #[error("An error during initializing CBC-AES cipher instance.")] Cipher(String), - #[error(display = "An error within the symmetric encryption process.")] + #[error("An error within the symmetric encryption process.")] Encryption, - #[error(display = "An error within the symmetric decryption process({})", _0)] + #[error("An error within the symmetric decryption process({})", _0)] Decryption(String), - #[error(display = "A generic I/O error")] - Io(#[source] IoError), - #[error(display = "Generic error({})", _0)] + #[error("A generic I/O error")] + Io(#[from] IoError), + #[error("Generic error({})", _0)] Generic(String), - #[error(display = "Serialisation error")] - Bincode(#[source] Box), - #[error(display = "deserialization")] + #[error("Serialisation error")] + Bincode(#[from] Box), + #[error("deserialization")] Deserialise, - #[error(display = "num parse error")] - NumParse(#[source] std::num::ParseIntError), - #[error(display = "Rng error")] - Rng(#[source] rand::Error), - #[error(display = "Unable to obtain lock")] + #[error("num parse error")] + NumParse(#[from] std::num::ParseIntError), + #[error("Rng error")] + Rng(#[from] rand::Error), + #[error("Unable to obtain lock")] Poison, }