-
-
Notifications
You must be signed in to change notification settings - Fork 644
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Update openDAL to address AWS/GHAC issues #21779
Changes from all commits
4087f06
ec68d72
612909f
533940f
5bc931e
89c9dd0
056170a
f291419
c2bc736
b5fbc1c
2e706fe
a01e1c6
a4494d9
dd89613
cfdb9a5
460a904
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -58,8 +58,7 @@ impl Provider { | |
// TODO: record Metric::RemoteStoreRequestTimeouts for timeouts | ||
TimeoutLayer::new() | ||
.with_timeout(options.timeout) | ||
// TimeoutLayer requires specifying a non-zero minimum transfer speed too. | ||
.with_speed(1), | ||
.with_io_timeout(options.timeout), | ||
) | ||
// TODO: RetryLayer doesn't seem to retry stores, but we should | ||
.layer(RetryLayer::new().with_max_times(options.retries + 1)) | ||
|
@@ -78,7 +77,7 @@ impl Provider { | |
|
||
pub fn fs(path: &str, scope: String, options: RemoteStoreOptions) -> Result<Provider, String> { | ||
let mut builder = opendal::services::Fs::default(); | ||
builder.root(path).enable_path_check(); | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. The compiler told me "path always checked since RFC-3243 List Prefix" and this There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Hm, I think we do still need to set
I think it's just Background: these calls are on the same line but operate independently: using the "builder pattern" in Rust, each of them mutates let mut builder = ...;
builder.root(path);
builder.enable_path_check(); There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This makes sense. Thanks for the info! I've put path back in, and it seems to be working fine |
||
builder.root(path); | ||
Provider::new(builder, scope, options) | ||
} | ||
|
||
|
@@ -161,19 +160,24 @@ impl Provider { | |
|
||
match mode { | ||
LoadMode::Validate => { | ||
let correct_digest = async_verified_copy(digest, false, &mut reader, destination) | ||
.await | ||
.map_err(|e| format!("failed to read {}: {}", path, e))?; | ||
let correct_digest = | ||
match async_verified_copy(digest, false, &mut reader, destination).await { | ||
Ok(correct_digest) => correct_digest, | ||
Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(false), | ||
Err(e) => return Err(format!("failed to read {}: {}", path, e)), | ||
}; | ||
|
||
if !correct_digest { | ||
// TODO: include the actual digest here | ||
return Err(format!("Remote CAS gave wrong digest: expected {digest:?}")); | ||
} | ||
} | ||
LoadMode::NoValidate => { | ||
tokio::io::copy(&mut reader, destination) | ||
.await | ||
.map_err(|e| format!("failed to read {}: {}", path, e))?; | ||
match tokio::io::copy(&mut reader, destination).await { | ||
Ok(result) => result, | ||
Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(false), | ||
Err(e) => return Err(format!("failed to read {}: {}", path, e)), | ||
}; | ||
} | ||
} | ||
Ok(true) | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
with_speed
has a deprecation warning, suggesting to usewith_io_timeout
instead. As I understand it,with_timeout
andwith_io_timeout
each apply a timeout to different operations, so both are required: https://opendal.apache.org/docs/rust/opendal/layers/struct.TimeoutLayer.html#notes