diff --git a/docs/notes/2.25.x.md b/docs/notes/2.25.x.md index 75995b77cd7..5fcc952a969 100644 --- a/docs/notes/2.25.x.md +++ b/docs/notes/2.25.x.md @@ -31,6 +31,8 @@ one in `remote_store_headers` or `remote_execution_headers`. Pants now supports the `{chroot}` replacement marker in remote execution contexts. (With local and Docker execution, the `{chroot}` marker is replaced with the absolute path of the sandbox directory if it appears in program arguments or environment variables. Pants will do the same as well in remote execution contexts. This requires `/bin/bash` to be available on the remote execution server.) +The OpenDAL library powering the Github Actions cache backend has been updated, picking up some bug fixes for Github Enterprise Server instances using AWS S3 as backing storage for the Github Actions cache. + ### New Options System The "legacy" options system is removed in this release. All options parsing is now handled by the new, native parser. diff --git a/src/rust/engine/Cargo.lock b/src/rust/engine/Cargo.lock index 49229d0e6b5..f4add886369 100644 --- a/src/rust/engine/Cargo.lock +++ b/src/rust/engine/Cargo.lock @@ -98,19 +98,6 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b" -[[package]] -name = "async-compat" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b48b4ff0c2026db683dea961cd8ea874737f56cffca86fa84415eaddc51c00d" -dependencies = [ - "futures-core", - "futures-io", - "once_cell", - "pin-project-lite", - "tokio", -] - [[package]] name = "async-lock" version = "2.8.0" @@ -309,11 +296,11 @@ dependencies = [ [[package]] name = "backon" -version = "0.4.1" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c1a6197b2120bb2185a267f6515038558b019e92b832bb0320e96d66268dcf9" +checksum = "d67782c3f868daa71d3533538e98a8e13713231969def7536e8039606fc46bf0" dependencies = [ - "fastrand", + "fastrand 2.3.0", "futures-core", "pin-project", "tokio", @@ -351,9 +338,9 @@ checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" [[package]] name = "base64" -version = "0.21.0" +version = "0.21.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4a4ddaa51a5bc52a6948f74c06d20aaaddb71924eab79b8c97a8c556e942d6a" +checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" [[package]] name = "base64" @@ -1114,9 +1101,9 @@ checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" [[package]] name = "encoding_rs" -version = "0.8.28" +version = "0.8.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80df024fbc5ac80f87dfef0d9f5209a252f2a497f7f42944cff24d8253cac065" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" dependencies = [ "cfg-if 1.0.0", ] @@ -1257,6 +1244,12 @@ dependencies = [ "instant", ] +[[package]] +name = "fastrand" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" + [[package]] name = "filetime" version = "0.2.13" @@ -1525,8 +1518,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c05aeb6a22b8f62540c194aac980f2115af067bfe15a0734d7277a768d396b31" dependencies = [ "cfg-if 1.0.0", + "js-sys", "libc", "wasi 0.11.0+wasi-snapshot-preview1", + "wasm-bindgen", ] [[package]] @@ -1771,9 +1766,9 @@ dependencies = [ [[package]] name = "http-body" -version = "0.4.5" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" +checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" dependencies = [ "bytes", "http 0.2.12", @@ -1829,9 +1824,9 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "hyper" -version = "0.14.27" +version = "0.14.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffb1cfd654a8219eaef89881fdb3bb3b1cdc5fa75ded05d6933b2b382e395468" +checksum = "41dfc780fdec9373c01bae43289ea34c972e40ee3c9f6b3c8801a35f35586ce7" dependencies = [ "bytes", "futures-channel", @@ -1839,12 +1834,12 @@ dependencies = [ "futures-util", "h2 0.3.26", "http 0.2.12", - "http-body 0.4.5", + "http-body 0.4.6", "httparse", "httpdate", "itoa", "pin-project-lite", - "socket2 0.4.9", + "socket2", "tokio", "tower-service", "tracing", @@ -1949,7 +1944,7 @@ dependencies = [ "http-body 1.0.0", "hyper 1.4.1", "pin-project-lite", - "socket2 0.5.7", + "socket2", "tokio", "tower", "tower-service", @@ -2608,29 +2603,26 @@ checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575" [[package]] name = "opendal" -version = "0.41.0" +version = "0.45.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e31b48f0af6de5b3b344c1acc1e06c4581dca3e13cd5ba05269927fc2abf953a" +checksum = "52c17c077f23fa2d2c25d9d22af98baa43b8bbe2ef0de80cf66339aa70401467" dependencies = [ "anyhow", - "async-compat", "async-trait", "backon", - "base64 0.21.0", + "base64 0.21.7", "bytes", "chrono", "flagset", "futures", + "getrandom 0.2.8", "http 0.2.12", - "hyper 0.14.27", "log", "md-5", "once_cell", - "parking_lot 0.12.3", "percent-encoding", - "pin-project", "quick-xml", - "reqwest 0.11.22", + "reqwest 0.11.27", "serde", "serde_json", "tokio", @@ -3191,9 +3183,9 @@ dependencies = [ [[package]] name = "quick-xml" -version = "0.30.0" +version = "0.31.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eff6510e86862b57b210fd8cbe8ed3f0d7d600b9c2863cd4549a2e033c66e956" +checksum = "1004a344b30a54e2ee58d66a71b32d2db2feb0a31f9a2d302bf0536f15de2a33" dependencies = [ "memchr", "serde", @@ -3472,19 +3464,19 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.11.22" +version = "0.11.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "046cd98826c46c2ac8ddecae268eb5c2e58628688a5fc7a2643704a73faba95b" +checksum = "dd67538700a17451e7cba03ac727fb961abb7607553461627b97de0b89cf4a62" dependencies = [ - "base64 0.21.0", + "base64 0.21.7", "bytes", "encoding_rs", "futures-core", "futures-util", "h2 0.3.26", "http 0.2.12", - "http-body 0.4.5", - "hyper 0.14.27", + "http-body 0.4.6", + "hyper 0.14.32", "ipnet", "js-sys", "log", @@ -3495,6 +3487,7 @@ dependencies = [ "serde", "serde_json", "serde_urlencoded", + "sync_wrapper 0.1.2", "system-configuration", "tokio", "tokio-util 0.7.12", @@ -3502,7 +3495,7 @@ dependencies = [ "url", "wasm-bindgen", "wasm-bindgen-futures", - "wasm-streams 0.3.0", + "wasm-streams", "web-sys", "winreg 0.50.0", ] @@ -3544,7 +3537,7 @@ dependencies = [ "url", "wasm-bindgen", "wasm-bindgen-futures", - "wasm-streams 0.4.0", + "wasm-streams", "web-sys", "webpki-roots", "winreg 0.52.0", @@ -3991,16 +3984,6 @@ version = "1.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" -[[package]] -name = "socket2" -version = "0.4.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64a4a911eed85daf18834cfaa86a79b7d266ff93ff5ba14005426219480ed662" -dependencies = [ - "libc", - "winapi", -] - [[package]] name = "socket2" version = "0.5.7" @@ -4208,7 +4191,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9fbec84f381d5795b08656e4912bec604d162bff9291d6189a78f4c8ab87998" dependencies = [ "cfg-if 1.0.0", - "fastrand", + "fastrand 1.9.0", "redox_syscall 0.3.5", "rustix 0.37.1", "windows-sys 0.45.0", @@ -4353,7 +4336,7 @@ dependencies = [ "parking_lot 0.12.3", "pin-project-lite", "signal-hook-registry", - "socket2 0.5.7", + "socket2", "tokio-macros", "windows-sys 0.52.0", ] @@ -4473,7 +4456,7 @@ dependencies = [ "prost", "rustls-native-certs 0.8.0", "rustls-pemfile", - "socket2 0.5.7", + "socket2", "tokio", "tokio-rustls 0.26.0", "tokio-stream", @@ -4866,19 +4849,6 @@ version = "0.2.93" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c62a0a307cb4a311d3a07867860911ca130c3494e8c2719593806c08bc5d0484" -[[package]] -name = "wasm-streams" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4609d447824375f43e1ffbc051b50ad8f4b3ae8219680c94452ea05eb240ac7" -dependencies = [ - "futures-util", - "js-sys", - "wasm-bindgen", - "wasm-bindgen-futures", - "web-sys", -] - [[package]] name = "wasm-streams" version = "0.4.0" diff --git a/src/rust/engine/Cargo.toml b/src/rust/engine/Cargo.toml index 0740d2a0fdc..1ed8b3d7a0c 100644 --- a/src/rust/engine/Cargo.toml +++ b/src/rust/engine/Cargo.toml @@ -275,7 +275,7 @@ notify = { git = "https://github.com/pantsbuild/notify", rev = "276af0f3c5f300bf num_cpus = "1" num_enum = "0.5" once_cell = "1.20" -opendal = { version = "0.41", default-features = false, features = [ +opendal = { version = "0.45.1", default-features = false, features = [ "services-memory", "services-fs", "services-ghac", diff --git a/src/rust/engine/remote_provider/remote_provider_opendal/src/lib.rs b/src/rust/engine/remote_provider/remote_provider_opendal/src/lib.rs index 06203340359..1d754ea06bb 100644 --- a/src/rust/engine/remote_provider/remote_provider_opendal/src/lib.rs +++ b/src/rust/engine/remote_provider/remote_provider_opendal/src/lib.rs @@ -58,8 +58,7 @@ impl Provider { // TODO: record Metric::RemoteStoreRequestTimeouts for timeouts TimeoutLayer::new() .with_timeout(options.timeout) - // TimeoutLayer requires specifying a non-zero minimum transfer speed too. - .with_speed(1), + .with_io_timeout(options.timeout), ) // TODO: RetryLayer doesn't seem to retry stores, but we should .layer(RetryLayer::new().with_max_times(options.retries + 1)) @@ -78,7 +77,7 @@ impl Provider { pub fn fs(path: &str, scope: String, options: RemoteStoreOptions) -> Result { let mut builder = opendal::services::Fs::default(); - builder.root(path).enable_path_check(); + builder.root(path); Provider::new(builder, scope, options) } @@ -161,9 +160,12 @@ impl Provider { match mode { LoadMode::Validate => { - let correct_digest = async_verified_copy(digest, false, &mut reader, destination) - .await - .map_err(|e| format!("failed to read {}: {}", path, e))?; + let correct_digest = + match async_verified_copy(digest, false, &mut reader, destination).await { + Ok(correct_digest) => correct_digest, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(false), + Err(e) => return Err(format!("failed to read {}: {}", path, e)), + }; if !correct_digest { // TODO: include the actual digest here @@ -171,9 +173,11 @@ impl Provider { } } LoadMode::NoValidate => { - tokio::io::copy(&mut reader, destination) - .await - .map_err(|e| format!("failed to read {}: {}", path, e))?; + match tokio::io::copy(&mut reader, destination).await { + Ok(result) => result, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(false), + Err(e) => return Err(format!("failed to read {}: {}", path, e)), + }; } } Ok(true)