Refactor uv retrayble strategy to use a single code path

Refactoring that allows uv's retryable strategy to return Some(Retryable::Fatal), which is also helpful for #16245.

We need to change the way we're reporting retries to avoid both the retry middleware and our own retry context to report the retry numbers.
This commit is contained in:
konstin 2025-12-12 15:36:06 +01:00
parent a2d64aa224
commit 7571ffa385
4 changed files with 72 additions and 50 deletions

View File

@ -20,8 +20,8 @@ use reqwest::{Client, ClientBuilder, IntoUrl, Proxy, Request, Response, multipar
use reqwest_middleware::{ClientWithMiddleware, Middleware}; use reqwest_middleware::{ClientWithMiddleware, Middleware};
use reqwest_retry::policies::ExponentialBackoff; use reqwest_retry::policies::ExponentialBackoff;
use reqwest_retry::{ use reqwest_retry::{
DefaultRetryableStrategy, RetryTransientMiddleware, Retryable, RetryableStrategy, RetryTransientMiddleware, Retryable, RetryableStrategy, default_on_request_error,
default_on_request_error, default_on_request_success,
}; };
use thiserror::Error; use thiserror::Error;
use tracing::{debug, trace}; use tracing::{debug, trace};
@ -1015,21 +1015,15 @@ impl<'a> RequestBuilder<'a> {
} }
} }
/// Extends [`DefaultRetryableStrategy`], to log transient request failures and additional retry cases. /// An extension over [`DefaultRetryableStrategy`] that logs transient request failures and
/// adds additional retry cases.
pub struct UvRetryableStrategy; pub struct UvRetryableStrategy;
impl RetryableStrategy for UvRetryableStrategy { impl RetryableStrategy for UvRetryableStrategy {
fn handle(&self, res: &Result<Response, reqwest_middleware::Error>) -> Option<Retryable> { fn handle(&self, res: &Result<Response, reqwest_middleware::Error>) -> Option<Retryable> {
// Use the default strategy and check for additional transient error cases. let retryable = match res {
let retryable = match DefaultRetryableStrategy.handle(res) { Ok(success) => default_on_request_success(success),
None | Some(Retryable::Fatal) Err(err) => retryable_on_request_failure(err),
if res
.as_ref()
.is_err_and(|err| is_transient_network_error(err)) =>
{
Some(Retryable::Transient)
}
default => default,
}; };
// Log on transient errors // Log on transient errors
@ -1055,11 +1049,13 @@ impl RetryableStrategy for UvRetryableStrategy {
/// Whether the error looks like a network error that should be retried. /// Whether the error looks like a network error that should be retried.
/// ///
/// There are two cases that the default retry strategy is missing: /// This is an extension over [`reqwest_middleware::default_on_request_failure`], which is missing
/// a number of cases:
/// * Inside the reqwest or reqwest-middleware error is an `io::Error` such as a broken pipe /// * Inside the reqwest or reqwest-middleware error is an `io::Error` such as a broken pipe
/// * When streaming a response, a reqwest error may be hidden several layers behind errors /// * When streaming a response, a reqwest error may be hidden several layers behind errors
/// of different crates processing the stream, including `io::Error` layers. /// of different crates processing the stream, including `io::Error` layers
pub fn is_transient_network_error(err: &(dyn Error + 'static)) -> bool { /// * Any `h2` error
fn retryable_on_request_failure(err: &(dyn Error + 'static)) -> Option<Retryable> {
// First, try to show a nice trace log // First, try to show a nice trace log
if let Some((Some(status), Some(url))) = find_source::<WrappedReqwestError>(&err) if let Some((Some(status), Some(url))) = find_source::<WrappedReqwestError>(&err)
.map(|request_err| (request_err.status(), request_err.url())) .map(|request_err| (request_err.status(), request_err.url()))
@ -1074,29 +1070,32 @@ pub fn is_transient_network_error(err: &(dyn Error + 'static)) -> bool {
// crates // crates
let mut current_source = Some(err); let mut current_source = Some(err);
while let Some(source) = current_source { while let Some(source) = current_source {
if let Some(reqwest_err) = source.downcast_ref::<WrappedReqwestError>() { // Handle different kinds of reqwest error nesting not accessible by downcast.
has_known_error = true; let reqwest_err = if let Some(reqwest_err) = source.downcast_ref::<reqwest::Error>() {
if let reqwest_middleware::Error::Reqwest(reqwest_err) = &**reqwest_err { Some(reqwest_err)
if default_on_request_error(reqwest_err) == Some(Retryable::Transient) { } else if let Some(reqwest_err) = source
trace!("Retrying nested reqwest middleware error"); .downcast_ref::<WrappedReqwestError>()
return true; .and_then(|err| err.inner())
} {
if is_retryable_status_error(reqwest_err) { Some(reqwest_err)
trace!("Retrying nested reqwest middleware status code error"); } else if let Some(reqwest_middleware::Error::Reqwest(reqwest_err)) =
return true; source.downcast_ref::<reqwest_middleware::Error>()
} {
} Some(reqwest_err)
} else {
None
};
trace!("Cannot retry nested reqwest middleware error"); if let Some(reqwest_err) = reqwest_err {
} else if let Some(reqwest_err) = source.downcast_ref::<reqwest::Error>() {
has_known_error = true; has_known_error = true;
// Ignore the default retry strategy returning fatal.
if default_on_request_error(reqwest_err) == Some(Retryable::Transient) { if default_on_request_error(reqwest_err) == Some(Retryable::Transient) {
trace!("Retrying nested reqwest error"); trace!("Retrying nested reqwest error");
return true; return Some(Retryable::Transient);
} }
if is_retryable_status_error(reqwest_err) { if is_retryable_status_error(reqwest_err) {
trace!("Retrying nested reqwest status code error"); trace!("Retrying nested reqwest status code error");
return true; return Some(Retryable::Transient);
} }
trace!("Cannot retry nested reqwest error"); trace!("Cannot retry nested reqwest error");
@ -1104,7 +1103,7 @@ pub fn is_transient_network_error(err: &(dyn Error + 'static)) -> bool {
// All h2 errors look like errors that should be retried // All h2 errors look like errors that should be retried
// https://github.com/astral-sh/uv/issues/15916 // https://github.com/astral-sh/uv/issues/15916
trace!("Retrying nested h2 error"); trace!("Retrying nested h2 error");
return true; return Some(Retryable::Transient);
} else if let Some(io_err) = source.downcast_ref::<io::Error>() { } else if let Some(io_err) = source.downcast_ref::<io::Error>() {
has_known_error = true; has_known_error = true;
let retryable_io_err_kinds = [ let retryable_io_err_kinds = [
@ -1121,7 +1120,7 @@ pub fn is_transient_network_error(err: &(dyn Error + 'static)) -> bool {
]; ];
if retryable_io_err_kinds.contains(&io_err.kind()) { if retryable_io_err_kinds.contains(&io_err.kind()) {
trace!("Retrying error: `{}`", io_err.kind()); trace!("Retrying error: `{}`", io_err.kind());
return true; return Some(Retryable::Transient);
} }
trace!( trace!(
@ -1136,7 +1135,12 @@ pub fn is_transient_network_error(err: &(dyn Error + 'static)) -> bool {
if !has_known_error { if !has_known_error {
trace!("Cannot retry error: Neither an IO error nor a reqwest error"); trace!("Cannot retry error: Neither an IO error nor a reqwest error");
} }
false
None
}
pub fn is_transient_network_error(err: &(dyn Error + 'static)) -> bool {
retryable_on_request_failure(err) == Some(Retryable::Transient)
} }
/// Whether the error is a status code error that is retryable. /// Whether the error is a status code error that is retryable.
@ -1397,7 +1401,7 @@ mod tests {
.await; .await;
let middleware_retry = let middleware_retry =
DefaultRetryableStrategy.handle(&response) == Some(Retryable::Transient); UvRetryableStrategy.handle(&response) == Some(Retryable::Transient);
let response = client let response = client
.get(format!("{}/{}", server.uri(), status)) .get(format!("{}/{}", server.uri(), status))

View File

@ -425,13 +425,33 @@ impl WrappedReqwestError {
problem_details: Option<ProblemDetails>, problem_details: Option<ProblemDetails>,
) -> Self { ) -> Self {
Self { Self {
error, error: Self::filter_retries_from_error(error),
problem_details: problem_details.map(Box::new), problem_details: problem_details.map(Box::new),
} }
} }
/// Drop `RetryError::WithRetries` to avoid reporting the number of retries twice.
///
/// We attach the number of errors outside by adding the retry counts from the retry middleware
/// and from uv's outer retry loop for streaming bodies. Stripping the inner count from the
/// error context avoids showing two numbers.
fn filter_retries_from_error(error: reqwest_middleware::Error) -> reqwest_middleware::Error {
match error {
reqwest_middleware::Error::Middleware(error) => {
match error.downcast::<reqwest_retry::RetryError>() {
Ok(
reqwest_retry::RetryError::WithRetries { err, .. }
| reqwest_retry::RetryError::Error(err),
) => err,
Err(error) => reqwest_middleware::Error::Middleware(error),
}
}
error @ reqwest_middleware::Error::Reqwest(_) => error,
}
}
/// Return the inner [`reqwest::Error`] from the error chain, if it exists. /// Return the inner [`reqwest::Error`] from the error chain, if it exists.
fn inner(&self) -> Option<&reqwest::Error> { pub fn inner(&self) -> Option<&reqwest::Error> {
match &self.error { match &self.error {
reqwest_middleware::Error::Reqwest(err) => Some(err), reqwest_middleware::Error::Reqwest(err) => Some(err),
reqwest_middleware::Error::Middleware(err) => err.chain().find_map(|err| { reqwest_middleware::Error::Middleware(err) => err.chain().find_map(|err| {
@ -495,6 +515,7 @@ impl WrappedReqwestError {
impl From<reqwest::Error> for WrappedReqwestError { impl From<reqwest::Error> for WrappedReqwestError {
fn from(error: reqwest::Error) -> Self { fn from(error: reqwest::Error) -> Self {
Self { Self {
// No need to filter retries as this error does not have retries.
error: error.into(), error: error.into(),
problem_details: None, problem_details: None,
} }
@ -504,7 +525,7 @@ impl From<reqwest::Error> for WrappedReqwestError {
impl From<reqwest_middleware::Error> for WrappedReqwestError { impl From<reqwest_middleware::Error> for WrappedReqwestError {
fn from(error: reqwest_middleware::Error) -> Self { fn from(error: reqwest_middleware::Error) -> Self {
Self { Self {
error, error: Self::filter_retries_from_error(error),
problem_details: None, problem_details: None,
} }
} }

View File

@ -131,14 +131,11 @@ impl Error {
if let Self::NetworkErrorWithRetries { retries, .. } = self { if let Self::NetworkErrorWithRetries { retries, .. } = self {
return retries + 1; return retries + 1;
} }
// TODO(jack): let-chains are stable as of Rust 1.88. We should use them here as soon as if let Self::NetworkMiddlewareError(_, anyhow_error) = self
// our rust-version is high enough. && let Some(RetryError::WithRetries { retries, .. }) =
if let Self::NetworkMiddlewareError(_, anyhow_error) = self {
if let Some(RetryError::WithRetries { retries, .. }) =
anyhow_error.downcast_ref::<RetryError>() anyhow_error.downcast_ref::<RetryError>()
{ {
return retries + 1; return retries + 1;
}
} }
1 1
} }

View File

@ -83,8 +83,8 @@ async fn simple_io_err() {
----- stdout ----- ----- stdout -----
----- stderr ----- ----- stderr -----
error: Failed to fetch: `[SERVER]/tqdm/` error: Request failed after 3 retries
Caused by: Request failed after 3 retries Caused by: Failed to fetch: `[SERVER]/tqdm/`
Caused by: error sending request for url ([SERVER]/tqdm/) Caused by: error sending request for url ([SERVER]/tqdm/)
Caused by: client error (SendRequest) Caused by: client error (SendRequest)
Caused by: connection closed before message completed Caused by: connection closed before message completed
@ -141,8 +141,8 @@ async fn find_links_io_error() {
----- stderr ----- ----- stderr -----
error: Failed to read `--find-links` URL: [SERVER]/ error: Failed to read `--find-links` URL: [SERVER]/
Caused by: Failed to fetch: `[SERVER]/`
Caused by: Request failed after 3 retries Caused by: Request failed after 3 retries
Caused by: Failed to fetch: `[SERVER]/`
Caused by: error sending request for url ([SERVER]/) Caused by: error sending request for url ([SERVER]/)
Caused by: client error (SendRequest) Caused by: client error (SendRequest)
Caused by: connection closed before message completed Caused by: connection closed before message completed
@ -200,8 +200,8 @@ async fn direct_url_io_error() {
----- stderr ----- ----- stderr -----
× Failed to download `tqdm @ [SERVER]/packages/d0/30/dc54f88dd4a2b5dc8a0279bdd7270e735851848b762aeb1c1184ed1f6b14/tqdm-4.67.1-py3-none-any.whl` × Failed to download `tqdm @ [SERVER]/packages/d0/30/dc54f88dd4a2b5dc8a0279bdd7270e735851848b762aeb1c1184ed1f6b14/tqdm-4.67.1-py3-none-any.whl`
Failed to fetch: `[SERVER]/packages/d0/30/dc54f88dd4a2b5dc8a0279bdd7270e735851848b762aeb1c1184ed1f6b14/tqdm-4.67.1-py3-none-any.whl`
Request failed after 3 retries Request failed after 3 retries
Failed to fetch: `[SERVER]/packages/d0/30/dc54f88dd4a2b5dc8a0279bdd7270e735851848b762aeb1c1184ed1f6b14/tqdm-4.67.1-py3-none-any.whl`
error sending request for url ([SERVER]/packages/d0/30/dc54f88dd4a2b5dc8a0279bdd7270e735851848b762aeb1c1184ed1f6b14/tqdm-4.67.1-py3-none-any.whl) error sending request for url ([SERVER]/packages/d0/30/dc54f88dd4a2b5dc8a0279bdd7270e735851848b762aeb1c1184ed1f6b14/tqdm-4.67.1-py3-none-any.whl)
client error (SendRequest) client error (SendRequest)
connection closed before message completed connection closed before message completed