Fix timeout layer not working with retry layer (#5775)

* Add test that shows the problem

* Fix metastore layer order

* Revert "Add test that shows the problem"

This reverts commit 09fc80f.

Author: rdettai

quickwit/quickwit-common/src/tower/timeout.rs

@@ -95,6 +95,9 @@ where
 ///
 /// If the inner service does not complete within the specified duration,
 /// the response will be aborted with the error `TimeoutExceeded`.
+///
+/// Note that when used in combination with a retry layer, this should be
+/// stacked on top of it for the timeout to be retried.
 #[derive(Debug, Clone)]
 pub struct TimeoutLayer {
     timeout: Duration,

quickwit/quickwit-serve/src/lib.rs

@@ -463,8 +463,8 @@ pub async fn serve_quickwit(
                 bail!("could not find any metastore node in the cluster");
             }
             MetastoreServiceClient::tower()
-                .stack_layer(TimeoutLayer::new(GRPC_METASTORE_SERVICE_TIMEOUT))
                 .stack_layer(RetryLayer::new(RetryPolicy::from(RetryParams::standard())))
+                .stack_layer(TimeoutLayer::new(GRPC_METASTORE_SERVICE_TIMEOUT))
                 .stack_layer(METASTORE_GRPC_CLIENT_METRICS_LAYER.clone())
                 .stack_layer(tower::limit::GlobalConcurrencyLimitLayer::new(
                     get_metastore_client_max_concurrency(),