chore: Add timeout setting for find_ttl. (#5088)

GreptimeTeam · Dec 6, 2024 · 19373d8 · 19373d8
1 parent 3133f3f
commit 19373d8
Show file tree

Hide file tree

Showing 3 changed files with 25 additions and 7 deletions.
diff --git a/src/mito2/src/compaction.rs b/src/mito2/src/compaction.rs
@@ -51,6 +51,7 @@ use crate::config::MitoConfig;
 use crate::error::{
     CompactRegionSnafu, Error, GetSchemaMetadataSnafu, RegionClosedSnafu, RegionDroppedSnafu,
     RegionTruncatedSnafu, RemoteCompactionSnafu, Result, TimeRangePredicateOverflowSnafu,
+    TimeoutSnafu,
 };
 use crate::metrics::COMPACTION_STAGE_ELAPSED;
 use crate::read::projection::ProjectionMapper;
@@ -445,13 +446,17 @@ async fn find_ttl(
         return Ok(table_ttl);
     }
 
-    let ttl = schema_metadata_manager
-        .get_schema_options_by_table_id(table_id)
-        .await
-        .context(GetSchemaMetadataSnafu)?
-        .and_then(|options| options.ttl)
-        .unwrap_or_default()
-        .into();
+    let ttl = tokio::time::timeout(
+        crate::config::FETCH_OPTION_TIMEOUT,
+        schema_metadata_manager.get_schema_options_by_table_id(table_id),
+    )
+    .await
+    .context(TimeoutSnafu)?
+    .context(GetSchemaMetadataSnafu)?
+    .and_then(|options| options.ttl)
+    .unwrap_or_default()
+    .into();
+
     Ok(ttl)
 }
 

diff --git a/src/mito2/src/config.rs b/src/mito2/src/config.rs
@@ -45,6 +45,9 @@ const PAGE_CACHE_SIZE_FACTOR: u64 = 8;
 /// Use `1/INDEX_CREATE_MEM_THRESHOLD_FACTOR` of OS memory size as mem threshold for creating index
 const INDEX_CREATE_MEM_THRESHOLD_FACTOR: u64 = 16;
 
+/// Fetch option timeout
+pub(crate) const FETCH_OPTION_TIMEOUT: Duration = Duration::from_secs(10);
+
 /// Configuration for [MitoEngine](crate::engine::MitoEngine).
 #[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)]
 #[serde(default)]

diff --git a/src/mito2/src/error.rs b/src/mito2/src/error.rs
@@ -30,6 +30,7 @@ use snafu::{Location, Snafu};
 use store_api::logstore::provider::Provider;
 use store_api::manifest::ManifestVersion;
 use store_api::storage::RegionId;
+use tokio::time::error::Elapsed;
 
 use crate::cache::file_cache::FileType;
 use crate::region::{RegionLeaderState, RegionRoleState};
@@ -877,6 +878,14 @@ pub enum Error {
         #[snafu(implicit)]
         location: Location,
     },
+
+    #[snafu(display("Timeout"))]
+    Timeout {
+        #[snafu(source)]
+        error: Elapsed,
+        #[snafu(implicit)]
+        location: Location,
+    },
 }
 
 pub type Result<T, E = Error> = std::result::Result<T, E>;
@@ -1010,6 +1019,7 @@ impl ErrorExt for Error {
             DecodeStats { .. } | StatsNotPresent { .. } => StatusCode::Internal,
             RegionBusy { .. } => StatusCode::RegionBusy,
             GetSchemaMetadata { source, .. } => source.status_code(),
+            Timeout { .. } => StatusCode::Cancelled,
         }
     }