diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index ccdb9ffe22c23a..45a4c8dc631329 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -1739,6 +1739,7 @@ DEFINE_Validator(concurrency_stats_dump_interval_ms, DEFINE_mBool(cloud_mow_sync_rowsets_when_load_txn_begin, "true"); DEFINE_mBool(enable_cloud_make_rs_visible_on_be, "false"); +DEFINE_mInt32(file_handles_deplenish_frequency_times, "3"); // clang-format off #ifdef BE_TEST diff --git a/be/src/common/config.h b/be/src/common/config.h index 39e176e467250d..75d3b8e717b77a 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -1787,6 +1787,7 @@ DECLARE_mInt32(concurrency_stats_dump_interval_ms); DECLARE_mBool(cloud_mow_sync_rowsets_when_load_txn_begin); DECLARE_mBool(enable_cloud_make_rs_visible_on_be); +DECLARE_mInt32(file_handles_deplenish_frequency_times); #ifdef BE_TEST // test s3 diff --git a/be/src/common/metrics/system_metrics.cpp b/be/src/common/metrics/system_metrics.cpp index 5f28be0d5541fe..f111c04b491196 100644 --- a/be/src/common/metrics/system_metrics.cpp +++ b/be/src/common/metrics/system_metrics.cpp @@ -26,6 +26,7 @@ #include #include "common/cast_set.h" +#include "common/config.h" #include "runtime/memory/jemalloc_control.h" #include "util/cgroup_util.h" #include "util/perf_counters.h" @@ -432,6 +433,18 @@ void SystemMetrics::_update_cpu_metrics() { char buf[64]; LOG(WARNING) << "open /proc/stat failed, errno=" << errno << ", message=" << strerror_r(errno, buf, 64); + if (errno == 24) { + _file_handle_deplenish_counter++; + } else { + _file_handle_deplenish_counter = 0; + } + // Threshold of the number of consecutive failures + if (_file_handle_deplenish_counter >= config::file_handles_deplenish_frequency_times) { + LOG(FATAL) << "The system file handles are insufficient, causing service exceptions" + << ", BE will exit. please check the configs 'soft nofile'" + << " and 'hard nofile' of /etc/security/limits.conf "; + exit(-1); + } return; } diff --git a/be/src/common/metrics/system_metrics.h b/be/src/common/metrics/system_metrics.h index c2c7c8fa333443..52ca7299f703ac 100644 --- a/be/src/common/metrics/system_metrics.h +++ b/be/src/common/metrics/system_metrics.h @@ -121,6 +121,7 @@ class SystemMetrics { IntGauge* max_disk_io_util_percent = nullptr; IntGauge* max_network_send_bytes_rate = nullptr; IntGauge* max_network_receive_bytes_rate = nullptr; + int _file_handle_deplenish_counter = 0; }; } // namespace doris