17 #define dprintf(...) printf(__VA_ARGS__)
19 #define dprintf(...) (void(0))
27 const char *m_traceID =
"ResourceMonitor";
48 m_dir_scan_mutex.
Lock();
49 if (m_dir_scan_in_progress) {
50 bool dir_checked =
false;
51 m_dir_scan_open_requests.push_back({lfn, cond, dir_checked});
54 while ( ! dir_checked)
62 void ResourceMonitor::check_inter_dir_scan_open_requests()
67 auto now = std::chrono::steady_clock::now();
68 if (now - m_dir_scan_last_check < s_dir_scan_check_interval)
70 m_dir_scan_last_check = now;
72 process_inter_dir_scan_open_requests();
75 void ResourceMonitor::process_inter_dir_scan_open_requests()
77 m_dir_scan_mutex.
Lock();
80 size_t n_to_process = m_dir_scan_open_requests.size();
81 if (n_to_process == 0)
89 while (n_to_process-- > 0)
91 LfnCondRecord &lcr = m_dir_scan_open_requests.front();
94 cross_check_or_process_oob_lfn(lcr.f_lfn, fst);
100 m_dir_scan_mutex.
Lock();
101 m_dir_scan_open_requests.pop_front();
103 m_dir_scan_mutex.
UnLock();
106 void ResourceMonitor::cross_check_or_process_oob_lfn(
const std::string &lfn,
FsTraversal &fst)
110 static const char *trc_pfx =
"cross_check_or_process_oob_lfn() ";
115 DirState *last_existing_ds =
nullptr;
120 size_t pos = lfn.find_last_of(
"/");
121 std::string dir = (pos == std::string::npos) ?
"" : lfn.substr(0, pos);
133 if (it->second.has_data && it->second.has_cinfo) {
134 here.
m_StBlocks += it->second.stat_data.st_blocks;
147 dprintf(
"In scan_dir_and_recurse for '%s', size of dir_vec = %d, file_stat_map = %d\n",
158 dprintf(
"would be doing something with %s ... has_data=%d, has_cinfo=%d\n",
159 it->first.c_str(), it->second.has_data, it->second.has_cinfo);
167 if (it->second.has_data && it->second.has_cinfo) {
168 here.
m_StBlocks += it->second.stat_data.st_blocks;
176 std::vector<std::string> dirs;
179 check_inter_dir_scan_open_requests();
183 for (
auto &dname : dirs)
205 fst.
m_readdir_hook = [
this]() { check_inter_dir_scan_open_requests(); };
220 m_dir_scan_in_progress =
false;
223 while ( ! m_dir_scan_open_requests.empty())
225 LfnCondRecord &lcr = m_dir_scan_open_requests.front();
227 lcr.f_checked =
true;
231 m_dir_scan_open_requests.pop_front();
249 static const char *trc_pfx =
"process_queues() ";
260 n_records += m_file_open_q.swap_queues();
261 n_records += m_file_update_stats_q.swap_queues();
262 n_records += m_file_close_q.swap_queues();
263 n_records += m_file_purge_q1.swap_queues();
264 n_records += m_file_purge_q2.swap_queues();
265 n_records += m_file_purge_q3.swap_queues();
269 for (
auto &i : m_file_open_q.read_queue())
272 AccessToken &at =
token(i.id);
273 dprintf(
"process file open for token %d, time %ld -- %s\n",
274 i.id, i.record.m_open_time, at.m_filename.c_str());
279 DirState *last_existing_ds =
nullptr;
287 for (
auto &i : m_file_update_stats_q.read_queue())
290 AccessToken &at =
token(i.id);
293 dprintf(
"process file update for token %d, %p -- %s\n",
294 i.id, ds, at.m_filename.c_str());
297 m_current_usage_in_st_blocks += i.record.m_StBlocksAdded;
300 for (
auto &i : m_file_close_q.read_queue())
303 AccessToken &at =
token(i.id);
304 dprintf(
"process file close for token %d, time %ld -- %s\n",
305 i.id, i.record.m_close_time, at.m_filename.c_str());
314 for (
auto &i : m_file_close_q.read_queue())
318 for (
auto &i : m_file_purge_q1.read_queue())
324 m_current_usage_in_st_blocks -= i.record.m_size_in_st_blocks;
326 for (
auto &i : m_file_purge_q2.read_queue())
331 TRACE(
Error, trc_pfx <<
"DirState not found for directory path '" << i.id <<
"'.");
337 m_current_usage_in_st_blocks -= i.record.m_size_in_st_blocks;
339 for (
auto &i : m_file_purge_q3.read_queue())
344 TRACE(
Error, trc_pfx <<
"DirState not found for LFN path '" << i.id <<
"'.");
349 m_current_usage_in_st_blocks -= i.record;
364 static const char *tpfx =
"heart_beat() ";
368 const int s_queue_proc_interval = 10;
371 const int s_purge_check_interval = 60;
377 time_t now = time(0);
378 time_t next_queue_proc_time = now + s_queue_proc_interval;
379 time_t next_sshot_report_time = (now / 60) * 60 + 60;
380 time_t next_purge_check_time = now + s_purge_check_interval;
381 time_t next_purge_report_time = now + s_purge_report_interval;
382 time_t next_purge_cold_files_time = now + s_purge_cold_files_interval;
384 long long n_reported_token_failures = 0;
388 time_t start = time(0);
389 time_t next_event = std::min({ next_queue_proc_time, next_sshot_report_time,
390 next_purge_check_time, next_purge_report_time, next_purge_cold_files_time });
392 if (next_event > start)
394 unsigned int t_sleep = next_event - start;
395 TRACE(Dump, tpfx <<
"sleeping for " << t_sleep <<
" seconds until the next beat.");
410 time_t queue_swap_time = time(0);
414 next_queue_proc_time = queue_swap_time + s_queue_proc_interval;
415 TRACE(Dump, tpfx <<
"process_queues -- n_records=" << n_processed);
420 if (taf > n_reported_token_failures) {
421 TRACE(
Error, tpfx <<
"access-token space exhausted "
422 << taf - n_reported_token_failures <<
" time(s) (total " << taf
423 <<
"); those files are open but not accounted for. Raise MaxChunks "
424 "on ResourceMonitor::m_access_tokens.");
425 n_reported_token_failures = taf;
436 bool do_sshot_report = next_sshot_report_time <= now;
437 bool do_purge_check = next_purge_check_time <= now;
438 bool do_purge_report = next_purge_report_time <= now;
439 bool do_purge_cold_files = next_purge_cold_files_time <= now;
442 if (do_sshot_report || do_purge_check || do_purge_report || do_purge_cold_files)
444 unlink_func unlink_foo = [&](
const std::string &dp)->
int {
445 int ret = m_oss.
Unlink(dp.c_str());
447 TRACE(
Info, tpfx <<
"Empty dir unlink error: " << ret <<
" at " << dp);
449 TRACE(
Debug, tpfx <<
"Empty dir unlink success: " << dp);
477 next_sshot_report_time = ((now + 1) / s_sshot_report_interval) * s_sshot_report_interval + s_sshot_report_interval;
493 const char* dumpfile =
"/pfc-stats/DirStat.json";
498 if (do_purge_check || do_purge_report || do_purge_cold_files)
502 next_purge_check_time = now + s_purge_check_interval;
503 if (do_purge_report) next_purge_report_time = now + s_purge_report_interval;
504 if (do_purge_cold_files) next_purge_cold_files_time = now + s_purge_cold_files_interval;
516 std::vector<DirStateElement> &vec,
519 const int pos0 = (int) vec.size();
520 const int n_children = (int) parent_ds.
m_subdirs.size();
523 if (parent_ds.
m_depth >= max_depth)
526 if (n_children == 0) {
539 if (n_children == 0)
return;
556 std::vector<DirPurgeElement> &vec,
559 const int pos0 = (int) vec.size();
560 const int n_children = (int) parent_ds.
m_subdirs.size();
563 if (parent_ds.
m_depth >= max_depth)
566 if (n_children == 0) {
579 if (n_children == 0)
return;
600 static const char *trc_pfx =
"update_vs_and_file_usage_info() ";
606 if (m_oss.
StatVS(&vsi, conf.m_data_space.c_str(), 1) < 0) {
607 TRACE(
Error, trc_pfx <<
"can't get StatVS for oss space '" << conf.m_data_space <<
"'. This is a fatal error.");
612 m_fs_state.
m_file_usage = 512ll * m_current_usage_in_st_blocks;
613 if (m_oss.
StatVS(&vsi, conf.m_meta_space.c_str(), 1) < 0) {
614 TRACE(
Error, trc_pfx <<
"can't get StatVS for oss space '" << conf.m_meta_space <<
"'. This is a fatal error.");
621 long long ResourceMonitor::get_file_usage_bytes_to_remove(
const DataFsPurgeshot &ps,
long long write_estimate,
int tl)
637 long long delta = write_estimate;
638 TRACE_INT(tl,
"file usage increased since the previous purge interval in bytes: " << delta );
640 long long bytes_to_remove = 0;
643 auto clamp = [&x, &bytes_to_remove](
long long lowval,
long long highval)
646 long long newval = val - bytes_to_remove;
655 if (newval > highval)
657 return val - highval;
660 return bytes_to_remove;
671 float frac_u =
static_cast<float>(u - w2) / (T - w2);
672 float frac_x =
static_cast<float>(x - f0) / (f1 - f0);
676 bytes_to_remove = u -w1;
683 bytes_to_remove = (frac_x - frac_u) * (f1 - f0);
684 bytes_to_remove += delta;
685 bytes_to_remove = clamp(f0, f1);
690 bytes_to_remove = clamp(f0, f2);
692 return bytes_to_remove;
698 if (u > w1 && x > f1)
700 float frac_u =
static_cast<float>(u - w1) / (w2 - w1);
701 float frac_x =
static_cast<float>(x - f1) / (f2 - f1);
704 TRACE_INT(tl,
"Disproportional file quota usage comapared to disc usage (frac_x/frac_u) = " << frac_x <<
"/"<< frac_u);
705 bytes_to_remove = (frac_x - frac_u) * (f2 - f1);
706 bytes_to_remove += delta;
710 bytes_to_remove = clamp(f0, f2);
711 return bytes_to_remove;
721 TRACE_INT(tl,
"File usage exceeds maxim file usage. Total disk usage is under lowWatermark. Clearing to low file usage.");
722 long long f2delta = std::max(f2 - delta, f0);
723 bytes_to_remove = clamp(f0, f2delta);
724 return bytes_to_remove;
727 return bytes_to_remove;
732 static const char *trc_pfx =
"perform_purge_check() ";
735 std::unique_ptr<DataFsPurgeshot> psp(
new DataFsPurgeshot(m_fs_state) );
743 TRACE_INT(tl, trc_pfx <<
"Purge check:");
777 TRACE(
Info, trc_pfx <<
"purge not required.");
782 TRACE(
Warning, trc_pfx <<
"purge required but previous purge task is still active!");
786 TRACE(
Info, trc_pfx <<
"scheduling purge task.");
806 dprintf(
"purge dir count recursive=%d vs from_usage=%d\n", n_pshot_dirs, n_calc_dirs);
807 const int discrepancy = n_calc_dirs - n_pshot_dirs;
808 if (discrepancy != m_dir_count_discrepancy)
810 if (discrepancy != 0) {
811 TRACE(
Error, trc_pfx <<
"directory count mismatch: usage counters " << n_calc_dirs
812 <<
", tree " << n_pshot_dirs <<
", previous difference " << m_dir_count_discrepancy);
814 TRACE(
Info, trc_pfx <<
"directory count consistent again: " << n_pshot_dirs);
816 m_dir_count_discrepancy = discrepancy;
825 struct PurgeDriverJob :
public XrdJob
830 XrdJob(
"XrdPfc::ResourceMonitor::PurgeDriver"),
831 m_purge_shot_ptr(psp)
839 delete m_purge_shot_ptr;
893 m_dir_scan_in_progress =
true;
898 const char *tpfx =
"main_thread_function ";
900 time_t is_start = time(0);
902 TRACE(
Info, tpfx <<
"Stating initial directory scan.");
905 TRACE(
Error, tpfx <<
"Initial directory scan has failed. This is a terminal error, aborting.")
910 time_t is_duration = time(0) - is_start;
911 TRACE(
Info, tpfx <<
"Initial directory scan complete, duration=" << is_duration <<
"s");
915 TRACE(
Info, tpfx <<
"First process_queues finished, n_records=" << n_proc_is);
918 if (is_duration > 30 || n_proc_is > 3000)
920 m_file_open_q.shrink_read_queue();
921 m_file_update_stats_q.shrink_read_queue();
922 m_file_close_q.shrink_read_queue();
923 m_file_purge_q1.shrink_read_queue();
924 m_file_purge_q2.shrink_read_queue();
925 m_file_purge_q3.shrink_read_queue();
971 time_t heartbeat_start = time(0);
1015 int heartbeat_duration = time(0) - heartbeat_start;
1020 int sleep_time = 60 - heartbeat_duration;
int DoIt(int argpnt, int argc, char **argv, bool singleshot)
void Proto_ResourceMonitorHeartBeat()
#define TRACE_INT(act, x)
virtual int Opendir(const char *path, XrdOucEnv &env)
virtual XrdOssDF * newDir(const char *tident)=0
virtual int StatVS(XrdOssVSInfo *vsP, const char *sname=0, int updt=0)
virtual int Unlink(const char *path, int Opts=0, XrdOucEnv *envP=0)=0
struct XrdOucCacheStats::CacheStats X
XrdOucCacheStats Statistics
PurgePin * GetPurgePin() const
static const Configuration & Conf()
static ResourceMonitor & ResMon()
void ClearPurgeProtectedSet()
static Cache & GetInstance()
Singleton access.
XrdSysTrace * GetTrace() const
static XrdScheduler * schedP
long long WritesSinceLastCall()
void AddUp(const DirStats &s)
long long m_StBlocksRemoved
std::vector< std::string > m_current_dirs
std::string m_current_path
std::function< void()> m_readdir_hook
XrdOucEnv & default_env()
void slurp_dir_ll(XrdOssDF &dh, int dir_level, const char *path, const char *trc_pfx)
bool begin_traversal(DirState *root, const char *root_path)
std::set< std::string > m_protected_top_dirs
bool cd_down(const std::string &dir_name)
std::map< std::string, FilePairStat > m_current_files
Status of cached file. Can be read from and written into a binary file.
virtual bool CallPeriodically()
bool m_purge_task_complete
void CrossCheckIfScanIsInProgress(const std::string &lfn, XrdSysCondVar &cond)
void perform_purge_check(bool purge_cold_files, int tl)
void fill_sshot_vec_children(const DirState &parent_ds, int parent_idx, std::vector< DirStateElement > &vec, int max_depth)
ResourceMonitor(XrdOss &oss)
void perform_purge_task(DataFsPurgeshot &ps)
void scan_dir_and_recurse(FsTraversal &fst)
void fill_pshot_vec_children(const DirState &parent_ds, int parent_idx, std::vector< DirPurgeElement > &vec, int max_depth)
XrdSysCondVar m_purge_task_cond
void perform_purge_task_cleanup()
void update_vs_and_file_usage_info()
bool perform_initial_scan()
AccessToken & token(int i)
time_t m_purge_task_start
void main_thread_function()
void release(int i)
Clear the slot and return it for reuse. Caller must hold its mutex.
void Schedule(XrdJob *jp)
void OldStylePurgeDriver(DataFsPurgeshot &ps)
void note_file_open(DirState *node, DirState *last_existing, bool existing_file)
std::function< int(const std::string &)> unlink_func
Contains parameters configurable from the xrootd config file.
long long m_RamAbsAvailable
available from configuration
long long m_diskTotalSpace
total disk space on configured partition or oss space
long long m_fileUsageMax
cache purge - files usage maximum
long long m_fileUsageBaseline
cache purge - files usage baseline
int m_dirStatsStoreDepth
maximum depth for statistics write out
long long m_diskUsageHWM
cache purge - disk usage high water mark
bool are_file_usage_limits_set() const
long long m_fileUsageNominal
cache purge - files usage nominal
int m_purgeAgeBasedPeriod
peform cold file / uvkeep purge every this many purge cycles
long long m_diskUsageLWM
cache purge - disk usage low water mark
int m_dirStatsInterval
time between resource monitor statistics dump in seconds
bool is_age_based_purge_in_effect() const
int m_purgeInterval
sleep interval between cache purges
bool is_dir_stat_reporting_on() const
std::vector< DirPurgeElement > m_dir_vec
long long m_bytes_to_remove
long long m_estimated_writes_from_writeq
void write_json_file(const std::string &fname, XrdOss &oss, bool include_preamble)
std::vector< DirStateElement > m_dir_states
void reset_stats(time_t last_update)
void dump_recursively(int max_depth) const
void init_stat_reset_times(time_t t)
time_t m_sshot_stats_reset_time
void update_stats_and_usages(time_t last_update, bool purge_empty_dirs, unlink_func unlink_foo)
DirState * find_dirstate_for_lfn(const std::string &lfn, DirState **last_existing_dir=nullptr)
void reset_sshot_stats(time_t last_update)
DirUsage m_recursive_subdir_usage
int count_dirs_to_level(int max_depth) const
DirState * find_path(const std::string &path, int max_depth, bool parse_as_lfn, bool create_subdirs, DirState **last_existing_dir=nullptr)
void upward_propagate_initial_scan_usages()