XRootD
XrdPfc::ResourceMonitor Class Reference

#include <XrdPfcResourceMonitor.hh>

+ Collaboration diagram for XrdPfc::ResourceMonitor:

Public Member Functions

 ResourceMonitor (XrdOss &oss)
 
 ~ResourceMonitor ()
 
void CrossCheckIfScanIsInProgress (const std::string &lfn, XrdSysCondVar &cond)
 
void fill_pshot_vec_children (const DirState &parent_ds, int parent_idx, std::vector< DirPurgeElement > &vec, int max_depth)
 
void fill_sshot_vec_children (const DirState &parent_ds, int parent_idx, std::vector< DirStateElement > &vec, int max_depth)
 
void heart_beat ()
 
void init_before_main ()
 
void main_thread_function ()
 
bool perform_initial_scan ()
 
void perform_purge_check (bool purge_cold_files, int tl)
 
void perform_purge_task (DataFsPurgeshot &ps)
 
void perform_purge_task_cleanup ()
 
int process_queues ()
 
void register_file_close (int token_id, time_t close_timestamp, const Stats &full_stats)
 
int register_file_open (const std::string &filename, time_t open_timestamp, bool existing_file)
 
void register_file_purge (const std::string &filename, long long size_in_st_blocks)
 
void register_file_purge (DirState *target, long long size_in_st_blocks)
 
void register_file_update_stats (int token_id, const Stats &stats)
 
void register_multi_file_purge (const std::string &target, long long size_in_st_blocks, int n_files)
 
void register_multi_file_purge (DirState *target, long long size_in_st_blocks, int n_files)
 
void scan_dir_and_recurse (FsTraversal &fst)
 
AccessToken & token (int i)
 
void update_vs_and_file_usage_info ()
 

Public Attributes

bool m_purge_task_active {false}
 
bool m_purge_task_complete {false}
 
XrdSysCondVar m_purge_task_cond {0}
 
time_t m_purge_task_end {0}
 
time_t m_purge_task_start {0}
 

Detailed Description

Definition at line 39 of file XrdPfcResourceMonitor.hh.

Constructor & Destructor Documentation

◆ ResourceMonitor()

ResourceMonitor::ResourceMonitor ( XrdOss &  oss)

Definition at line 32 of file XrdPfcResourceMonitor.cc.

32  :
33  m_fs_state(* new DataFsState),
34  m_oss(oss)
35 {}

◆ ~ResourceMonitor()

ResourceMonitor::~ResourceMonitor ( )

Definition at line 37 of file XrdPfcResourceMonitor.cc.

38 {
39  delete &m_fs_state;
40 }

Member Function Documentation

◆ CrossCheckIfScanIsInProgress()

void ResourceMonitor::CrossCheckIfScanIsInProgress ( const std::string &  lfn,
XrdSysCondVar &  cond 
)

Definition at line 46 of file XrdPfcResourceMonitor.cc.

47 {
48  m_dir_scan_mutex.Lock();
49  if (m_dir_scan_in_progress) {
50  bool dir_checked = false;
51  m_dir_scan_open_requests.push_back({lfn, cond, dir_checked});
52  cond.Lock();
53  m_dir_scan_mutex.UnLock();
54  while ( ! dir_checked)
55  cond.Wait();
56  cond.UnLock();
57  } else {
58  m_dir_scan_mutex.UnLock();
59  }
60 }

References XrdSysCondVar::Lock(), XrdSysMutex::Lock(), XrdSysCondVar::UnLock(), XrdSysMutex::UnLock(), and XrdSysCondVar::Wait().

+ Here is the call graph for this function:

◆ fill_pshot_vec_children()

void ResourceMonitor::fill_pshot_vec_children ( const DirState &  parent_ds,
int  parent_idx,
std::vector< DirPurgeElement > &  vec,
int  max_depth 
)

Definition at line 554 of file XrdPfcResourceMonitor.cc.

558 {
559  const int pos0 = (int) vec.size();
560  const int n_children = (int) parent_ds.m_subdirs.size();
561 
562  // maxdepth 0 is the root alone: a node at max_depth is emitted, its children are not.
563  if (parent_ds.m_depth >= max_depth)
564  {
565  // Truncated here: keep the daughter range at (-1, -1). No children: empty range.
566  if (n_children == 0) {
567  DirPurgeElement &dse = vec[parent_idx];
568  dse.m_daughters_begin = dse.m_daughters_end = pos0;
569  }
570  return;
571  }
572 
573  { // scoped: the emplace_back loop below can reallocate vec
574  DirPurgeElement &parent_dse = vec[parent_idx];
575  parent_dse.m_daughters_begin = pos0;
576  parent_dse.m_daughters_end = pos0 + n_children;
577  }
578 
579  if (n_children == 0) return;
580 
581  for (auto const & [name, child] : parent_ds.m_subdirs)
582  {
583  vec.emplace_back( DirPurgeElement(child, child.m_here_usage, child.m_recursive_subdir_usage, parent_idx) );
584  }
585 
586  int pos = pos0;
587  for (auto const & [name, child] : parent_ds.m_subdirs)
588  {
589  fill_pshot_vec_children(child, pos, vec, max_depth);
590  ++pos;
591  }
592 }
static void child()
void fill_pshot_vec_children(const DirState &parent_ds, int parent_idx, std::vector< DirPurgeElement > &vec, int max_depth)

References child(), XrdPfc::DirPurgeElement::m_daughters_begin, XrdPfc::DirPurgeElement::m_daughters_end, XrdPfc::DirState::m_depth, and XrdPfc::DirState::m_subdirs.

Referenced by perform_purge_check().

+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ fill_sshot_vec_children()

void ResourceMonitor::fill_sshot_vec_children ( const DirState &  parent_ds,
int  parent_idx,
std::vector< DirStateElement > &  vec,
int  max_depth 
)

Definition at line 514 of file XrdPfcResourceMonitor.cc.

518 {
519  const int pos0 = (int) vec.size();
520  const int n_children = (int) parent_ds.m_subdirs.size();
521 
522  // maxdepth 0 is the root alone: a node at max_depth is emitted, its children are not.
523  if (parent_ds.m_depth >= max_depth)
524  {
525  // Truncated here: keep the daughter range at (-1, -1). No children: empty range.
526  if (n_children == 0) {
527  DirStateElement &dse = vec[parent_idx];
528  dse.m_daughters_begin = dse.m_daughters_end = pos0;
529  }
530  return;
531  }
532 
533  { // scoped: the emplace_back loop below can reallocate vec
534  DirStateElement &parent_dse = vec[parent_idx];
535  parent_dse.m_daughters_begin = pos0;
536  parent_dse.m_daughters_end = pos0 + n_children;
537  }
538 
539  if (n_children == 0) return;
540 
541  for (auto const & [name, child] : parent_ds.m_subdirs)
542  {
543  vec.emplace_back( DirStateElement(child, parent_idx) );
544  }
545 
546  int pos = pos0;
547  for (auto const & [name, child] : parent_ds.m_subdirs)
548  {
549  fill_sshot_vec_children(child, pos, vec, max_depth);
550  ++pos;
551  }
552 }
void fill_sshot_vec_children(const DirState &parent_ds, int parent_idx, std::vector< DirStateElement > &vec, int max_depth)

References child(), XrdPfc::DirStateElement::m_daughters_begin, XrdPfc::DirStateElement::m_daughters_end, XrdPfc::DirState::m_depth, and XrdPfc::DirState::m_subdirs.

Referenced by heart_beat().

+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ heart_beat()

void ResourceMonitor::heart_beat ( )

Definition at line 362 of file XrdPfcResourceMonitor.cc.

363 {
364  static const char *tpfx = "heart_beat() ";
365 
366  const Configuration &conf = Cache::Conf();
367 
368  const int s_queue_proc_interval = 10;
369  const int s_sshot_report_interval = conf.m_dirStatsInterval; // one of 60, 120, 300,
370  // 600, 900, 1200, 1800, 3600 s
371  const int s_purge_check_interval = 60;
372  const int s_purge_report_interval = conf.m_purgeInterval;
373  const int s_purge_cold_files_interval = conf.m_purgeInterval * conf.m_purgeAgeBasedPeriod;
374 
375  // initial scan performed as part of config
376 
377  time_t now = time(0);
378  time_t next_queue_proc_time = now + s_queue_proc_interval;
379  time_t next_sshot_report_time = (now / 60) * 60 + 60; // at next full minute
380  time_t next_purge_check_time = now + s_purge_check_interval;
381  time_t next_purge_report_time = now + s_purge_report_interval;
382  time_t next_purge_cold_files_time = now + s_purge_cold_files_interval;
383 
384  long long n_reported_token_failures = 0;
385 
386  while (true)
387  {
388  time_t start = time(0);
389  time_t next_event = std::min({ next_queue_proc_time, next_sshot_report_time,
390  next_purge_check_time, next_purge_report_time, next_purge_cold_files_time });
391 
392  if (next_event > start)
393  {
394  unsigned int t_sleep = next_event - start;
395  TRACE(Dump, tpfx << "sleeping for " << t_sleep << " seconds until the next beat.");
396  sleep(t_sleep);
397  }
398 
399  // Check if purge has been running and has completed yet.
400  // For now this is only used to prevent removal of empty leaf directories
401  // during stat propagation so we do not need to wait for the condition in
402  // the above sleep.
403  if (m_purge_task_active) {
405  if (m_purge_task_complete) {
407  }
408  }
409 
410  time_t queue_swap_time = time(0);
411 
412  // Always process the queues.
413  int n_processed = process_queues();
414  next_queue_proc_time = queue_swap_time + s_queue_proc_interval;
415  TRACE(Dump, tpfx << "process_queues -- n_records=" << n_processed);
416 
417  { // Access-token space exhausted -- should be unreachable, so say so loudly.
418  long long taf;
419  { XrdSysMutexHelper _lock(&m_queue_mutex); taf = m_token_alloc_failures; }
420  if (taf > n_reported_token_failures) {
421  TRACE(Error, tpfx << "access-token space exhausted "
422  << taf - n_reported_token_failures << " time(s) (total " << taf
423  << "); those files are open but not accounted for. Raise MaxChunks "
424  "on ResourceMonitor::m_access_tokens.");
425  n_reported_token_failures = taf;
426  }
427  }
428 
429  // Always update basic info on m_fs_state (space, usage, file_usage).
431 
432  now = time(0);
433 
434  // Make planning for fs_state_update, sshot dump and purge task.
435  // Second two require the first, so figure out what is going to happen.
436  bool do_sshot_report = next_sshot_report_time <= now;
437  bool do_purge_check = next_purge_check_time <= now;
438  bool do_purge_report = next_purge_report_time <= now;
439  bool do_purge_cold_files = next_purge_cold_files_time <= now;
440 
441  // Update stats in usages if any secondary activity will happen.
442  if (do_sshot_report || do_purge_check || do_purge_report || do_purge_cold_files)
443  {
444  unlink_func unlink_foo = [&](const std::string &dp)->int {
445  int ret = m_oss.Unlink(dp.c_str());
446  if (ret != 0) {
447  TRACE(Info, tpfx << "Empty dir unlink error: " << ret << " at " << dp);
448  } else {
449  TRACE(Debug, tpfx << "Empty dir unlink success: " << dp);
450  }
451  return ret;
452  };
453 
454  // Potentially prune the empty leaf dirs even less frequently, once per hour, maybe?
455  bool purge_leaf_dirs = do_sshot_report && ! m_purge_task_active;
456  m_fs_state.update_stats_and_usages(queue_swap_time, purge_leaf_dirs, unlink_foo);
457 
458  // This reporting into log/stdout is to be removed.
459  // Meaning of conf.is_dir_stat_reporting_on() etc is to be clarified / improved.
460  if (do_sshot_report && conf.is_dir_stat_reporting_on())
461  {
462  const int store_depth = conf.m_dirStatsStoreDepth;
463  #ifdef RM_DEBUG
464  const DirState &root_ds = *m_fs_state.get_root();
465  dprintf("Snapshot n_dirs=%d, total n_dirs=%d\n", root_ds.count_dirs_to_level(store_depth),
467  #endif
468  m_fs_state.dump_recursively(store_depth);
469  }
470 
471  m_fs_state.reset_stats(queue_swap_time);
472  }
473 
474  if (do_sshot_report)
475  {
476  // Sshot reports are equidistant, at "full" reporting interval.
477  next_sshot_report_time = ((now + 1) / s_sshot_report_interval) * s_sshot_report_interval + s_sshot_report_interval;
478 
479  // This should dump out binary snapshot into /pfc-stats/, if so configured.
480 
481  // json dump to std::out for debug purpose
482  DataFsSnapshot ss(m_fs_state, m_fs_state.m_sshot_stats_reset_time);
483  const DirState &root_ds = *m_fs_state.get_root();
484  const int store_depth = conf.m_dirStatsStoreDepth;
485  const int n_sshot_dirs = root_ds.count_dirs_to_level(store_depth);
486  ss.m_dir_states.reserve(n_sshot_dirs);
487  ss.m_dir_states.emplace_back( DirStateElement(root_ds, -1) );
488  fill_sshot_vec_children(root_ds, 0, ss.m_dir_states, store_depth);
489 
490  // This should really be export to a file (preferably binary, but then bin->json command is needed, too).
491  // ss.dump();
492 
493  const char* dumpfile = "/pfc-stats/DirStat.json";
494  ss.write_json_file(dumpfile, m_oss, false);
495  m_fs_state.reset_sshot_stats(queue_swap_time);
496  }
497 
498  if (do_purge_check || do_purge_report || do_purge_cold_files)
499  {
500  perform_purge_check(do_purge_cold_files, do_purge_report ? TRACE_Info : TRACE_Debug);
501 
502  next_purge_check_time = now + s_purge_check_interval;
503  if (do_purge_report) next_purge_report_time = now + s_purge_report_interval;
504  if (do_purge_cold_files) next_purge_cold_files_time = now + s_purge_cold_files_interval;
505  }
506 
507  } // end while forever
508 }
#define TRACE_Debug
Definition: XrdCmsTrace.hh:37
#define TRACE_Info
#define dprintf(...)
bool Debug
@ Error
#define TRACE(act, x)
Definition: XrdTrace.hh:63
virtual int Unlink(const char *path, int Opts=0, XrdOucEnv *envP=0)=0
static const Configuration & Conf()
Definition: XrdPfc.cc:134
Status of cached file. Can be read from and written into a binary file.
Definition: XrdPfcInfo.hh:41
void perform_purge_check(bool purge_cold_files, int tl)
std::function< int(const std::string &)> unlink_func
Contains parameters configurable from the xrootd config file.
Definition: XrdPfc.hh:64
int m_dirStatsStoreDepth
maximum depth for statistics write out
Definition: XrdPfc.hh:106
int m_purgeAgeBasedPeriod
peform cold file / uvkeep purge every this many purge cycles
Definition: XrdPfc.hh:100
int m_dirStatsInterval
time between resource monitor statistics dump in seconds
Definition: XrdPfc.hh:105
int m_purgeInterval
sleep interval between cache purges
Definition: XrdPfc.hh:98
bool is_dir_stat_reporting_on() const
Definition: XrdPfc.hh:70
void reset_stats(time_t last_update)
void dump_recursively(int max_depth) const
void update_stats_and_usages(time_t last_update, bool purge_empty_dirs, unlink_func unlink_foo)
void reset_sshot_stats(time_t last_update)
DirUsage m_recursive_subdir_usage
int count_dirs_to_level(int max_depth) const

References XrdPfc::Cache::Conf(), XrdPfc::DirState::count_dirs_to_level(), Debug, dprintf, XrdPfc::DataFsState::dump_recursively(), Error, fill_sshot_vec_children(), XrdPfc::DataFsState::get_root(), XrdPfc::Configuration::is_dir_stat_reporting_on(), XrdPfc::DataFsSnapshot::m_dir_states, XrdPfc::Configuration::m_dirStatsInterval, XrdPfc::Configuration::m_dirStatsStoreDepth, XrdPfc::DirState::m_here_usage, XrdPfc::DirUsage::m_NDirectories, m_purge_task_active, m_purge_task_complete, m_purge_task_cond, XrdPfc::Configuration::m_purgeAgeBasedPeriod, XrdPfc::Configuration::m_purgeInterval, XrdPfc::DirState::m_recursive_subdir_usage, XrdPfc::DataFsState::m_sshot_stats_reset_time, perform_purge_check(), process_queues(), XrdPfc::DataFsState::reset_sshot_stats(), XrdPfc::DataFsState::reset_stats(), TRACE, TRACE_Debug, TRACE_Info, XrdOss::Unlink(), XrdPfc::DataFsState::update_stats_and_usages(), update_vs_and_file_usage_info(), and XrdPfc::DataFsSnapshot::write_json_file().

Referenced by main_thread_function().

+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ init_before_main()

void ResourceMonitor::init_before_main ( )

Definition at line 889 of file XrdPfcResourceMonitor.cc.

890 {
891  // setup for in-scan -- this is called from initial setup.
892  MutexHolder _lck(m_dir_scan_mutex);
893  m_dir_scan_in_progress = true;
894 }

Referenced by XrdPfc::Cache::Config().

+ Here is the caller graph for this function:

◆ main_thread_function()

void ResourceMonitor::main_thread_function ( )

Definition at line 896 of file XrdPfcResourceMonitor.cc.

897 {
898  const char *tpfx = "main_thread_function ";
899  {
900  time_t is_start = time(0);
901  m_fs_state.init_stat_reset_times(is_start);
902  TRACE(Info, tpfx << "Stating initial directory scan.");
903 
904  if ( ! perform_initial_scan()) {
905  TRACE(Error, tpfx << "Initial directory scan has failed. This is a terminal error, aborting.")
906  _exit(1);
907  }
908  // Reset of m_dir_scan_in_progress is done in perform_initial_scan()
909 
910  time_t is_duration = time(0) - is_start;
911  TRACE(Info, tpfx << "Initial directory scan complete, duration=" << is_duration <<"s");
912 
913  // run first process queues
914  int n_proc_is = process_queues();
915  TRACE(Info, tpfx << "First process_queues finished, n_records=" << n_proc_is);
916 
917  // shrink queues if scan time was longer than 30s.
918  if (is_duration > 30 || n_proc_is > 3000)
919  {
920  m_file_open_q.shrink_read_queue();
921  m_file_update_stats_q.shrink_read_queue();
922  m_file_close_q.shrink_read_queue();
923  m_file_purge_q1.shrink_read_queue();
924  m_file_purge_q2.shrink_read_queue();
925  m_file_purge_q3.shrink_read_queue();
926  }
927  }
928  heart_beat();
929 }
if(Avsz)
void init_stat_reset_times(time_t t)

References Error, heart_beat(), XrdPfc::DataFsState::init_stat_reset_times(), perform_initial_scan(), process_queues(), and TRACE.

Referenced by ResourceMonitorThread().

+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ perform_initial_scan()

bool ResourceMonitor::perform_initial_scan ( )

Definition at line 194 of file XrdPfcResourceMonitor.cc.

195 {
196  // Called after PFC configuration is complete, but before full startup of the daemon.
197  // Base line usages are accumulated as part of the file-system, traversal.
198 
200 
201  DirState *root_ds = m_fs_state.get_root();
202  FsTraversal fst(m_oss);
203  fst.m_protected_top_dirs.insert("pfc-stats"); // XXXX This should come from config. Also: N2N?
204  // Keep servicing file-open requests while reading huge directories.
205  fst.m_readdir_hook = [this]() { check_inter_dir_scan_open_requests(); };
206 
207  if ( ! fst.begin_traversal(root_ds, "/"))
208  return false;
209 
210  // m_dir_scan_in_progress is initialized to true in ResourceMonitor.hh to avoid a race at startup.
211 
213 
214  fst.end_traversal();
215 
216  // We have all directories scanned, available in DirState tree, let all remaining files go
217  // and then we shall do the upward propagation of usages.
218  {
219  XrdSysMutexHelper _lock(m_dir_scan_mutex);
220  m_dir_scan_in_progress = false;
221  }
222  // m_dir_scan_open_requests should now be final, ie, no new entries will be added.
223  while ( ! m_dir_scan_open_requests.empty())
224  {
225  LfnCondRecord &lcr = m_dir_scan_open_requests.front();
226  lcr.f_cond.Lock();
227  lcr.f_checked = true;
228  lcr.f_cond.Signal();
229  lcr.f_cond.UnLock();
230 
231  m_dir_scan_open_requests.pop_front();
232  }
233 
234  // Do upward propagation of usages.
236  m_current_usage_in_st_blocks = root_ds->m_here_usage.m_StBlocks +
239 
240  return true;
241 }
void scan_dir_and_recurse(FsTraversal &fst)
void upward_propagate_initial_scan_usages()

References XrdPfc::FsTraversal::begin_traversal(), XrdPfc::FsTraversal::end_traversal(), XrdPfc::DataFsState::get_root(), XrdPfc::DirState::m_here_usage, XrdPfc::FsTraversal::m_protected_top_dirs, XrdPfc::FsTraversal::m_readdir_hook, XrdPfc::DirState::m_recursive_subdir_usage, XrdPfc::DirUsage::m_StBlocks, scan_dir_and_recurse(), update_vs_and_file_usage_info(), and XrdPfc::DirState::upward_propagate_initial_scan_usages().

Referenced by main_thread_function().

+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ perform_purge_check()

void ResourceMonitor::perform_purge_check ( bool  purge_cold_files,
int  tl 
)

Definition at line 730 of file XrdPfcResourceMonitor.cc.

731 {
732  static const char *trc_pfx = "perform_purge_check() ";
733  const Configuration &conf = Cache::Conf();
734 
735  std::unique_ptr<DataFsPurgeshot> psp( new DataFsPurgeshot(m_fs_state) );
736  DataFsPurgeshot &ps = *psp;
737 
738  ps.m_file_usage = 512ll * m_current_usage_in_st_blocks;
739  // These are potentially wrong as cache might be writing over preallocated byte ranges.
741  // Can have another estimate based on eiter writes or st-blocks from purge-stats, once we have them.
742 
743  TRACE_INT(tl, trc_pfx << "Purge check:");
744 
745  ps.m_bytes_to_remove = 0;
746  if (conf.are_file_usage_limits_set())
747  {
748  ps.m_bytes_to_remove = get_file_usage_bytes_to_remove(ps, ps.m_estimated_writes_from_writeq, tl);
749  }
750  else
751  {
752  if (ps.m_disk_used > conf.m_diskUsageHWM)
753  {
754  TRACE_INT(tl, "Disk usage: " << ps.m_disk_used << " exceed highWatermark.");
756  }
757  }
758 
759  ps.m_space_based_purge = ps.m_bytes_to_remove ? 1 : 0;
760 
761  // Purge precheck -- check if age-based purge is required
762  // We ignore uvkeep time, it requires reading of cinfo files and it is enforced in File::Open() anyway.
763 
764  if (purge_cold_files && conf.is_age_based_purge_in_effect()) // || conf.is_uvkeep_purge_in_effect())
765  {
766  ps.m_age_based_purge = true;
767  }
768 
769  TRACE_INT(tl, "\tbytes_to_remove = " << ps.m_bytes_to_remove << " B");
770  TRACE_INT(tl, "\tspace_based_purge = " << ps.m_space_based_purge);
771  TRACE_INT(tl, "\tage_based_purge = " << ps.m_age_based_purge);
772 
773  bool periodic = Cache::GetInstance().GetPurgePin() ?
775 
776  if ( ! ps.m_space_based_purge && ! ps.m_age_based_purge && !periodic ) {
777  TRACE(Info, trc_pfx << "purge not required.");
779  return;
780  }
781  if (m_purge_task_active) {
782  TRACE(Warning, trc_pfx << "purge required but previous purge task is still active!");
783  return;
784  }
785 
786  TRACE(Info, trc_pfx << "scheduling purge task.");
787 
788  // At this point we have all the information: report, decide on action.
789  // There is still some missing infrastructure, especially as regards to purge-plugin:
790  // - at what point do we start bugging the pu-pin to start coughing up purge lists?
791  // - have a new parameter or just do it "one cycle before full"?
792  // - what if it doesn't -- when do we do the old-stlye scan & purge?
793  // - how do we do age-based purge and uvkeep purge?
794  // - they are really quite different -- and could run separately, registering
795  // files into a purge-candidate list. This has to be rechecked before the actual
796  // deletion -- eg, by comparing stat time of cinfo + doing the is-active / is-purge-protected.
797 
798  const DirState &root_ds = *m_fs_state.get_root();
799 
800  // Size from the tree itself; the usage counters can be off (#2808).
801  const int n_pshot_dirs = root_ds.count_dirs_to_level(9999);
802 
803  // Report a counters-vs-tree mismatch when it appears or changes.
804  const int n_calc_dirs = 1 + root_ds.m_here_usage.m_NDirectories
806  dprintf("purge dir count recursive=%d vs from_usage=%d\n", n_pshot_dirs, n_calc_dirs);
807  const int discrepancy = n_calc_dirs - n_pshot_dirs;
808  if (discrepancy != m_dir_count_discrepancy)
809  {
810  if (discrepancy != 0) {
811  TRACE(Error, trc_pfx << "directory count mismatch: usage counters " << n_calc_dirs
812  << ", tree " << n_pshot_dirs << ", previous difference " << m_dir_count_discrepancy);
813  } else {
814  TRACE(Info, trc_pfx << "directory count consistent again: " << n_pshot_dirs);
815  }
816  m_dir_count_discrepancy = discrepancy;
817  }
818 
819  ps.m_dir_vec.reserve(n_pshot_dirs);
820  ps.m_dir_vec.emplace_back( DirPurgeElement(root_ds, root_ds.m_here_usage, root_ds.m_recursive_subdir_usage, -1) );
821  fill_pshot_vec_children(root_ds, 0, ps.m_dir_vec, 9999);
822 
823  m_purge_task_active = true;
824 
825  struct PurgeDriverJob : public XrdJob
826  {
827  DataFsPurgeshot *m_purge_shot_ptr;
828 
829  PurgeDriverJob(DataFsPurgeshot *psp) :
830  XrdJob("XrdPfc::ResourceMonitor::PurgeDriver"),
831  m_purge_shot_ptr(psp)
832  {}
833 
834  void DoIt() override
835  {
836  Cache::ResMon().perform_purge_task(*m_purge_shot_ptr);
838 
839  delete m_purge_shot_ptr;
840  delete this;
841  }
842  };
843 
844  Cache::schedP->Schedule( new PurgeDriverJob(psp.release()) );
845 }
int DoIt(int argpnt, int argc, char **argv, bool singleshot)
Definition: XrdAccTest.cc:262
#define TRACE_INT(act, x)
Definition: XrdPfcTrace.hh:52
Definition: XrdJob.hh:43
PurgePin * GetPurgePin() const
Definition: XrdPfc.hh:286
static ResourceMonitor & ResMon()
Definition: XrdPfc.cc:135
void ClearPurgeProtectedSet()
Definition: XrdPfc.cc:691
static Cache & GetInstance()
Singleton access.
Definition: XrdPfc.cc:132
static XrdScheduler * schedP
Definition: XrdPfc.hh:304
long long WritesSinceLastCall()
Definition: XrdPfc.cc:324
virtual bool CallPeriodically()
void perform_purge_task(DataFsPurgeshot &ps)
void Schedule(XrdJob *jp)
@ Warning
long long m_diskUsageHWM
cache purge - disk usage high water mark
Definition: XrdPfc.hh:94
bool are_file_usage_limits_set() const
Definition: XrdPfc.hh:67
long long m_diskUsageLWM
cache purge - disk usage low water mark
Definition: XrdPfc.hh:93
bool is_age_based_purge_in_effect() const
Definition: XrdPfc.hh:68
std::vector< DirPurgeElement > m_dir_vec

References XrdPfc::Configuration::are_file_usage_limits_set(), XrdPfc::PurgePin::CallPeriodically(), XrdPfc::Cache::ClearPurgeProtectedSet(), XrdPfc::Cache::Conf(), XrdPfc::DirState::count_dirs_to_level(), DoIt(), dprintf, Error, fill_pshot_vec_children(), XrdPfc::DataFsState::get_root(), XrdPfc::Cache::GetInstance(), XrdPfc::Cache::GetPurgePin(), XrdPfc::Configuration::is_age_based_purge_in_effect(), XrdPfc::DataFsPurgeshot::m_age_based_purge, XrdPfc::DataFsPurgeshot::m_bytes_to_remove, XrdPfc::DataFsPurgeshot::m_dir_vec, XrdPfc::DataFsStateBase::m_disk_used, XrdPfc::Configuration::m_diskUsageHWM, XrdPfc::Configuration::m_diskUsageLWM, XrdPfc::DataFsPurgeshot::m_estimated_writes_from_writeq, XrdPfc::DataFsStateBase::m_file_usage, XrdPfc::DirState::m_here_usage, XrdPfc::DirUsage::m_NDirectories, m_purge_task_active, XrdPfc::DirState::m_recursive_subdir_usage, XrdPfc::DataFsPurgeshot::m_space_based_purge, perform_purge_task(), perform_purge_task_cleanup(), XrdPfc::Cache::ResMon(), XrdPfc::Cache::schedP, XrdScheduler::Schedule(), TRACE, TRACE_INT, TPC::Warning, and XrdPfc::Cache::WritesSinceLastCall().

Referenced by heart_beat().

+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ perform_purge_task()

void ResourceMonitor::perform_purge_task ( DataFsPurgeshot &  ps)

Definition at line 852 of file XrdPfcResourceMonitor.cc.

853 {
854  // BEWARE: Runs in a dedicated thread - is only to communicate back to the
855  // hear_beat() / data structs via the purge queues and condition variable.
856 
857  // const char *tpfx = "perform_purge_task ";
858 
859  {
861  m_purge_task_start = time(0);
862  }
863 
864  // For now, fall back to the old purge ... to be improved with:
865  // - new scan, following the DataFsPurgeshot;
866  // - usage of cinfo stat mtime for time of last access (touch already done at output);
867  // - use DirState* to report back purged files.
868  // Already changed to report back purged files --- but using the string / path variant.
869  OldStylePurgeDriver(ps); // In XrdPfcPurge.cc
870 }
void OldStylePurgeDriver(DataFsPurgeshot &ps)
Definition: XrdPfcPurge.cc:99

References m_purge_task_cond, m_purge_task_start, and XrdPfc::OldStylePurgeDriver().

Referenced by perform_purge_check().

+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ perform_purge_task_cleanup()

void ResourceMonitor::perform_purge_task_cleanup ( )

Definition at line 872 of file XrdPfcResourceMonitor.cc.

873 {
874  // Separated out so the purge_task can exit without post-checks.
875 
876  {
878  m_purge_task_end = time(0);
879  m_purge_task_complete = true;
881  }
883 }

References XrdPfc::Cache::ClearPurgeProtectedSet(), XrdPfc::Cache::GetInstance(), m_purge_task_complete, m_purge_task_cond, m_purge_task_end, and XrdSysCondVar::Signal().

Referenced by perform_purge_check().

+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ process_queues()

int ResourceMonitor::process_queues ( )

Definition at line 247 of file XrdPfcResourceMonitor.cc.

248 {
249  static const char *trc_pfx = "process_queues() ";
250 
251  // Assure that we pick up only entries that are present now.
252  // We really want all open records to be processed before file-stats updates
253  // and all those before the close records.
254  // Purges are sort of tangential as they really just modify bytes / number
255  // of files in a directory and do not deal with any persistent file id tokens.
256 
257  int n_records = 0;
258  {
259  XrdSysMutexHelper _lock(&m_queue_mutex);
260  n_records += m_file_open_q.swap_queues();
261  n_records += m_file_update_stats_q.swap_queues();
262  n_records += m_file_close_q.swap_queues();
263  n_records += m_file_purge_q1.swap_queues();
264  n_records += m_file_purge_q2.swap_queues();
265  n_records += m_file_purge_q3.swap_queues();
266  ++m_queue_swap_u1;
267  }
268 
269  for (auto &i : m_file_open_q.read_queue())
270  {
271  // i.id: LFN, i.record: OpenRecord
272  AccessToken &at = token(i.id);
273  dprintf("process file open for token %d, time %ld -- %s\n",
274  i.id, i.record.m_open_time, at.m_filename.c_str());
275 
276  // Resolve fname into DirState.
277  // We could clear the filename after this ... or keep it, should we need it later on.
278  // For now it is just used for debug printouts.
279  DirState *last_existing_ds = nullptr;
280  DirState *ds = m_fs_state.find_dirstate_for_lfn(at.m_filename, &last_existing_ds);
281  at.m_dir_state = ds;
282  note_file_open(ds, last_existing_ds, i.record.m_existing_file);
283 
284  ds->m_here_usage.m_LastOpenTime = i.record.m_open_time;
285  }
286 
287  for (auto &i : m_file_update_stats_q.read_queue())
288  {
289  // i.id: token, i.record: Stats
290  AccessToken &at = token(i.id);
291  // Stats
292  DirState *ds = at.m_dir_state;
293  dprintf("process file update for token %d, %p -- %s\n",
294  i.id, ds, at.m_filename.c_str());
295 
296  ds->m_here_stats.AddUp(i.record);
297  m_current_usage_in_st_blocks += i.record.m_StBlocksAdded;
298  }
299 
300  for (auto &i : m_file_close_q.read_queue())
301  {
302  // i.id: token, i.record: CloseRecord
303  AccessToken &at = token(i.id);
304  dprintf("process file close for token %d, time %ld -- %s\n",
305  i.id, i.record.m_close_time, at.m_filename.c_str());
306 
307  DirState *ds = at.m_dir_state;
308  ds->m_here_stats.m_NFilesClosed += 1;
309  ds->m_here_usage.m_LastCloseTime = i.record.m_close_time;
310 
311  }
312  { // Under the lock: clear() writes fields register_file_update_stats() uses.
313  XrdSysMutexHelper _lock(&m_queue_mutex);
314  for (auto &i : m_file_close_q.read_queue())
315  m_access_tokens.release(i.id);
316  }
317 
318  for (auto &i : m_file_purge_q1.read_queue())
319  {
320  // i.id: DirState*, i.record: PurgeRecord
321  DirState *ds = i.id;
322  ds->m_here_stats.m_StBlocksRemoved += i.record.m_size_in_st_blocks;
323  ds->m_here_stats.m_NFilesRemoved += i.record.m_n_files;
324  m_current_usage_in_st_blocks -= i.record.m_size_in_st_blocks;
325  }
326  for (auto &i : m_file_purge_q2.read_queue())
327  {
328  // i.id: directory-path, i.record: PurgeRecord
329  DirState *ds = m_fs_state.get_root()->find_path(i.id, -1, false, false);
330  if ( ! ds) {
331  TRACE(Error, trc_pfx << "DirState not found for directory path '" << i.id << "'.");
332  // find_path can return the last dir found ... but this clearly isn't a valid purge record.
333  continue;
334  }
335  ds->m_here_stats.m_StBlocksRemoved += i.record.m_size_in_st_blocks;
336  ds->m_here_stats.m_NFilesRemoved += i.record.m_n_files;
337  m_current_usage_in_st_blocks -= i.record.m_size_in_st_blocks;
338  }
339  for (auto &i : m_file_purge_q3.read_queue())
340  {
341  // i.id: LFN, i.record: size of file in st_blocks
342  DirState *ds = m_fs_state.get_root()->find_path(i.id, -1, true, false);
343  if ( ! ds) {
344  TRACE(Error, trc_pfx << "DirState not found for LFN path '" << i.id << "'.");
345  continue;
346  }
347  ds->m_here_stats.m_StBlocksRemoved += i.record;
348  ds->m_here_stats.m_NFilesRemoved += 1;
349  m_current_usage_in_st_blocks -= i.record;
350  }
351 
352  // Read queues / vectors are cleared at swap time.
353  // We might consider reducing their capacity by half if, say, their usage is below 25%.
354 
355  return n_records;
356 }
void AddUp(const DirStats &s)
Definition: XrdPfcStats.hh:192
long long m_StBlocksRemoved
Definition: XrdPfcStats.hh:149
void release(int i)
Clear the slot and return it for reuse. Caller must hold its mutex.
void note_file_open(DirState *node, DirState *last_existing, bool existing_file)
DirState * find_dirstate_for_lfn(const std::string &lfn, DirState **last_existing_dir=nullptr)
DirState * find_path(const std::string &path, int max_depth, bool parse_as_lfn, bool create_subdirs, DirState **last_existing_dir=nullptr)

References XrdPfc::DirStats::AddUp(), dprintf, Error, XrdPfc::DataFsState::find_dirstate_for_lfn(), XrdPfc::DirState::find_path(), XrdPfc::DataFsState::get_root(), XrdPfc::DirState::m_here_stats, XrdPfc::DirState::m_here_usage, XrdPfc::DirUsage::m_LastCloseTime, XrdPfc::DirUsage::m_LastOpenTime, XrdPfc::DirStats::m_NFilesClosed, XrdPfc::DirStats::m_NFilesRemoved, XrdPfc::DirStats::m_StBlocksRemoved, XrdPfc::note_file_open(), XrdPfc::StableChunkPool< T, InnerShift, MaxChunks >::release(), token(), and TRACE.

Referenced by heart_beat(), and main_thread_function().

+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ register_file_close()

void XrdPfc::ResourceMonitor::register_file_close ( int  token_id,
time_t  close_timestamp,
const Stats &  full_stats 
)
inline

Definition at line 196 of file XrdPfcResourceMonitor.hh.

196  {
197  if (token_id < 0) return; // register_file_open() could not get a token
198  XrdSysMutexHelper _lock(&m_queue_mutex);
199  m_file_close_q.push(token_id, {close_timestamp, full_stats});
200  }

Referenced by XrdPfc::Cache::ExecuteCommandUrl().

+ Here is the caller graph for this function:

◆ register_file_open()

int XrdPfc::ResourceMonitor::register_file_open ( const std::string &  filename,
time_t  open_timestamp,
bool  existing_file 
)
inline

Definition at line 160 of file XrdPfcResourceMonitor.hh.

160  {
161  // Simply return a token, we will resolve it in the actual processing of the queue.
162  XrdSysMutexHelper _lock(&m_queue_mutex);
163  int token_id = m_access_tokens.acquire();
164  if (token_id < 0) {
165  // Unreachable in practice; heart_beat() reports the count.
166  ++m_token_alloc_failures;
167  return -1;
168  }
169  AccessToken &at = m_access_tokens[token_id];
170  at.m_filename = filename;
171  at.m_last_queue_swap_u1 = m_queue_swap_u1 - 1;
172 
173  m_file_open_q.push(token_id, {open_timestamp, existing_file});
174  return token_id;
175  }

Referenced by XrdPfc::Cache::ExecuteCommandUrl().

+ Here is the caller graph for this function:

◆ register_file_purge() [1/2]

void XrdPfc::ResourceMonitor::register_file_purge ( const std::string &  filename,
long long  size_in_st_blocks 
)
inline

Definition at line 216 of file XrdPfcResourceMonitor.hh.

216  {
217  XrdSysMutexHelper _lock(&m_queue_mutex);
218  m_file_purge_q3.push(filename, size_in_st_blocks);
219  }

◆ register_file_purge() [2/2]

void XrdPfc::ResourceMonitor::register_file_purge ( DirState *  target,
long long  size_in_st_blocks 
)
inline

Definition at line 204 of file XrdPfcResourceMonitor.hh.

204  {
205  XrdSysMutexHelper _lock(&m_queue_mutex);
206  m_file_purge_q1.push(target, {size_in_st_blocks, 1});
207  }

Referenced by XrdPfc::Cache::UnlinkFile().

+ Here is the caller graph for this function:

◆ register_file_update_stats()

void XrdPfc::ResourceMonitor::register_file_update_stats ( int  token_id,
const Stats &  stats 
)
inline

Definition at line 177 of file XrdPfcResourceMonitor.hh.

177  {
178  if (token_id < 0) return; // register_file_open() could not get a token
179  XrdSysMutexHelper _lock(&m_queue_mutex);
180  AccessToken &at = token(token_id);
181  // Check if this is the first update within this queue swap cycle.
182  if (at.m_last_queue_swap_u1 != m_queue_swap_u1) {
183  m_file_update_stats_q.push(token_id, stats);
184  at.m_last_queue_swap_u1 = m_queue_swap_u1;
185  at.m_last_write_queue_pos = m_file_update_stats_q.write_queue_size() - 1;
186  } else {
187  Stats &existing_stats = m_file_update_stats_q.write_record(at.m_last_write_queue_pos);
188  existing_stats.AddUp(stats);
189  }
190  // Optionally, one could return "scaler" to moodify stat-reporting
191  // frequency in the file ... if it comes too often or too rarely.
192  // See also the logic for determining reporting interval (in N_bytes_read)
193  // in File::Open().
194  }
XrdPosixStats Stats
Definition: XrdPosixFile.cc:64

References XrdPfc::Stats::AddUp(), and token().

Referenced by XrdPfc::Cache::ExecuteCommandUrl().

+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ register_multi_file_purge() [1/2]

void XrdPfc::ResourceMonitor::register_multi_file_purge ( const std::string &  target,
long long  size_in_st_blocks,
int  n_files 
)
inline

Definition at line 212 of file XrdPfcResourceMonitor.hh.

212  {
213  XrdSysMutexHelper _lock(&m_queue_mutex);
214  m_file_purge_q2.push(target, {size_in_st_blocks, n_files});
215  }

◆ register_multi_file_purge() [2/2]

void XrdPfc::ResourceMonitor::register_multi_file_purge ( DirState *  target,
long long  size_in_st_blocks,
int  n_files 
)
inline

Definition at line 208 of file XrdPfcResourceMonitor.hh.

208  {
209  XrdSysMutexHelper _lock(&m_queue_mutex);
210  m_file_purge_q1.push(target, {size_in_st_blocks, n_files});
211  }

◆ scan_dir_and_recurse()

void ResourceMonitor::scan_dir_and_recurse ( FsTraversal &  fst)

Definition at line 145 of file XrdPfcResourceMonitor.cc.

146 {
147  dprintf("In scan_dir_and_recurse for '%s', size of dir_vec = %d, file_stat_map = %d\n",
148  fst.m_current_path.c_str(),
149  (int)fst.m_current_dirs.size(), (int)fst.m_current_files.size());
150 
151  // Breadth first, accumulate into "here", unless it was already scanned via an
152  // OOB open file request.
153  if ( ! fst.m_dir_state->m_scanned)
154  {
155  DirUsage &here = fst.m_dir_state->m_here_usage;
156  for (auto it = fst.m_current_files.begin(); it != fst.m_current_files.end(); ++it)
157  {
158  dprintf("would be doing something with %s ... has_data=%d, has_cinfo=%d\n",
159  it->first.c_str(), it->second.has_data, it->second.has_cinfo);
160 
161  // XXX Make some of these optional?
162  // Remove files that do not have both cinfo and data?
163  // Remove empty directories before even descending?
164  // Leave this for some consistency pass?
165  // Note that FsTraversal supports ignored paths ... some details (config, N2N) to be clarified.
166 
167  if (it->second.has_data && it->second.has_cinfo) {
168  here.m_StBlocks += it->second.stat_data.st_blocks;
169  here.m_NFiles += 1;
170  }
171  }
172  fst.m_dir_state->m_scanned = true;
173  }
174 
175  // Swap-out directories as recursion reuses the FsTraversal.
176  std::vector<std::string> dirs;
177  dirs.swap(fst.m_current_dirs);
178 
179  check_inter_dir_scan_open_requests();
180 
181  // Descend into sub-dirs, do not accumulate into recursive_subdir_usage yet. This is done
182  // in a separate pass to allow for proper accounting of files being opened during the initial scan.
183  for (auto &dname : dirs)
184  {
185  if (fst.cd_down(dname))
186  {
188  fst.cd_up();
189  }
190  // XXX else try to remove it?
191  }
192 }
std::vector< std::string > m_current_dirs
bool cd_down(const std::string &dir_name)
std::map< std::string, FilePairStat > m_current_files

References XrdPfc::FsTraversal::cd_down(), XrdPfc::FsTraversal::cd_up(), dprintf, XrdPfc::FsTraversal::m_current_dirs, XrdPfc::FsTraversal::m_current_files, XrdPfc::FsTraversal::m_current_path, XrdPfc::FsTraversal::m_dir_state, XrdPfc::DirState::m_here_usage, XrdPfc::DirUsage::m_NFiles, XrdPfc::DirState::m_scanned, and XrdPfc::DirUsage::m_StBlocks.

Referenced by perform_initial_scan().

+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ token()

AccessToken& XrdPfc::ResourceMonitor::token ( int  i)
inline

Definition at line 229 of file XrdPfcResourceMonitor.hh.

229 { return m_access_tokens[i]; }

Referenced by process_queues(), and register_file_update_stats().

+ Here is the caller graph for this function:

◆ update_vs_and_file_usage_info()

void ResourceMonitor::update_vs_and_file_usage_info ( )

Definition at line 598 of file XrdPfcResourceMonitor.cc.

599 {
600  static const char *trc_pfx = "update_vs_and_file_usage_info() ";
601 
602  const auto &conf = Cache::Conf();
603  XrdOssVSInfo vsi;
604 
605  // StatVS error (after it succeeded in config) implies a memory corruption (according to Mr. H).
606  if (m_oss.StatVS(&vsi, conf.m_data_space.c_str(), 1) < 0) {
607  TRACE(Error, trc_pfx << "can't get StatVS for oss space '" << conf.m_data_space << "'. This is a fatal error.");
608  _exit(1);
609  }
610  m_fs_state.m_disk_total = vsi.Total;
611  m_fs_state.m_disk_used = vsi.Total - vsi.Free;
612  m_fs_state.m_file_usage = 512ll * m_current_usage_in_st_blocks;
613  if (m_oss.StatVS(&vsi, conf.m_meta_space.c_str(), 1) < 0) {
614  TRACE(Error, trc_pfx << "can't get StatVS for oss space '" << conf.m_meta_space << "'. This is a fatal error.");
615  _exit(1);
616  }
617  m_fs_state.m_meta_total = vsi.Total;
618  m_fs_state.m_meta_used = vsi.Total - vsi.Free;
619 }
long long Total
Definition: XrdOssVS.hh:90
long long Free
Definition: XrdOssVS.hh:91
virtual int StatVS(XrdOssVSInfo *vsP, const char *sname=0, int updt=0)
Definition: XrdOss.cc:117

References XrdPfc::Cache::Conf(), Error, XrdOssVSInfo::Free, XrdPfc::DataFsStateBase::m_disk_total, XrdPfc::DataFsStateBase::m_disk_used, XrdPfc::DataFsStateBase::m_file_usage, XrdPfc::DataFsStateBase::m_meta_total, XrdPfc::DataFsStateBase::m_meta_used, XrdOss::StatVS(), XrdOssVSInfo::Total, and TRACE.

Referenced by heart_beat(), and perform_initial_scan().

+ Here is the call graph for this function:
+ Here is the caller graph for this function:

Member Data Documentation

◆ m_purge_task_active

bool XrdPfc::ResourceMonitor::m_purge_task_active {false}

Definition at line 260 of file XrdPfcResourceMonitor.hh.

Referenced by heart_beat(), and perform_purge_check().

◆ m_purge_task_complete

bool XrdPfc::ResourceMonitor::m_purge_task_complete {false}

Definition at line 261 of file XrdPfcResourceMonitor.hh.

Referenced by heart_beat(), and perform_purge_task_cleanup().

◆ m_purge_task_cond

XrdSysCondVar XrdPfc::ResourceMonitor::m_purge_task_cond {0}

◆ m_purge_task_end

time_t XrdPfc::ResourceMonitor::m_purge_task_end {0}

Definition at line 259 of file XrdPfcResourceMonitor.hh.

Referenced by perform_purge_task_cleanup().

◆ m_purge_task_start

time_t XrdPfc::ResourceMonitor::m_purge_task_start {0}

Definition at line 258 of file XrdPfcResourceMonitor.hh.

Referenced by perform_purge_task().


The documentation for this class was generated from the following files: