XRootD
XrdPfcResourceMonitor.cc
Go to the documentation of this file.
2 #include "XrdPfc.hh"
4 #include "XrdPfcFsTraversal.hh"
5 #include "XrdPfcDirState.hh"
8 #include "XrdPfcTrace.hh"
9 #include "XrdPfcPurgePin.hh"
10 
11 #include "XrdOss/XrdOss.hh"
12 
13 #include <algorithm>
14 
15 // #define RM_DEBUG
16 #ifdef RM_DEBUG
17 #define dprintf(...) printf(__VA_ARGS__)
18 #else
19 #define dprintf(...) (void(0))
20 #endif
21 
22 using namespace XrdPfc;
23 
24 namespace
25 {
26  XrdSysTrace* GetTrace() { return Cache::GetInstance().GetTrace(); }
27  const char *m_traceID = "ResourceMonitor";
28 }
29 
30 //------------------------------------------------------------------------------
31 
33  m_fs_state(* new DataFsState),
34  m_oss(oss)
35 {
36  // XXXX This should come from config. Also: N2N?
37  m_protected_top_dirs.insert("pfc-stats");
38 }
39 
41 {
42  delete &m_fs_state;
43 }
44 
45 //------------------------------------------------------------------------------
46 // Initial scan
47 //------------------------------------------------------------------------------
48 
50 {
51  m_dir_scan_mutex.Lock();
52  if (m_dir_scan_in_progress) {
53  bool dir_checked = false;
54  m_dir_scan_open_requests.push_back({lfn, cond, dir_checked});
55  cond.Lock();
56  m_dir_scan_mutex.UnLock();
57  while ( ! dir_checked)
58  cond.Wait();
59  cond.UnLock();
60  } else {
61  m_dir_scan_mutex.UnLock();
62  }
63 }
64 
65 void ResourceMonitor::process_inter_dir_scan_open_requests(FsTraversal &fst)
66 {
67  m_dir_scan_mutex.Lock();
68  while ( ! m_dir_scan_open_requests.empty())
69  {
70  LfnCondRecord &lcr = m_dir_scan_open_requests.front();
71  m_dir_scan_mutex.UnLock();
72 
73  cross_check_or_process_oob_lfn(lcr.f_lfn, fst);
74  lcr.f_cond.Lock();
75  lcr.f_checked = true;
76  lcr.f_cond.Signal();
77  lcr.f_cond.UnLock();
78 
79  m_dir_scan_mutex.Lock();
80  m_dir_scan_open_requests.pop_front();
81  }
82  m_dir_scan_mutex.UnLock();
83 }
84 
85 void ResourceMonitor::cross_check_or_process_oob_lfn(const std::string &lfn, FsTraversal &fst)
86 {
87  // Check if lfn has already been processed ... or process it now and mark
88  // the DirState accordingly (partially processed oob).
89  static const char *trc_pfx = "cross_check_or_process_oob_lfn() ";
90 
91  // last-existing-dir-state is currently not used. It was expected some work would need
92  // to be done along the path ... but we know they are all empty as they were just
93  // created. We could tag them all as already scanned.
94  DirState *last_existing_ds = nullptr;
95  DirState *ds = m_fs_state.find_dirstate_for_lfn(lfn, &last_existing_ds);
96  if (ds->m_scanned)
97  return;
98 
99  size_t pos = lfn.find_last_of("/");
100  std::string dir = (pos == std::string::npos) ? "" : lfn.substr(0, pos);
101 
102  XrdOssDF *dhp = m_oss.newDir(trc_pfx);
103  bool opened = (dhp->Opendir(dir.c_str(), fst.default_env()) == XrdOssOK);
104  if (opened)
105  {
106  fst.slurp_dir_ll(*dhp, ds->m_depth, dir.c_str(), trc_pfx);
107 
108  // XXXX clone of function below .... move somewhere? Esp. removal of non-paired files?
109  DirUsage &here = ds->m_here_usage;
110  for (auto it = fst.m_current_files.begin(); it != fst.m_current_files.end(); ++it)
111  {
112  if (it->second.has_data && it->second.has_cinfo) {
113  here.m_StBlocks += it->second.stat_data.st_blocks;
114  here.m_NFiles += 1;
115  seed_last_times_from_cinfo(here, it->second.stat_cinfo);
116  }
117  }
118  }
119  delete dhp;
120  // Only a directory that was read counts as scanned, else its files are skipped.
121  if (opened)
122  ds->m_scanned = true;
123 }
124 
126 {
127  dprintf("In scan_dir_and_recurse for '%s', size of dir_vec = %d, file_stat_map = %d\n",
128  fst.m_current_path.c_str(),
129  (int)fst.m_current_dirs.size(), (int)fst.m_current_files.size());
130 
131  // Breadth first, accumulate into "here", unless it was already scanned via an
132  // OOB open file request.
133  if ( ! fst.m_dir_state->m_scanned)
134  {
135  DirUsage &here = fst.m_dir_state->m_here_usage;
136  for (auto it = fst.m_current_files.begin(); it != fst.m_current_files.end(); ++it)
137  {
138  dprintf("would be doing something with %s ... has_data=%d, has_cinfo=%d\n",
139  it->first.c_str(), it->second.has_data, it->second.has_cinfo);
140 
141  // XXX Make some of these optional?
142  // Remove files that do not have both cinfo and data?
143  // Remove empty directories before even descending?
144  // Leave this for some consistency pass?
145  // Note that FsTraversal supports ignored paths ... some details (config, N2N) to be clarified.
146 
147  if (it->second.has_data && it->second.has_cinfo) {
148  here.m_StBlocks += it->second.stat_data.st_blocks;
149  here.m_NFiles += 1;
150  seed_last_times_from_cinfo(here, it->second.stat_cinfo);
151  }
152  }
153  fst.m_dir_state->m_scanned = true;
154  }
155 
156  // Swap-out directories as inter_dir_scan can use the FsTraversal.
157  std::vector<std::string> dirs;
158  dirs.swap(fst.m_current_dirs);
159 
160  if (++m_dir_scan_check_counter >= 100)
161  {
162  process_inter_dir_scan_open_requests(fst);
163  m_dir_scan_check_counter = 0;
164  }
165 
166  // Descend into sub-dirs, do not accumulate into recursive_subdir_usage yet. This is done
167  // in a separate pass to allow for proper accounting of files being opened during the initial scan.
168  for (auto &dname : dirs)
169  {
170  if (fst.cd_down(dname))
171  {
173  fst.cd_up();
174  }
175  // XXX else try to remove it?
176  }
177 }
178 
180 {
181  // Called after PFC configuration is complete, but before full startup of the daemon.
182  // Base line usages are accumulated as part of the file-system, traversal.
183 
185 
186  DirState *root_ds = m_fs_state.get_root();
187  FsTraversal fst(m_oss);
188  fst.m_protected_top_dirs = m_protected_top_dirs;
189 
190  if ( ! fst.begin_traversal(root_ds, "/"))
191  return false;
192 
193  // The following are initialized in ResourceMonitor.hh to avoid a race at startup:
194  // m_dir_scan_in_progress = true;
195  // m_dir_scan_check_counter = 0;
196 
198 
199  fst.end_traversal();
200 
201  // We have all directories scanned, available in DirState tree, let all remaining files go
202  // and then we shall do the upward propagation of usages.
203  {
204  XrdSysMutexHelper _lock(m_dir_scan_mutex);
205  m_dir_scan_in_progress = false;
206  m_dir_scan_check_counter = 0;
207  }
208  // m_dir_scan_open_requests should now be final, ie, no new entries will be added.
209  while ( ! m_dir_scan_open_requests.empty())
210  {
211  LfnCondRecord &lcr = m_dir_scan_open_requests.front();
212  lcr.f_cond.Lock();
213  lcr.f_checked = true;
214  lcr.f_cond.Signal();
215  lcr.f_cond.UnLock();
216 
217  m_dir_scan_open_requests.pop_front();
218  }
219 
220  // Do upward propagation of usages.
222  m_current_usage_in_st_blocks = root_ds->m_here_usage.m_StBlocks +
225 
226  return true;
227 }
228 
229 //------------------------------------------------------------------------------
230 // Processing of queues
231 //------------------------------------------------------------------------------
232 
234 {
235  static const char *trc_pfx = "process_queues() ";
236 
237  // Assure that we pick up only entries that are present now.
238  // We really want all open records to be processed before file-stats updates
239  // and all those before the close records.
240  // Purges are sort of tangential as they really just modify bytes / number
241  // of files in a directory and do not deal with any persistent file id tokens.
242 
243  int n_records = 0;
244  {
245  XrdSysMutexHelper _lock(&m_queue_mutex);
246  n_records += m_file_open_q.swap_queues();
247  n_records += m_file_update_stats_q.swap_queues();
248  n_records += m_file_close_q.swap_queues();
249  n_records += m_file_purge_q1.swap_queues();
250  n_records += m_file_purge_q2.swap_queues();
251  n_records += m_file_purge_q3.swap_queues();
252  ++m_queue_swap_u1;
253  }
254 
255  for (auto &i : m_file_open_q.read_queue())
256  {
257  // i.id: LFN, i.record: OpenRecord
258  AccessToken &at = token(i.id);
259  dprintf("process file open for token %d, time %ld -- %s\n",
260  i.id, i.record.m_open_time, at.m_filename.c_str());
261 
262  // Resolve fname into DirState.
263  // We could clear the filename after this ... or keep it, should we need it later on.
264  // For now it is just used for debug printouts.
265  DirState *last_existing_ds = nullptr;
266  DirState *ds = m_fs_state.find_dirstate_for_lfn(at.m_filename, &last_existing_ds);
267  at.m_dir_state = ds;
268  note_file_open(ds, last_existing_ds, i.record.m_existing_file);
269 
270  ds->m_here_usage.m_LastOpenTime = i.record.m_open_time;
271  }
272 
273  for (auto &i : m_file_update_stats_q.read_queue())
274  {
275  // i.id: token, i.record: Stats
276  AccessToken &at = token(i.id);
277  // Stats
278  DirState *ds = at.m_dir_state;
279  dprintf("process file update for token %d, %p -- %s\n",
280  i.id, ds, at.m_filename.c_str());
281 
282  ds->m_here_stats.AddUp(i.record);
283  m_current_usage_in_st_blocks += i.record.m_StBlocksAdded;
284  }
285 
286  for (auto &i : m_file_close_q.read_queue())
287  {
288  // i.id: token, i.record: CloseRecord
289  AccessToken &at = token(i.id);
290  dprintf("process file close for token %d, time %ld -- %s\n",
291  i.id, i.record.m_close_time, at.m_filename.c_str());
292 
293  DirState *ds = at.m_dir_state;
294  ds->m_here_stats.m_NFilesClosed += 1;
295  ds->m_here_usage.m_LastCloseTime = i.record.m_close_time;
296 
297  }
298  { // Under the lock: clear() writes fields register_file_update_stats() uses.
299  XrdSysMutexHelper _lock(&m_queue_mutex);
300  for (auto &i : m_file_close_q.read_queue())
301  m_access_tokens.release(i.id);
302  }
303 
304  for (auto &i : m_file_purge_q1.read_queue())
305  {
306  // i.id: DirState*, i.record: PurgeRecord
307  DirState *ds = i.id;
308  ds->m_here_stats.m_StBlocksRemoved += i.record.m_size_in_st_blocks;
309  ds->m_here_stats.m_NFilesRemoved += i.record.m_n_files;
310  m_current_usage_in_st_blocks -= i.record.m_size_in_st_blocks;
311  }
312  for (auto &i : m_file_purge_q2.read_queue())
313  {
314  // i.id: directory-path, i.record: PurgeRecord
315  DirState *ds = m_fs_state.get_root()->find_path(i.id, -1, false, false);
316  if ( ! ds) {
317  if (is_in_protected_top_dir(i.id)) {
318  TRACE(Debug, trc_pfx << "no DirState for '" << i.id
319  << "', as expected under a protected top directory.");
320  } else {
321  TRACE(Error, trc_pfx << "DirState not found for directory path '" << i.id << "'.");
322  }
323  // find_path can return the last dir found ... but this clearly isn't a valid purge record.
324  continue;
325  }
326  ds->m_here_stats.m_StBlocksRemoved += i.record.m_size_in_st_blocks;
327  ds->m_here_stats.m_NFilesRemoved += i.record.m_n_files;
328  m_current_usage_in_st_blocks -= i.record.m_size_in_st_blocks;
329  }
330  for (auto &i : m_file_purge_q3.read_queue())
331  {
332  // i.id: LFN, i.record: size of file in st_blocks
333  DirState *ds = m_fs_state.get_root()->find_path(i.id, -1, true, false);
334  if ( ! ds) {
335  if (is_in_protected_top_dir(i.id)) {
336  TRACE(Debug, trc_pfx << "no DirState for '" << i.id
337  << "', as expected under a protected top directory.");
338  } else {
339  TRACE(Error, trc_pfx << "DirState not found for LFN path '" << i.id << "'.");
340  }
341  continue;
342  }
343  ds->m_here_stats.m_StBlocksRemoved += i.record;
344  ds->m_here_stats.m_NFilesRemoved += 1;
345  m_current_usage_in_st_blocks -= i.record;
346  }
347 
348  // Read queues / vectors are cleared at swap time.
349  // We might consider reducing their capacity by half if, say, their usage is below 25%.
350 
351  return n_records;
352 }
353 
354 //------------------------------------------------------------------------------
355 // Heart beat
356 //------------------------------------------------------------------------------
357 
359 {
360  static const char *tpfx = "heart_beat() ";
361 
362  const Configuration &conf = Cache::Conf();
363 
364  const int s_queue_proc_interval = 10;
365  const int s_sshot_report_interval = conf.m_dirStatsInterval; // one of 60, 120, 300,
366  // 600, 900, 1200, 1800, 3600 s
367  const int s_purge_check_interval = 60;
368  const int s_purge_report_interval = conf.m_purgeInterval;
369  const int s_purge_cold_files_interval = conf.m_purgeInterval * conf.m_purgeAgeBasedPeriod;
370 
371  // initial scan performed as part of config
372 
373  time_t now = time(0);
374  time_t next_queue_proc_time = now + s_queue_proc_interval;
375  time_t next_sshot_report_time = (now / 60) * 60 + 60; // at next full minute
376  time_t next_purge_check_time = now + s_purge_check_interval;
377  time_t next_purge_report_time = now + s_purge_report_interval;
378  time_t next_purge_cold_files_time = now + s_purge_cold_files_interval;
379 
380  long long n_reported_token_failures = 0;
381 
382  while (true)
383  {
384  time_t start = time(0);
385  time_t next_event = std::min({ next_queue_proc_time, next_sshot_report_time,
386  next_purge_check_time, next_purge_report_time, next_purge_cold_files_time });
387 
388  if (next_event > start)
389  {
390  unsigned int t_sleep = next_event - start;
391  TRACE(Dump, tpfx << "sleeping for " << t_sleep << " seconds until the next beat.");
392  sleep(t_sleep);
393  }
394 
395  // Check if purge has been running and has completed yet.
396  // For now this is only used to prevent removal of empty leaf directories
397  // while it runs, so we do not need to wait for the condition in the above sleep.
398  if (m_purge_task_active) {
400  if (m_purge_task_complete) {
402  }
403  }
404 
405  time_t queue_swap_time = time(0);
406 
407  // Always process the queues.
408  int n_processed = process_queues();
409  next_queue_proc_time = queue_swap_time + s_queue_proc_interval;
410  TRACE(Dump, tpfx << "process_queues -- n_records=" << n_processed);
411 
412  { // Access-token space exhausted -- should be unreachable, so say so loudly.
413  long long taf;
414  { XrdSysMutexHelper _lock(&m_queue_mutex); taf = m_token_alloc_failures; }
415  if (taf > n_reported_token_failures) {
416  TRACE(Error, tpfx << "access-token space exhausted "
417  << taf - n_reported_token_failures << " time(s) (total " << taf
418  << "); those files are open but not accounted for. Raise MaxChunks "
419  "on ResourceMonitor::m_access_tokens.");
420  n_reported_token_failures = taf;
421  }
422  }
423 
424  // Always update basic info on m_fs_state (space, usage, file_usage).
426 
427  now = time(0);
428 
429  // Make planning for fs_state_update, sshot dump and purge task.
430  // Second two require the first, so figure out what is going to happen.
431  bool do_sshot_report = next_sshot_report_time <= now;
432  bool do_purge_check = next_purge_check_time <= now;
433  bool do_purge_report = next_purge_report_time <= now;
434  bool do_purge_cold_files = next_purge_cold_files_time <= now;
435 
436  // Update stats in usages if any secondary activity will happen.
437  if (do_sshot_report || do_purge_check || do_purge_report || do_purge_cold_files)
438  {
439  m_fs_state.update_stats_and_usages(queue_swap_time, do_sshot_report);
440 
441  // This reporting into log/stdout is to be removed.
442  // Meaning of conf.is_dir_stat_reporting_on() etc is to be clarified / improved.
443  if (do_sshot_report && conf.is_dir_stat_reporting_on())
444  {
445  const int store_depth = conf.m_dirStatsStoreDepth;
446  #ifdef RM_DEBUG
447  const DirState &root_ds = *m_fs_state.get_root();
448  dprintf("Snapshot n_dirs=%d, total n_dirs=%d\n", root_ds.count_dirs_to_level(store_depth),
450  #endif
451  m_fs_state.dump_recursively(store_depth);
452  }
453 
454  }
455 
456  if (do_sshot_report)
457  {
458  // Sshot reports are equidistant, at "full" reporting interval.
459  next_sshot_report_time = ((now + 1) / s_sshot_report_interval) * s_sshot_report_interval + s_sshot_report_interval;
460 
461  // This should dump out binary snapshot into /pfc-stats/, if so configured.
462 
463  // json dump to std::out for debug purpose
464  DataFsSnapshot ss(m_fs_state, m_fs_state.m_sshot_stats_reset_time);
465  ss.m_st_blocks_fuzz = m_fs_state.m_st_blocks_fuzz;
466  const DirState &root_ds = *m_fs_state.get_root();
467  const int store_depth = conf.m_dirStatsStoreDepth;
468  const int n_sshot_dirs = root_ds.count_dirs_to_level(store_depth);
469  ss.m_dir_states.reserve(n_sshot_dirs);
470  ss.m_dir_states.emplace_back( DirStateElement(root_ds, -1) );
471  fill_sshot_vec_children(root_ds, 0, ss.m_dir_states, store_depth);
472 
473  // This should really be export to a file (preferably binary, but then bin->json command is needed, too).
474  // ss.dump();
475 
476  const char* dumpfile = "/pfc-stats/DirStat.json";
477  ss.write_json_file(dumpfile, m_oss, false);
478  m_fs_state.reset_sshot_stats(queue_swap_time);
479  }
480 
481  if (do_purge_check || do_purge_report || do_purge_cold_files)
482  {
483  perform_purge_check(do_purge_cold_files, do_purge_report ? TRACE_Info : TRACE_Debug);
484 
485  next_purge_check_time = now + s_purge_check_interval;
486  if (do_purge_report) next_purge_report_time = now + s_purge_report_interval;
487  if (do_purge_cold_files) next_purge_cold_files_time = now + s_purge_cold_files_interval;
488  }
489 
490  // Prune the empty leaves the last snapshot reported, after the purge check, which
491  // needs the tree as propagated, and never while a purge task traverses the
492  // directories: one started above defers this to a later beat.
493  if ( ! m_fs_state.m_reap_candidates.empty() && ! m_purge_task_active)
494  {
495  unlink_func unlink_foo = [&](const std::string &dp)->int {
496  int ret = m_oss.Unlink(dp.c_str());
497  if (ret != 0) {
498  TRACE(Info, tpfx << "Empty dir unlink error: " << ret << " at " << dp);
499  } else {
500  TRACE(Debug, tpfx << "Empty dir unlink success: " << dp);
501  }
502  return ret;
503  };
504  // The residue leaves the tree at the next propagation; keep the total in step.
505  m_current_usage_in_st_blocks -= m_fs_state.reap_empty_leaf_dirs(unlink_foo);
506  }
507 
508  } // end while forever
509 }
510 
511 //------------------------------------------------------------------------------
512 // DirState export helpers
513 //------------------------------------------------------------------------------
514 
516  int parent_idx,
517  std::vector<DirStateElement> &vec,
518  int max_depth)
519 {
520  const int pos0 = (int) vec.size();
521  const int n_children = (int) parent_ds.m_subdirs.size();
522 
523  // maxdepth 0 is the root alone: a node at max_depth is emitted, its children are not.
524  if (parent_ds.m_depth >= max_depth)
525  {
526  // Truncated here: keep the daughter range at (-1, -1). No children: empty range.
527  if (n_children == 0) {
528  DirStateElement &dse = vec[parent_idx];
529  dse.m_daughters_begin = dse.m_daughters_end = pos0;
530  }
531  return;
532  }
533 
534  { // scoped: the emplace_back loop below can reallocate vec
535  DirStateElement &parent_dse = vec[parent_idx];
536  parent_dse.m_daughters_begin = pos0;
537  parent_dse.m_daughters_end = pos0 + n_children;
538  }
539 
540  if (n_children == 0) return;
541 
542  for (auto const & [name, child] : parent_ds.m_subdirs)
543  {
544  vec.emplace_back( DirStateElement(child, parent_idx) );
545  }
546 
547  int pos = pos0;
548  for (auto const & [name, child] : parent_ds.m_subdirs)
549  {
550  fill_sshot_vec_children(child, pos, vec, max_depth);
551  ++pos;
552  }
553 }
554 
556  int parent_idx,
557  std::vector<DirPurgeElement> &vec,
558  int max_depth)
559 {
560  const int pos0 = (int) vec.size();
561  const int n_children = (int) parent_ds.m_subdirs.size();
562 
563  // maxdepth 0 is the root alone: a node at max_depth is emitted, its children are not.
564  if (parent_ds.m_depth >= max_depth)
565  {
566  // Truncated here: keep the daughter range at (-1, -1). No children: empty range.
567  if (n_children == 0) {
568  DirPurgeElement &dse = vec[parent_idx];
569  dse.m_daughters_begin = dse.m_daughters_end = pos0;
570  }
571  return;
572  }
573 
574  { // scoped: the emplace_back loop below can reallocate vec
575  DirPurgeElement &parent_dse = vec[parent_idx];
576  parent_dse.m_daughters_begin = pos0;
577  parent_dse.m_daughters_end = pos0 + n_children;
578  }
579 
580  if (n_children == 0) return;
581 
582  for (auto const & [name, child] : parent_ds.m_subdirs)
583  {
584  vec.emplace_back( DirPurgeElement(child, child.m_here_usage, child.m_recursive_subdir_usage, parent_idx) );
585  }
586 
587  int pos = pos0;
588  for (auto const & [name, child] : parent_ds.m_subdirs)
589  {
590  fill_pshot_vec_children(child, pos, vec, max_depth);
591  ++pos;
592  }
593 }
594 
595 //------------------------------------------------------------------------------
596 // Purge helpers, drivers, etc.
597 //------------------------------------------------------------------------------
598 
600 {
601  static const char *trc_pfx = "update_vs_and_file_usage_info() ";
602 
603  const auto &conf = Cache::Conf();
604  XrdOssVSInfo vsi;
605 
606  // StatVS error (after it succeeded in config) implies a memory corruption (according to Mr. H).
607  if (m_oss.StatVS(&vsi, conf.m_data_space.c_str(), 1) < 0) {
608  TRACE(Error, trc_pfx << "can't get StatVS for oss space '" << conf.m_data_space << "'. This is a fatal error.");
609  _exit(1);
610  }
611  m_fs_state.m_disk_total = vsi.Total;
612  m_fs_state.m_disk_used = vsi.Total - vsi.Free;
613  m_fs_state.m_file_usage = 512ll * m_current_usage_in_st_blocks;
614  if (m_oss.StatVS(&vsi, conf.m_meta_space.c_str(), 1) < 0) {
615  TRACE(Error, trc_pfx << "can't get StatVS for oss space '" << conf.m_meta_space << "'. This is a fatal error.");
616  _exit(1);
617  }
618  m_fs_state.m_meta_total = vsi.Total;
619  m_fs_state.m_meta_used = vsi.Total - vsi.Free;
620 }
621 
622 long long ResourceMonitor::get_file_usage_bytes_to_remove(const DataFsPurgeshot &ps, long long write_estimate, int tl)
623 {
624  // short names from config values
625  const Configuration &conf = Cache::Conf();
626  long long f0 = conf.m_fileUsageBaseline;
627  long long f1 = conf.m_fileUsageNominal;
628  long long f2 = conf.m_fileUsageMax;
629  long long w1 = conf.m_diskUsageLWM;
630  long long w2 = conf.m_diskUsageHWM;
631 
632  // get usage from purge snapshot
633  long long T = ps.m_disk_total;
634  long long x = ps.m_file_usage;
635  long long u = ps.m_disk_used;
636 
637  // get file usage increase from the previous time interval check
638  long long delta = write_estimate;
639  TRACE_INT(tl, "file usage increased since the previous purge interval in bytes: " << delta );
640 
641  long long bytes_to_remove = 0;
642 
643  // helper lambda function
644  auto clamp = [&x, &bytes_to_remove](long long lowval, long long highval)
645  {
646  long long val = x;
647  long long newval = val - bytes_to_remove;
648 
649  // removed too much
650  if (newval < lowval)
651  {
652  return lowval - val;
653  }
654 
655  // removed too little
656  if (newval > highval)
657  {
658  return val - highval;
659  }
660  // keep the original value
661  return bytes_to_remove;
662  };
663 
664  // under file quota, nothing to do
665  if (x < f0)
666  return 0;
667 
668  // total disk usage exceeds highWatermark
669  if (u >= w2)
670  {
671  TRACE_INT(tl, "Disk usage: " << ps.m_disk_used << " exceed highWatermark " << conf.m_diskUsageHWM);
672  float frac_u = static_cast<float>(u - w2) / (T - w2);
673  float frac_x = static_cast<float>(x - f0) / (f1 - f0);
674 
675  if (w2 == T)
676  {
677  bytes_to_remove = u -w1;
678  }
679  else
680  {
681  if (frac_x > frac_u)
682  {
683  // the cache is the reason for going out of w2 range
684  bytes_to_remove = (frac_x - frac_u) * (f1 - f0);
685  bytes_to_remove += delta;
686  bytes_to_remove = clamp(f0, f1);
687  }
688  else
689  {
690  // someone else is filling disk space, go to f1
691  bytes_to_remove = clamp(f0, f2);
692  }
693  return bytes_to_remove;
694  }
695  }
696 
697  // file quota and total disk usage is within normal range, check if this space usage is
698  // proportinal to disk usage and correct it
699  if (u > w1 && x > f1)
700  {
701  float frac_u = static_cast<float>(u - w1) / (w2 - w1);
702  float frac_x = static_cast<float>(x - f1) / (f2 - f1);
703  if (frac_x > frac_u)
704  {
705  TRACE_INT(tl, "Disproportional file quota usage comapared to disc usage (frac_x/frac_u) = " << frac_x << "/"<< frac_u);
706  bytes_to_remove = (frac_x - frac_u) * (f2 - f1);
707  bytes_to_remove += delta;
708  }
709 
710  // check the new x val will not be below f0
711  bytes_to_remove = clamp(f0, f2);
712  return bytes_to_remove;
713  }
714 
715  // final check: disk useage is lower that w1, check if exceed the max file usage f2
716  if (x > f2)
717  {
718  // drop usage to f2
719  // compare with global disk usage in the previous purge cycle (default 300s)
720  // check delta is not overflowing f2, else set numver of bytes to remove according remove to f0
721 
722  TRACE_INT(tl, "File usage exceeds maxim file usage. Total disk usage is under lowWatermark. Clearing to low file usage.");
723  long long f2delta = std::max(f2 - delta, f0);
724  bytes_to_remove = clamp(f0, f2delta);
725  return bytes_to_remove;
726  }
727 
728  return bytes_to_remove;
729 }
730 
731 void ResourceMonitor::perform_purge_check(bool purge_cold_files, int tl)
732 {
733  static const char *trc_pfx = "perform_purge_check() ";
734  const Configuration &conf = Cache::Conf();
735 
736  std::unique_ptr<DataFsPurgeshot> psp( new DataFsPurgeshot(m_fs_state) );
737  DataFsPurgeshot &ps = *psp;
738 
739  ps.m_file_usage = 512ll * m_current_usage_in_st_blocks;
740  // These are potentially wrong as cache might be writing over preallocated byte ranges.
742  // Can have another estimate based on eiter writes or st-blocks from purge-stats, once we have them.
743 
744  TRACE_INT(tl, trc_pfx << "Purge check:");
745 
746  ps.m_bytes_to_remove = 0;
747  if (conf.are_file_usage_limits_set())
748  {
749  ps.m_bytes_to_remove = get_file_usage_bytes_to_remove(ps, ps.m_estimated_writes_from_writeq, tl);
750  }
751  else
752  {
753  if (ps.m_disk_used > conf.m_diskUsageHWM)
754  {
755  TRACE_INT(tl, "Disk usage: " << ps.m_disk_used << " exceed highWatermark.");
757  }
758  }
759 
760  ps.m_space_based_purge = ps.m_bytes_to_remove ? 1 : 0;
761 
762  // Purge precheck -- check if age-based purge is required
763  // We ignore uvkeep time, it requires reading of cinfo files and it is enforced in File::Open() anyway.
764 
765  if (purge_cold_files && conf.is_age_based_purge_in_effect()) // || conf.is_uvkeep_purge_in_effect())
766  {
767  ps.m_age_based_purge = true;
768  }
769 
770  TRACE_INT(tl, "\tbytes_to_remove = " << ps.m_bytes_to_remove << " B");
771  TRACE_INT(tl, "\tspace_based_purge = " << ps.m_space_based_purge);
772  TRACE_INT(tl, "\tage_based_purge = " << ps.m_age_based_purge);
773 
774  bool periodic = Cache::GetInstance().GetPurgePin() ?
776 
777  if ( ! ps.m_space_based_purge && ! ps.m_age_based_purge && !periodic ) {
778  TRACE(Info, trc_pfx << "purge not required.");
780  return;
781  }
782  if (m_purge_task_active) {
783  TRACE(Warning, trc_pfx << "purge required but previous purge task is still active!");
784  return;
785  }
786 
787  TRACE(Info, trc_pfx << "scheduling purge task.");
788 
789  // At this point we have all the information: report, decide on action.
790  // There is still some missing infrastructure, especially as regards to purge-plugin:
791  // - at what point do we start bugging the pu-pin to start coughing up purge lists?
792  // - have a new parameter or just do it "one cycle before full"?
793  // - what if it doesn't -- when do we do the old-stlye scan & purge?
794  // - how do we do age-based purge and uvkeep purge?
795  // - they are really quite different -- and could run separately, registering
796  // files into a purge-candidate list. This has to be rechecked before the actual
797  // deletion -- eg, by comparing stat time of cinfo + doing the is-active / is-purge-protected.
798 
799  const DirState &root_ds = *m_fs_state.get_root();
800 
801  // Size from the tree itself; the usage counters can be off (#2808).
802  const int n_pshot_dirs = root_ds.count_dirs_to_level(9999);
803 
804  // Report a counters-vs-tree mismatch when it appears or changes.
805  const int n_calc_dirs = 1 + root_ds.m_here_usage.m_NDirectories
807  dprintf("purge dir count recursive=%d vs from_usage=%d\n", n_pshot_dirs, n_calc_dirs);
808  const int discrepancy = n_calc_dirs - n_pshot_dirs;
809  if (discrepancy != m_dir_count_discrepancy)
810  {
811  if (discrepancy != 0) {
812  TRACE(Error, trc_pfx << "directory count mismatch: usage counters " << n_calc_dirs
813  << ", tree " << n_pshot_dirs << ", previous difference " << m_dir_count_discrepancy);
814  } else {
815  TRACE(Info, trc_pfx << "directory count consistent again: " << n_pshot_dirs);
816  }
817  m_dir_count_discrepancy = discrepancy;
818  }
819 
820  ps.m_dir_vec.reserve(n_pshot_dirs);
821  ps.m_dir_vec.emplace_back( DirPurgeElement(root_ds, root_ds.m_here_usage, root_ds.m_recursive_subdir_usage, -1) );
822  fill_pshot_vec_children(root_ds, 0, ps.m_dir_vec, 9999);
823 
824  m_purge_task_active = true;
825 
826  struct PurgeDriverJob : public XrdJob
827  {
828  DataFsPurgeshot *m_purge_shot_ptr;
829 
830  PurgeDriverJob(DataFsPurgeshot *psp) :
831  XrdJob("XrdPfc::ResourceMonitor::PurgeDriver"),
832  m_purge_shot_ptr(psp)
833  {}
834 
835  void DoIt() override
836  {
837  Cache::ResMon().perform_purge_task(*m_purge_shot_ptr);
839 
840  delete m_purge_shot_ptr;
841  delete this;
842  }
843  };
844 
845  Cache::schedP->Schedule( new PurgeDriverJob(psp.release()) );
846 }
847 
848 namespace XrdPfc
849 {
851 }
852 
854 {
855  // BEWARE: Runs in a dedicated thread - is only to communicate back to the
856  // hear_beat() / data structs via the purge queues and condition variable.
857 
858  // const char *tpfx = "perform_purge_task ";
859 
860  {
862  m_purge_task_start = time(0);
863  }
864 
865  // For now, fall back to the old purge ... to be improved with:
866  // - new scan, following the DataFsPurgeshot;
867  // - usage of cinfo stat mtime for time of last access (touch already done at output);
868  // - use DirState* to report back purged files.
869  // Already changed to report back purged files --- but using the string / path variant.
870  OldStylePurgeDriver(ps); // In XrdPfcPurge.cc
871 }
872 
874 {
875  // Separated out so the purge_task can exit without post-checks.
876 
877  {
879  m_purge_task_end = time(0);
880  m_purge_task_complete = true;
882  }
884 }
885 
886 //==============================================================================
887 // Main thread function, do initial test, then enter heart_beat().
888 //==============================================================================
889 
891 {
892  // setup for in-scan -- this is called from initial setup.
893  MutexHolder _lck(m_dir_scan_mutex);
894  m_dir_scan_in_progress = true;
895 }
896 
898 {
899  const char *tpfx = "main_thread_function ";
900  {
901  time_t is_start = time(0);
902  m_fs_state.init_stat_reset_times(is_start);
903  TRACE(Info, tpfx << "Stating initial directory scan.");
904 
905  if ( ! perform_initial_scan()) {
906  TRACE(Error, tpfx << "Initial directory scan has failed. This is a terminal error, aborting.")
907  _exit(1);
908  }
909  // Reset of m_dir_scan_in_progress is done in perform_initial_scan()
910 
911  time_t is_duration = time(0) - is_start;
912  TRACE(Info, tpfx << "Initial directory scan complete, duration=" << is_duration <<"s");
913 
914  // run first process queues
915  int n_proc_is = process_queues();
916  TRACE(Info, tpfx << "First process_queues finished, n_records=" << n_proc_is);
917 
918  // shrink queues if scan time was longer than 30s.
919  if (is_duration > 30 || n_proc_is > 3000)
920  {
921  m_file_open_q.shrink_read_queue();
922  m_file_update_stats_q.shrink_read_queue();
923  m_file_close_q.shrink_read_queue();
924  m_file_purge_q1.shrink_read_queue();
925  m_file_purge_q2.shrink_read_queue();
926  m_file_purge_q3.shrink_read_queue();
927  }
928  }
929  heart_beat();
930 }
931 
932 //==============================================================================
933 // Old prototype from Cache / Purge, now to go into heart_beat() here, above.
934 //==============================================================================
935 
937 {
938  // static const char *trc_pfx = "ResourceMonitorHeartBeat() ";
939 
940  // Pause before initial run
941  sleep(1);
942 
943  // XXXX Setup initial / constant stats (total RAM, total disk, ???)
944 
947 
948  S.Lock();
949 
951 
953 
954  S.UnLock();
955 
956  // XXXX Schedule initial disk scan, time it!
957  //
958  // TRACE(Info, trc_pfx << "scheduling intial disk scan.");
959  // schedP->Schedule( new ScanAndPurgeJob("XrdPfc::ScanAndPurge") );
960  //
961  // bool scan_and_purge_running = true;
962 
963  // XXXX Could we really hold last-usage for all files in memory?
964 
965  // XXXX Think how to handle disk-full, scan/purge not finishing:
966  // - start dropping things out of write queue, but only when RAM gets near full;
967  // - monitoring this then becomes a high-priority job, inner loop with sleep of,
968  // say, 5 or 10 seconds.
969 
970  while (true)
971  {
972  time_t heartbeat_start = time(0);
973 
974  // TRACE(Info, trc_pfx << "HeartBeat starting ...");
975 
976  // if sumary monitoring configured, pupulate OucCacheStats:
977  S.Lock();
978 
979  // - available / used disk space (files usage calculated elsewhere (maybe))
980 
981  // - RAM usage
982  /* XXXX From Cache
983  { XrdSysMutexHelper lck(&m_RAM_mutex);
984  X.MemUsed = m_RAM_used;
985  X.MemWriteQ = m_RAM_write_queue;
986  }
987  */
988 
989  // - files opened / closed etc
990 
991  // do estimate of available space
992  S.UnLock();
993 
994  // if needed, schedule purge in a different thread.
995  // purge is:
996  // - deep scan + gather FSPurgeState
997  // - actual purge
998  //
999  // this thread can continue running and, if needed, stop writing to disk
1000  // if purge is taking too long.
1001 
1002  // think how data is passed / synchronized between this and purge thread
1003 
1004  // !!!! think how stat collection is done and propgated upwards;
1005  // until now it was done once per purge-interval.
1006  // now stats will be added up more often, but purge will be done
1007  // only occasionally.
1008  // also, do we report cumulative values or deltas? cumulative should
1009  // be easier and consistent with summary data.
1010  // still, some are state - like disk usage, num of files.
1011 
1012  // Do we take care of directories that need to be newly added into DirState hierarchy?
1013  // I.e., when user creates new directories and these are covered by either full
1014  // spec or by root + depth declaration.
1015 
1016  int heartbeat_duration = time(0) - heartbeat_start;
1017 
1018  // TRACE(Info, trc_pfx << "HeartBeat finished, heartbeat_duration " << heartbeat_duration);
1019 
1020  // int sleep_time = m_fs_state..m_purgeInterval - heartbeat_duration;
1021  int sleep_time = 60 - heartbeat_duration;
1022  if (sleep_time > 0)
1023  {
1024  sleep(sleep_time);
1025  }
1026  }
1027 }
int DoIt(int argpnt, int argc, char **argv, bool singleshot)
Definition: XrdAccTest.cc:262
static void child()
#define TRACE_Debug
Definition: XrdCmsTrace.hh:37
#define TRACE_Info
#define XrdOssOK
Definition: XrdOss.hh:54
#define dprintf(...)
void Proto_ResourceMonitorHeartBeat()
#define TRACE_INT(act, x)
Definition: XrdPfcTrace.hh:52
bool Debug
@ Error
#define TRACE(act, x)
Definition: XrdTrace.hh:63
Definition: XrdJob.hh:43
virtual int Opendir(const char *path, XrdOucEnv &env)
Definition: XrdOss.hh:84
long long Total
Definition: XrdOssVS.hh:90
long long Free
Definition: XrdOssVS.hh:91
virtual XrdOssDF * newDir(const char *tident)=0
virtual int StatVS(XrdOssVSInfo *vsP, const char *sname=0, int updt=0)
Definition: XrdOss.cc:117
virtual int Unlink(const char *path, int Opts=0, XrdOucEnv *envP=0)=0
struct XrdOucCacheStats::CacheStats X
XrdOucCacheStats Statistics
Definition: XrdOucCache.hh:740
PurgePin * GetPurgePin() const
Definition: XrdPfc.hh:310
static const Configuration & Conf()
Definition: XrdPfc.cc:138
static ResourceMonitor & ResMon()
Definition: XrdPfc.cc:139
void ClearPurgeProtectedSet()
Definition: XrdPfc.cc:696
static Cache & GetInstance()
Singleton access.
Definition: XrdPfc.cc:136
XrdSysTrace * GetTrace() const
Definition: XrdPfc.hh:321
static XrdScheduler * schedP
Definition: XrdPfc.hh:328
long long WritesSinceLastCall()
Definition: XrdPfc.cc:328
void AddUp(const DirStats &s)
Definition: XrdPfcStats.hh:192
long long m_StBlocksRemoved
Definition: XrdPfcStats.hh:149
std::vector< std::string > m_current_dirs
XrdOucEnv & default_env()
void slurp_dir_ll(XrdOssDF &dh, int dir_level, const char *path, const char *trc_pfx)
bool begin_traversal(DirState *root, const char *root_path)
std::set< std::string > m_protected_top_dirs
bool cd_down(const std::string &dir_name)
std::map< std::string, FilePairStat > m_current_files
Status of cached file. Can be read from and written into a binary file.
Definition: XrdPfcInfo.hh:41
virtual bool CallPeriodically()
void CrossCheckIfScanIsInProgress(const std::string &lfn, XrdSysCondVar &cond)
void perform_purge_check(bool purge_cold_files, int tl)
void fill_sshot_vec_children(const DirState &parent_ds, int parent_idx, std::vector< DirStateElement > &vec, int max_depth)
void perform_purge_task(DataFsPurgeshot &ps)
void scan_dir_and_recurse(FsTraversal &fst)
void fill_pshot_vec_children(const DirState &parent_ds, int parent_idx, std::vector< DirPurgeElement > &vec, int max_depth)
void release(int i)
Clear the slot and return it for reuse. Caller must hold its mutex.
void Schedule(XrdJob *jp)
@ Warning
Definition: XrdPfc.hh:44
void OldStylePurgeDriver(DataFsPurgeshot &ps)
Definition: XrdPfcPurge.cc:99
void note_file_open(DirState *node, DirState *last_existing, bool existing_file)
std::function< int(const std::string &)> unlink_func
Contains parameters configurable from the xrootd config file.
Definition: XrdPfc.hh:67
long long m_RamAbsAvailable
available from configuration
Definition: XrdPfc.hh:112
long long m_diskTotalSpace
total disk space on configured partition or oss space
Definition: XrdPfc.hh:95
long long m_fileUsageMax
cache purge - files usage maximum
Definition: XrdPfc.hh:100
long long m_fileUsageBaseline
cache purge - files usage baseline
Definition: XrdPfc.hh:98
int m_dirStatsStoreDepth
maximum depth for statistics write out
Definition: XrdPfc.hh:109
long long m_diskUsageHWM
cache purge - disk usage high water mark
Definition: XrdPfc.hh:97
bool are_file_usage_limits_set() const
Definition: XrdPfc.hh:70
long long m_fileUsageNominal
cache purge - files usage nominal
Definition: XrdPfc.hh:99
int m_purgeAgeBasedPeriod
peform cold file / uvkeep purge every this many purge cycles
Definition: XrdPfc.hh:103
long long m_diskUsageLWM
cache purge - disk usage low water mark
Definition: XrdPfc.hh:96
int m_dirStatsInterval
time between resource monitor statistics dump in seconds
Definition: XrdPfc.hh:108
bool is_age_based_purge_in_effect() const
Definition: XrdPfc.hh:71
int m_purgeInterval
sleep interval between cache purges
Definition: XrdPfc.hh:101
bool is_dir_stat_reporting_on() const
Definition: XrdPfc.hh:73
std::vector< DirPurgeElement > m_dir_vec
void write_json_file(const std::string &fname, XrdOss &oss, bool include_preamble)
std::vector< DirStateElement > m_dir_states
long long m_st_blocks_fuzz
DataFsState::m_st_blocks_fuzz.
std::vector< DirState * > m_reap_candidates
void dump_recursively(int max_depth) const
void init_stat_reset_times(time_t t)
void update_stats_and_usages(time_t last_update, bool collect_reap_candidates)
DirState * find_dirstate_for_lfn(const std::string &lfn, DirState **last_existing_dir=nullptr)
long long reap_empty_leaf_dirs(unlink_func unlink_foo)
void reset_sshot_stats(time_t last_update)
DirUsage m_recursive_subdir_usage
int count_dirs_to_level(int max_depth) const
DirState * find_path(const std::string &path, int max_depth, bool parse_as_lfn, bool create_subdirs, DirState **last_existing_dir=nullptr)
void upward_propagate_initial_scan_usages()