XRootD
Loading...
Searching...
No Matches
XrdPfcResourceMonitor.cc
Go to the documentation of this file.
2#include "XrdPfc.hh"
5#include "XrdPfcDirState.hh"
8#include "XrdPfcTrace.hh"
9#include "XrdPfcPurgePin.hh"
10
11#include "XrdOss/XrdOss.hh"
12
13#include <algorithm>
14
15// #define RM_DEBUG
16#ifdef RM_DEBUG
17#define dprintf(...) printf(__VA_ARGS__)
18#else
19#define dprintf(...) (void(0))
20#endif
21
22using namespace XrdPfc;
23
24namespace
25{
26 XrdSysTrace* GetTrace() { return Cache::GetInstance().GetTrace(); }
27 const char *m_traceID = "ResourceMonitor";
28}
29
30//------------------------------------------------------------------------------
31
33 m_fs_state(* new DataFsState),
34 m_oss(oss)
35{}
36
38{
39 delete &m_fs_state;
40}
41
42//------------------------------------------------------------------------------
43// Initial scan
44//------------------------------------------------------------------------------
45
47{
48 m_dir_scan_mutex.Lock();
49 if (m_dir_scan_in_progress) {
50 bool dir_checked = false;
51 m_dir_scan_open_requests.push_back({lfn, cond, dir_checked});
52 cond.Lock();
53 m_dir_scan_mutex.UnLock();
54 while ( ! dir_checked)
55 cond.Wait();
56 cond.UnLock();
57 } else {
58 m_dir_scan_mutex.UnLock();
59 }
60}
61
62void ResourceMonitor::check_inter_dir_scan_open_requests()
63{
64 // Called from the scan thread between directories and periodically while reading
65 // a large directory. Rate-limit on time rather than on number of directories
66 // visited: a single directory can take minutes to stat on a large cache.
67 auto now = std::chrono::steady_clock::now();
68 if (now - m_dir_scan_last_check < s_dir_scan_check_interval)
69 return;
70 m_dir_scan_last_check = now;
71
72 process_inter_dir_scan_open_requests();
73}
74
75void ResourceMonitor::process_inter_dir_scan_open_requests()
76{
77 m_dir_scan_mutex.Lock();
78 // Only service the requests present on entry. Under a steady stream of opens the
79 // queue may never become empty and the initial scan would be starved.
80 size_t n_to_process = m_dir_scan_open_requests.size();
81 if (n_to_process == 0)
82 {
83 m_dir_scan_mutex.UnLock();
84 return;
85 }
86 // Use a private traversal object: this may be called while the scan's traversal
87 // is in the middle of reading a directory.
88 FsTraversal fst(m_oss);
89 while (n_to_process-- > 0)
90 {
91 LfnCondRecord &lcr = m_dir_scan_open_requests.front();
92 m_dir_scan_mutex.UnLock();
93
94 cross_check_or_process_oob_lfn(lcr.f_lfn, fst);
95 lcr.f_cond.Lock();
96 lcr.f_checked = true;
97 lcr.f_cond.Signal();
98 lcr.f_cond.UnLock();
99
100 m_dir_scan_mutex.Lock();
101 m_dir_scan_open_requests.pop_front();
102 }
103 m_dir_scan_mutex.UnLock();
104}
105
106void ResourceMonitor::cross_check_or_process_oob_lfn(const std::string &lfn, FsTraversal &fst)
107{
108 // Check if lfn has already been processed ... or process it now and mark
109 // the DirState accordingly (partially processed oob).
110 static const char *trc_pfx = "cross_check_or_process_oob_lfn() ";
111
112 // last-existing-dir-state is currently not used. It was expected some work would need
113 // to be done along the path ... but we know they are all empty as they were just
114 // created. We could tag them all as already scanned.
115 DirState *last_existing_ds = nullptr;
116 DirState *ds = m_fs_state.find_dirstate_for_lfn(lfn, &last_existing_ds);
117 if (ds->m_scanned)
118 return;
119
120 size_t pos = lfn.find_last_of("/");
121 std::string dir = (pos == std::string::npos) ? "" : lfn.substr(0, pos);
122
123 XrdOssDF *dhp = m_oss.newDir(trc_pfx);
124 bool opened = (dhp->Opendir(dir.c_str(), fst.default_env()) == XrdOssOK);
125 if (opened)
126 {
127 fst.slurp_dir_ll(*dhp, ds->m_depth, dir.c_str(), trc_pfx);
128
129 // XXXX clone of function below .... move somewhere? Esp. removal of non-paired files?
130 DirUsage &here = ds->m_here_usage;
131 for (auto it = fst.m_current_files.begin(); it != fst.m_current_files.end(); ++it)
132 {
133 if (it->second.has_data && it->second.has_cinfo) {
134 here.m_StBlocks += it->second.stat_data.st_blocks;
135 here.m_NFiles += 1;
136 }
137 }
138 }
139 delete dhp;
140 // Only a directory that was read counts as scanned, else its files are skipped.
141 if (opened)
142 ds->m_scanned = true;
143}
144
146{
147 dprintf("In scan_dir_and_recurse for '%s', size of dir_vec = %d, file_stat_map = %d\n",
148 fst.m_current_path.c_str(),
149 (int)fst.m_current_dirs.size(), (int)fst.m_current_files.size());
150
151 // Breadth first, accumulate into "here", unless it was already scanned via an
152 // OOB open file request.
153 if ( ! fst.m_dir_state->m_scanned)
154 {
155 DirUsage &here = fst.m_dir_state->m_here_usage;
156 for (auto it = fst.m_current_files.begin(); it != fst.m_current_files.end(); ++it)
157 {
158 dprintf("would be doing something with %s ... has_data=%d, has_cinfo=%d\n",
159 it->first.c_str(), it->second.has_data, it->second.has_cinfo);
160
161 // XXX Make some of these optional?
162 // Remove files that do not have both cinfo and data?
163 // Remove empty directories before even descending?
164 // Leave this for some consistency pass?
165 // Note that FsTraversal supports ignored paths ... some details (config, N2N) to be clarified.
166
167 if (it->second.has_data && it->second.has_cinfo) {
168 here.m_StBlocks += it->second.stat_data.st_blocks;
169 here.m_NFiles += 1;
170 }
171 }
172 fst.m_dir_state->m_scanned = true;
173 }
174
175 // Swap-out directories as recursion reuses the FsTraversal.
176 std::vector<std::string> dirs;
177 dirs.swap(fst.m_current_dirs);
178
179 check_inter_dir_scan_open_requests();
180
181 // Descend into sub-dirs, do not accumulate into recursive_subdir_usage yet. This is done
182 // in a separate pass to allow for proper accounting of files being opened during the initial scan.
183 for (auto &dname : dirs)
184 {
185 if (fst.cd_down(dname))
186 {
188 fst.cd_up();
189 }
190 // XXX else try to remove it?
191 }
192}
193
195{
196 // Called after PFC configuration is complete, but before full startup of the daemon.
197 // Base line usages are accumulated as part of the file-system, traversal.
198
200
201 DirState *root_ds = m_fs_state.get_root();
202 FsTraversal fst(m_oss);
203 fst.m_protected_top_dirs.insert("pfc-stats"); // XXXX This should come from config. Also: N2N?
204 // Keep servicing file-open requests while reading huge directories.
205 fst.m_readdir_hook = [this]() { check_inter_dir_scan_open_requests(); };
206
207 if ( ! fst.begin_traversal(root_ds, "/"))
208 return false;
209
210 // m_dir_scan_in_progress is initialized to true in ResourceMonitor.hh to avoid a race at startup.
211
213
214 fst.end_traversal();
215
216 // We have all directories scanned, available in DirState tree, let all remaining files go
217 // and then we shall do the upward propagation of usages.
218 {
219 XrdSysMutexHelper _lock(m_dir_scan_mutex);
220 m_dir_scan_in_progress = false;
221 }
222 // m_dir_scan_open_requests should now be final, ie, no new entries will be added.
223 while ( ! m_dir_scan_open_requests.empty())
224 {
225 LfnCondRecord &lcr = m_dir_scan_open_requests.front();
226 lcr.f_cond.Lock();
227 lcr.f_checked = true;
228 lcr.f_cond.Signal();
229 lcr.f_cond.UnLock();
230
231 m_dir_scan_open_requests.pop_front();
232 }
233
234 // Do upward propagation of usages.
236 m_current_usage_in_st_blocks = root_ds->m_here_usage.m_StBlocks +
239
240 return true;
241}
242
243//------------------------------------------------------------------------------
244// Processing of queues
245//------------------------------------------------------------------------------
246
248{
249 static const char *trc_pfx = "process_queues() ";
250
251 // Assure that we pick up only entries that are present now.
252 // We really want all open records to be processed before file-stats updates
253 // and all those before the close records.
254 // Purges are sort of tangential as they really just modify bytes / number
255 // of files in a directory and do not deal with any persistent file id tokens.
256
257 int n_records = 0;
258 {
259 XrdSysMutexHelper _lock(&m_queue_mutex);
260 n_records += m_file_open_q.swap_queues();
261 n_records += m_file_update_stats_q.swap_queues();
262 n_records += m_file_close_q.swap_queues();
263 n_records += m_file_purge_q1.swap_queues();
264 n_records += m_file_purge_q2.swap_queues();
265 n_records += m_file_purge_q3.swap_queues();
266 ++m_queue_swap_u1;
267 }
268
269 for (auto &i : m_file_open_q.read_queue())
270 {
271 // i.id: LFN, i.record: OpenRecord
272 AccessToken &at = token(i.id);
273 dprintf("process file open for token %d, time %ld -- %s\n",
274 i.id, i.record.m_open_time, at.m_filename.c_str());
275
276 // Resolve fname into DirState.
277 // We could clear the filename after this ... or keep it, should we need it later on.
278 // For now it is just used for debug printouts.
279 DirState *last_existing_ds = nullptr;
280 DirState *ds = m_fs_state.find_dirstate_for_lfn(at.m_filename, &last_existing_ds);
281 at.m_dir_state = ds;
282 note_file_open(ds, last_existing_ds, i.record.m_existing_file);
283
284 ds->m_here_usage.m_LastOpenTime = i.record.m_open_time;
285 }
286
287 for (auto &i : m_file_update_stats_q.read_queue())
288 {
289 // i.id: token, i.record: Stats
290 AccessToken &at = token(i.id);
291 // Stats
292 DirState *ds = at.m_dir_state;
293 dprintf("process file update for token %d, %p -- %s\n",
294 i.id, ds, at.m_filename.c_str());
295
296 ds->m_here_stats.AddUp(i.record);
297 m_current_usage_in_st_blocks += i.record.m_StBlocksAdded;
298 }
299
300 for (auto &i : m_file_close_q.read_queue())
301 {
302 // i.id: token, i.record: CloseRecord
303 AccessToken &at = token(i.id);
304 dprintf("process file close for token %d, time %ld -- %s\n",
305 i.id, i.record.m_close_time, at.m_filename.c_str());
306
307 DirState *ds = at.m_dir_state;
309 ds->m_here_usage.m_LastCloseTime = i.record.m_close_time;
310
311 }
312 { // Under the lock: clear() writes fields register_file_update_stats() uses.
313 XrdSysMutexHelper _lock(&m_queue_mutex);
314 for (auto &i : m_file_close_q.read_queue())
315 m_access_tokens.release(i.id);
316 }
317
318 for (auto &i : m_file_purge_q1.read_queue())
319 {
320 // i.id: DirState*, i.record: PurgeRecord
321 DirState *ds = i.id;
322 ds->m_here_stats.m_StBlocksRemoved += i.record.m_size_in_st_blocks;
323 ds->m_here_stats.m_NFilesRemoved += i.record.m_n_files;
324 m_current_usage_in_st_blocks -= i.record.m_size_in_st_blocks;
325 }
326 for (auto &i : m_file_purge_q2.read_queue())
327 {
328 // i.id: directory-path, i.record: PurgeRecord
329 DirState *ds = m_fs_state.get_root()->find_path(i.id, -1, false, false);
330 if ( ! ds) {
331 TRACE(Error, trc_pfx << "DirState not found for directory path '" << i.id << "'.");
332 // find_path can return the last dir found ... but this clearly isn't a valid purge record.
333 continue;
334 }
335 ds->m_here_stats.m_StBlocksRemoved += i.record.m_size_in_st_blocks;
336 ds->m_here_stats.m_NFilesRemoved += i.record.m_n_files;
337 m_current_usage_in_st_blocks -= i.record.m_size_in_st_blocks;
338 }
339 for (auto &i : m_file_purge_q3.read_queue())
340 {
341 // i.id: LFN, i.record: size of file in st_blocks
342 DirState *ds = m_fs_state.get_root()->find_path(i.id, -1, true, false);
343 if ( ! ds) {
344 TRACE(Error, trc_pfx << "DirState not found for LFN path '" << i.id << "'.");
345 continue;
346 }
347 ds->m_here_stats.m_StBlocksRemoved += i.record;
349 m_current_usage_in_st_blocks -= i.record;
350 }
351
352 // Read queues / vectors are cleared at swap time.
353 // We might consider reducing their capacity by half if, say, their usage is below 25%.
354
355 return n_records;
356}
357
358//------------------------------------------------------------------------------
359// Heart beat
360//------------------------------------------------------------------------------
361
363{
364 static const char *tpfx = "heart_beat() ";
365
366 const Configuration &conf = Cache::Conf();
367
368 const int s_queue_proc_interval = 10;
369 const int s_sshot_report_interval = conf.m_dirStatsInterval; // one of 60, 120, 300,
370 // 600, 900, 1200, 1800, 3600 s
371 const int s_purge_check_interval = 60;
372 const int s_purge_report_interval = conf.m_purgeInterval;
373 const int s_purge_cold_files_interval = conf.m_purgeInterval * conf.m_purgeAgeBasedPeriod;
374
375 // initial scan performed as part of config
376
377 time_t now = time(0);
378 time_t next_queue_proc_time = now + s_queue_proc_interval;
379 time_t next_sshot_report_time = (now / 60) * 60 + 60; // at next full minute
380 time_t next_purge_check_time = now + s_purge_check_interval;
381 time_t next_purge_report_time = now + s_purge_report_interval;
382 time_t next_purge_cold_files_time = now + s_purge_cold_files_interval;
383
384 long long n_reported_token_failures = 0;
385
386 while (true)
387 {
388 time_t start = time(0);
389 time_t next_event = std::min({ next_queue_proc_time, next_sshot_report_time,
390 next_purge_check_time, next_purge_report_time, next_purge_cold_files_time });
391
392 if (next_event > start)
393 {
394 unsigned int t_sleep = next_event - start;
395 TRACE(Dump, tpfx << "sleeping for " << t_sleep << " seconds until the next beat.");
396 sleep(t_sleep);
397 }
398
399 // Check if purge has been running and has completed yet.
400 // For now this is only used to prevent removal of empty leaf directories
401 // during stat propagation so we do not need to wait for the condition in
402 // the above sleep.
407 }
408 }
409
410 time_t queue_swap_time = time(0);
411
412 // Always process the queues.
413 int n_processed = process_queues();
414 next_queue_proc_time = queue_swap_time + s_queue_proc_interval;
415 TRACE(Dump, tpfx << "process_queues -- n_records=" << n_processed);
416
417 { // Access-token space exhausted -- should be unreachable, so say so loudly.
418 long long taf;
419 { XrdSysMutexHelper _lock(&m_queue_mutex); taf = m_token_alloc_failures; }
420 if (taf > n_reported_token_failures) {
421 TRACE(Error, tpfx << "access-token space exhausted "
422 << taf - n_reported_token_failures << " time(s) (total " << taf
423 << "); those files are open but not accounted for. Raise MaxChunks "
424 "on ResourceMonitor::m_access_tokens.");
425 n_reported_token_failures = taf;
426 }
427 }
428
429 // Always update basic info on m_fs_state (space, usage, file_usage).
431
432 now = time(0);
433
434 // Make planning for fs_state_update, sshot dump and purge task.
435 // Second two require the first, so figure out what is going to happen.
436 bool do_sshot_report = next_sshot_report_time <= now;
437 bool do_purge_check = next_purge_check_time <= now;
438 bool do_purge_report = next_purge_report_time <= now;
439 bool do_purge_cold_files = next_purge_cold_files_time <= now;
440
441 // Update stats in usages if any secondary activity will happen.
442 if (do_sshot_report || do_purge_check || do_purge_report || do_purge_cold_files)
443 {
444 unlink_func unlink_foo = [&](const std::string &dp)->int {
445 int ret = m_oss.Unlink(dp.c_str());
446 if (ret != 0) {
447 TRACE(Info, tpfx << "Empty dir unlink error: " << ret << " at " << dp);
448 } else {
449 TRACE(Debug, tpfx << "Empty dir unlink success: " << dp);
450 }
451 return ret;
452 };
453
454 // Potentially prune the empty leaf dirs even less frequently, once per hour, maybe?
455 bool purge_leaf_dirs = do_sshot_report && ! m_purge_task_active;
456 m_fs_state.update_stats_and_usages(queue_swap_time, purge_leaf_dirs, unlink_foo);
457
458 // This reporting into log/stdout is to be removed.
459 // Meaning of conf.is_dir_stat_reporting_on() etc is to be clarified / improved.
460 if (do_sshot_report && conf.is_dir_stat_reporting_on())
461 {
462 const int store_depth = conf.m_dirStatsStoreDepth;
463 #ifdef RM_DEBUG
464 const DirState &root_ds = *m_fs_state.get_root();
465 dprintf("Snapshot n_dirs=%d, total n_dirs=%d\n", root_ds.count_dirs_to_level(store_depth),
467 #endif
468 m_fs_state.dump_recursively(store_depth);
469 }
470
471 m_fs_state.reset_stats(queue_swap_time);
472 }
473
474 if (do_sshot_report)
475 {
476 // Sshot reports are equidistant, at "full" reporting interval.
477 next_sshot_report_time = ((now + 1) / s_sshot_report_interval) * s_sshot_report_interval + s_sshot_report_interval;
478
479 // This should dump out binary snapshot into /pfc-stats/, if so configured.
480
481 // json dump to std::out for debug purpose
482 DataFsSnapshot ss(m_fs_state, m_fs_state.m_sshot_stats_reset_time);
483 const DirState &root_ds = *m_fs_state.get_root();
484 const int store_depth = conf.m_dirStatsStoreDepth;
485 const int n_sshot_dirs = root_ds.count_dirs_to_level(store_depth);
486 ss.m_dir_states.reserve(n_sshot_dirs);
487 ss.m_dir_states.emplace_back( DirStateElement(root_ds, -1) );
488 fill_sshot_vec_children(root_ds, 0, ss.m_dir_states, store_depth);
489
490 // This should really be export to a file (preferably binary, but then bin->json command is needed, too).
491 // ss.dump();
492
493 const char* dumpfile = "/pfc-stats/DirStat.json";
494 ss.write_json_file(dumpfile, m_oss, false);
495 m_fs_state.reset_sshot_stats(queue_swap_time);
496 }
497
498 if (do_purge_check || do_purge_report || do_purge_cold_files)
499 {
500 perform_purge_check(do_purge_cold_files, do_purge_report ? TRACE_Info : TRACE_Debug);
501
502 next_purge_check_time = now + s_purge_check_interval;
503 if (do_purge_report) next_purge_report_time = now + s_purge_report_interval;
504 if (do_purge_cold_files) next_purge_cold_files_time = now + s_purge_cold_files_interval;
505 }
506
507 } // end while forever
508}
509
510//------------------------------------------------------------------------------
511// DirState export helpers
512//------------------------------------------------------------------------------
513
515 int parent_idx,
516 std::vector<DirStateElement> &vec,
517 int max_depth)
518{
519 const int pos0 = (int) vec.size();
520 const int n_children = (int) parent_ds.m_subdirs.size();
521
522 // maxdepth 0 is the root alone: a node at max_depth is emitted, its children are not.
523 if (parent_ds.m_depth >= max_depth)
524 {
525 // Truncated here: keep the daughter range at (-1, -1). No children: empty range.
526 if (n_children == 0) {
527 DirStateElement &dse = vec[parent_idx];
528 dse.m_daughters_begin = dse.m_daughters_end = pos0;
529 }
530 return;
531 }
532
533 { // scoped: the emplace_back loop below can reallocate vec
534 DirStateElement &parent_dse = vec[parent_idx];
535 parent_dse.m_daughters_begin = pos0;
536 parent_dse.m_daughters_end = pos0 + n_children;
537 }
538
539 if (n_children == 0) return;
540
541 for (auto const & [name, child] : parent_ds.m_subdirs)
542 {
543 vec.emplace_back( DirStateElement(child, parent_idx) );
544 }
545
546 int pos = pos0;
547 for (auto const & [name, child] : parent_ds.m_subdirs)
548 {
549 fill_sshot_vec_children(child, pos, vec, max_depth);
550 ++pos;
551 }
552}
553
555 int parent_idx,
556 std::vector<DirPurgeElement> &vec,
557 int max_depth)
558{
559 const int pos0 = (int) vec.size();
560 const int n_children = (int) parent_ds.m_subdirs.size();
561
562 // maxdepth 0 is the root alone: a node at max_depth is emitted, its children are not.
563 if (parent_ds.m_depth >= max_depth)
564 {
565 // Truncated here: keep the daughter range at (-1, -1). No children: empty range.
566 if (n_children == 0) {
567 DirPurgeElement &dse = vec[parent_idx];
568 dse.m_daughters_begin = dse.m_daughters_end = pos0;
569 }
570 return;
571 }
572
573 { // scoped: the emplace_back loop below can reallocate vec
574 DirPurgeElement &parent_dse = vec[parent_idx];
575 parent_dse.m_daughters_begin = pos0;
576 parent_dse.m_daughters_end = pos0 + n_children;
577 }
578
579 if (n_children == 0) return;
580
581 for (auto const & [name, child] : parent_ds.m_subdirs)
582 {
583 vec.emplace_back( DirPurgeElement(child, child.m_here_usage, child.m_recursive_subdir_usage, parent_idx) );
584 }
585
586 int pos = pos0;
587 for (auto const & [name, child] : parent_ds.m_subdirs)
588 {
589 fill_pshot_vec_children(child, pos, vec, max_depth);
590 ++pos;
591 }
592}
593
594//------------------------------------------------------------------------------
595// Purge helpers, drivers, etc.
596//------------------------------------------------------------------------------
597
599{
600 static const char *trc_pfx = "update_vs_and_file_usage_info() ";
601
602 const auto &conf = Cache::Conf();
603 XrdOssVSInfo vsi;
604
605 // StatVS error (after it succeeded in config) implies a memory corruption (according to Mr. H).
606 if (m_oss.StatVS(&vsi, conf.m_data_space.c_str(), 1) < 0) {
607 TRACE(Error, trc_pfx << "can't get StatVS for oss space '" << conf.m_data_space << "'. This is a fatal error.");
608 _exit(1);
609 }
610 m_fs_state.m_disk_total = vsi.Total;
611 m_fs_state.m_disk_used = vsi.Total - vsi.Free;
612 m_fs_state.m_file_usage = 512ll * m_current_usage_in_st_blocks;
613 if (m_oss.StatVS(&vsi, conf.m_meta_space.c_str(), 1) < 0) {
614 TRACE(Error, trc_pfx << "can't get StatVS for oss space '" << conf.m_meta_space << "'. This is a fatal error.");
615 _exit(1);
616 }
617 m_fs_state.m_meta_total = vsi.Total;
618 m_fs_state.m_meta_used = vsi.Total - vsi.Free;
619}
620
621long long ResourceMonitor::get_file_usage_bytes_to_remove(const DataFsPurgeshot &ps, long long write_estimate, int tl)
622{
623 // short names from config values
624 const Configuration &conf = Cache::Conf();
625 long long f0 = conf.m_fileUsageBaseline;
626 long long f1 = conf.m_fileUsageNominal;
627 long long f2 = conf.m_fileUsageMax;
628 long long w1 = conf.m_diskUsageLWM;
629 long long w2 = conf.m_diskUsageHWM;
630
631 // get usage from purge snapshot
632 long long T = ps.m_disk_total;
633 long long x = ps.m_file_usage;
634 long long u = ps.m_disk_used;
635
636 // get file usage increase from the previous time interval check
637 long long delta = write_estimate;
638 TRACE_INT(tl, "file usage increased since the previous purge interval in bytes: " << delta );
639
640 long long bytes_to_remove = 0;
641
642 // helper lambda function
643 auto clamp = [&x, &bytes_to_remove](long long lowval, long long highval)
644 {
645 long long val = x;
646 long long newval = val - bytes_to_remove;
647
648 // removed too much
649 if (newval < lowval)
650 {
651 return lowval - val;
652 }
653
654 // removed too little
655 if (newval > highval)
656 {
657 return val - highval;
658 }
659 // keep the original value
660 return bytes_to_remove;
661 };
662
663 // under file quota, nothing to do
664 if (x < f0)
665 return 0;
666
667 // total disk usage exceeds highWatermark
668 if (u >= w2)
669 {
670 TRACE_INT(tl, "Disk usage: " << ps.m_disk_used << " exceed highWatermark " << conf.m_diskUsageHWM);
671 float frac_u = static_cast<float>(u - w2) / (T - w2);
672 float frac_x = static_cast<float>(x - f0) / (f1 - f0);
673
674 if (w2 == T)
675 {
676 bytes_to_remove = u -w1;
677 }
678 else
679 {
680 if (frac_x > frac_u)
681 {
682 // the cache is the reason for going out of w2 range
683 bytes_to_remove = (frac_x - frac_u) * (f1 - f0);
684 bytes_to_remove += delta;
685 bytes_to_remove = clamp(f0, f1);
686 }
687 else
688 {
689 // someone else is filling disk space, go to f1
690 bytes_to_remove = clamp(f0, f2);
691 }
692 return bytes_to_remove;
693 }
694 }
695
696 // file quota and total disk usage is within normal range, check if this space usage is
697 // proportinal to disk usage and correct it
698 if (u > w1 && x > f1)
699 {
700 float frac_u = static_cast<float>(u - w1) / (w2 - w1);
701 float frac_x = static_cast<float>(x - f1) / (f2 - f1);
702 if (frac_x > frac_u)
703 {
704 TRACE_INT(tl, "Disproportional file quota usage comapared to disc usage (frac_x/frac_u) = " << frac_x << "/"<< frac_u);
705 bytes_to_remove = (frac_x - frac_u) * (f2 - f1);
706 bytes_to_remove += delta;
707 }
708
709 // check the new x val will not be below f0
710 bytes_to_remove = clamp(f0, f2);
711 return bytes_to_remove;
712 }
713
714 // final check: disk useage is lower that w1, check if exceed the max file usage f2
715 if (x > f2)
716 {
717 // drop usage to f2
718 // compare with global disk usage in the previous purge cycle (default 300s)
719 // check delta is not overflowing f2, else set numver of bytes to remove according remove to f0
720
721 TRACE_INT(tl, "File usage exceeds maxim file usage. Total disk usage is under lowWatermark. Clearing to low file usage.");
722 long long f2delta = std::max(f2 - delta, f0);
723 bytes_to_remove = clamp(f0, f2delta);
724 return bytes_to_remove;
725 }
726
727 return bytes_to_remove;
728}
729
730void ResourceMonitor::perform_purge_check(bool purge_cold_files, int tl)
731{
732 static const char *trc_pfx = "perform_purge_check() ";
733 const Configuration &conf = Cache::Conf();
734
735 std::unique_ptr<DataFsPurgeshot> psp( new DataFsPurgeshot(m_fs_state) );
736 DataFsPurgeshot &ps = *psp;
737
738 ps.m_file_usage = 512ll * m_current_usage_in_st_blocks;
739 // These are potentially wrong as cache might be writing over preallocated byte ranges.
741 // Can have another estimate based on eiter writes or st-blocks from purge-stats, once we have them.
742
743 TRACE_INT(tl, trc_pfx << "Purge check:");
744
745 ps.m_bytes_to_remove = 0;
746 if (conf.are_file_usage_limits_set())
747 {
748 ps.m_bytes_to_remove = get_file_usage_bytes_to_remove(ps, ps.m_estimated_writes_from_writeq, tl);
749 }
750 else
751 {
752 if (ps.m_disk_used > conf.m_diskUsageHWM)
753 {
754 TRACE_INT(tl, "Disk usage: " << ps.m_disk_used << " exceed highWatermark.");
756 }
757 }
758
760
761 // Purge precheck -- check if age-based purge is required
762 // We ignore uvkeep time, it requires reading of cinfo files and it is enforced in File::Open() anyway.
763
764 if (purge_cold_files && conf.is_age_based_purge_in_effect()) // || conf.is_uvkeep_purge_in_effect())
765 {
766 ps.m_age_based_purge = true;
767 }
768
769 TRACE_INT(tl, "\tbytes_to_remove = " << ps.m_bytes_to_remove << " B");
770 TRACE_INT(tl, "\tspace_based_purge = " << ps.m_space_based_purge);
771 TRACE_INT(tl, "\tage_based_purge = " << ps.m_age_based_purge);
772
773 bool periodic = Cache::GetInstance().GetPurgePin() ?
775
776 if ( ! ps.m_space_based_purge && ! ps.m_age_based_purge && !periodic ) {
777 TRACE(Info, trc_pfx << "purge not required.");
779 return;
780 }
782 TRACE(Warning, trc_pfx << "purge required but previous purge task is still active!");
783 return;
784 }
785
786 TRACE(Info, trc_pfx << "scheduling purge task.");
787
788 // At this point we have all the information: report, decide on action.
789 // There is still some missing infrastructure, especially as regards to purge-plugin:
790 // - at what point do we start bugging the pu-pin to start coughing up purge lists?
791 // - have a new parameter or just do it "one cycle before full"?
792 // - what if it doesn't -- when do we do the old-stlye scan & purge?
793 // - how do we do age-based purge and uvkeep purge?
794 // - they are really quite different -- and could run separately, registering
795 // files into a purge-candidate list. This has to be rechecked before the actual
796 // deletion -- eg, by comparing stat time of cinfo + doing the is-active / is-purge-protected.
797
798 const DirState &root_ds = *m_fs_state.get_root();
799
800 // Size from the tree itself; the usage counters can be off (#2808).
801 const int n_pshot_dirs = root_ds.count_dirs_to_level(9999);
802
803 // Report a counters-vs-tree mismatch when it appears or changes.
804 const int n_calc_dirs = 1 + root_ds.m_here_usage.m_NDirectories
806 dprintf("purge dir count recursive=%d vs from_usage=%d\n", n_pshot_dirs, n_calc_dirs);
807 const int discrepancy = n_calc_dirs - n_pshot_dirs;
808 if (discrepancy != m_dir_count_discrepancy)
809 {
810 if (discrepancy != 0) {
811 TRACE(Error, trc_pfx << "directory count mismatch: usage counters " << n_calc_dirs
812 << ", tree " << n_pshot_dirs << ", previous difference " << m_dir_count_discrepancy);
813 } else {
814 TRACE(Info, trc_pfx << "directory count consistent again: " << n_pshot_dirs);
815 }
816 m_dir_count_discrepancy = discrepancy;
817 }
818
819 ps.m_dir_vec.reserve(n_pshot_dirs);
820 ps.m_dir_vec.emplace_back( DirPurgeElement(root_ds, root_ds.m_here_usage, root_ds.m_recursive_subdir_usage, -1) );
821 fill_pshot_vec_children(root_ds, 0, ps.m_dir_vec, 9999);
822
823 m_purge_task_active = true;
824
825 struct PurgeDriverJob : public XrdJob
826 {
827 DataFsPurgeshot *m_purge_shot_ptr;
828
829 PurgeDriverJob(DataFsPurgeshot *psp) :
830 XrdJob("XrdPfc::ResourceMonitor::PurgeDriver"),
831 m_purge_shot_ptr(psp)
832 {}
833
834 void DoIt() override
835 {
836 Cache::ResMon().perform_purge_task(*m_purge_shot_ptr);
838
839 delete m_purge_shot_ptr;
840 delete this;
841 }
842 };
843
844 Cache::schedP->Schedule( new PurgeDriverJob(psp.release()) );
845}
846
847namespace XrdPfc
848{
850}
851
853{
854 // BEWARE: Runs in a dedicated thread - is only to communicate back to the
855 // hear_beat() / data structs via the purge queues and condition variable.
856
857 // const char *tpfx = "perform_purge_task ";
858
859 {
861 m_purge_task_start = time(0);
862 }
863
864 // For now, fall back to the old purge ... to be improved with:
865 // - new scan, following the DataFsPurgeshot;
866 // - usage of cinfo stat mtime for time of last access (touch already done at output);
867 // - use DirState* to report back purged files.
868 // Already changed to report back purged files --- but using the string / path variant.
869 OldStylePurgeDriver(ps); // In XrdPfcPurge.cc
870}
871
873{
874 // Separated out so the purge_task can exit without post-checks.
875
876 {
878 m_purge_task_end = time(0);
880 m_purge_task_cond.Signal();
881 }
883}
884
885//==============================================================================
886// Main thread function, do initial test, then enter heart_beat().
887//==============================================================================
888
890{
891 // setup for in-scan -- this is called from initial setup.
892 MutexHolder _lck(m_dir_scan_mutex);
893 m_dir_scan_in_progress = true;
894}
895
897{
898 const char *tpfx = "main_thread_function ";
899 {
900 time_t is_start = time(0);
901 m_fs_state.init_stat_reset_times(is_start);
902 TRACE(Info, tpfx << "Stating initial directory scan.");
903
904 if ( ! perform_initial_scan()) {
905 TRACE(Error, tpfx << "Initial directory scan has failed. This is a terminal error, aborting.")
906 _exit(1);
907 }
908 // Reset of m_dir_scan_in_progress is done in perform_initial_scan()
909
910 time_t is_duration = time(0) - is_start;
911 TRACE(Info, tpfx << "Initial directory scan complete, duration=" << is_duration <<"s");
912
913 // run first process queues
914 int n_proc_is = process_queues();
915 TRACE(Info, tpfx << "First process_queues finished, n_records=" << n_proc_is);
916
917 // shrink queues if scan time was longer than 30s.
918 if (is_duration > 30 || n_proc_is > 3000)
919 {
920 m_file_open_q.shrink_read_queue();
921 m_file_update_stats_q.shrink_read_queue();
922 m_file_close_q.shrink_read_queue();
923 m_file_purge_q1.shrink_read_queue();
924 m_file_purge_q2.shrink_read_queue();
925 m_file_purge_q3.shrink_read_queue();
926 }
927 }
928 heart_beat();
929}
930
931//==============================================================================
932// Old prototype from Cache / Purge, now to go into heart_beat() here, above.
933//==============================================================================
934
936{
937 // static const char *trc_pfx = "ResourceMonitorHeartBeat() ";
938
939 // Pause before initial run
940 sleep(1);
941
942 // XXXX Setup initial / constant stats (total RAM, total disk, ???)
943
946
947 S.Lock();
948
950
952
953 S.UnLock();
954
955 // XXXX Schedule initial disk scan, time it!
956 //
957 // TRACE(Info, trc_pfx << "scheduling intial disk scan.");
958 // schedP->Schedule( new ScanAndPurgeJob("XrdPfc::ScanAndPurge") );
959 //
960 // bool scan_and_purge_running = true;
961
962 // XXXX Could we really hold last-usage for all files in memory?
963
964 // XXXX Think how to handle disk-full, scan/purge not finishing:
965 // - start dropping things out of write queue, but only when RAM gets near full;
966 // - monitoring this then becomes a high-priority job, inner loop with sleep of,
967 // say, 5 or 10 seconds.
968
969 while (true)
970 {
971 time_t heartbeat_start = time(0);
972
973 // TRACE(Info, trc_pfx << "HeartBeat starting ...");
974
975 // if sumary monitoring configured, pupulate OucCacheStats:
976 S.Lock();
977
978 // - available / used disk space (files usage calculated elsewhere (maybe))
979
980 // - RAM usage
981 /* XXXX From Cache
982 { XrdSysMutexHelper lck(&m_RAM_mutex);
983 X.MemUsed = m_RAM_used;
984 X.MemWriteQ = m_RAM_write_queue;
985 }
986 */
987
988 // - files opened / closed etc
989
990 // do estimate of available space
991 S.UnLock();
992
993 // if needed, schedule purge in a different thread.
994 // purge is:
995 // - deep scan + gather FSPurgeState
996 // - actual purge
997 //
998 // this thread can continue running and, if needed, stop writing to disk
999 // if purge is taking too long.
1000
1001 // think how data is passed / synchronized between this and purge thread
1002
1003 // !!!! think how stat collection is done and propgated upwards;
1004 // until now it was done once per purge-interval.
1005 // now stats will be added up more often, but purge will be done
1006 // only occasionally.
1007 // also, do we report cumulative values or deltas? cumulative should
1008 // be easier and consistent with summary data.
1009 // still, some are state - like disk usage, num of files.
1010
1011 // Do we take care of directories that need to be newly added into DirState hierarchy?
1012 // I.e., when user creates new directories and these are covered by either full
1013 // spec or by root + depth declaration.
1014
1015 int heartbeat_duration = time(0) - heartbeat_start;
1016
1017 // TRACE(Info, trc_pfx << "HeartBeat finished, heartbeat_duration " << heartbeat_duration);
1018
1019 // int sleep_time = m_fs_state..m_purgeInterval - heartbeat_duration;
1020 int sleep_time = 60 - heartbeat_duration;
1021 if (sleep_time > 0)
1022 {
1023 sleep(sleep_time);
1024 }
1025 }
1026}
int DoIt(int argpnt, int argc, char **argv, bool singleshot)
static void child()
#define TRACE_Debug
#define TRACE_Info
#define XrdOssOK
Definition XrdOss.hh:50
#define dprintf(...)
void Proto_ResourceMonitorHeartBeat()
#define TRACE_INT(act, x)
bool Debug
if(ec< 0) ec
#define TRACE(act, x)
Definition XrdTrace.hh:63
virtual int Opendir(const char *path, XrdOucEnv &env)
Definition XrdOss.hh:79
long long Total
Definition XrdOssVS.hh:90
long long Free
Definition XrdOssVS.hh:91
struct XrdOucCacheStats::CacheStats X
XrdOucCacheStats Statistics
static const Configuration & Conf()
Definition XrdPfc.cc:134
static ResourceMonitor & ResMon()
Definition XrdPfc.cc:135
void ClearPurgeProtectedSet()
Definition XrdPfc.cc:691
static Cache & GetInstance()
Singleton access.
Definition XrdPfc.cc:132
XrdSysTrace * GetTrace() const
Definition XrdPfc.hh:297
static XrdScheduler * schedP
Definition XrdPfc.hh:304
long long WritesSinceLastCall()
Definition XrdPfc.cc:324
PurgePin * GetPurgePin() const
Definition XrdPfc.hh:286
void AddUp(const DirStats &s)
long long m_StBlocksRemoved
std::vector< std::string > m_current_dirs
std::function< void()> m_readdir_hook
void slurp_dir_ll(XrdOssDF &dh, int dir_level, const char *path, const char *trc_pfx)
bool begin_traversal(DirState *root, const char *root_path)
std::set< std::string > m_protected_top_dirs
bool cd_down(const std::string &dir_name)
std::map< std::string, FilePairStat > m_current_files
Status of cached file. Can be read from and written into a binary file.
Definition XrdPfcInfo.hh:41
virtual bool CallPeriodically()
void CrossCheckIfScanIsInProgress(const std::string &lfn, XrdSysCondVar &cond)
void perform_purge_check(bool purge_cold_files, int tl)
void fill_sshot_vec_children(const DirState &parent_ds, int parent_idx, std::vector< DirStateElement > &vec, int max_depth)
void perform_purge_task(DataFsPurgeshot &ps)
void scan_dir_and_recurse(FsTraversal &fst)
void fill_pshot_vec_children(const DirState &parent_ds, int parent_idx, std::vector< DirPurgeElement > &vec, int max_depth)
std::function< int(const std::string &)> unlink_func
void OldStylePurgeDriver(DataFsPurgeshot &ps)
void note_file_open(DirState *node, DirState *last_existing, bool existing_file)
Contains parameters configurable from the xrootd config file.
Definition XrdPfc.hh:64
long long m_RamAbsAvailable
available from configuration
Definition XrdPfc.hh:109
long long m_diskTotalSpace
total disk space on configured partition or oss space
Definition XrdPfc.hh:92
long long m_fileUsageMax
cache purge - files usage maximum
Definition XrdPfc.hh:97
long long m_fileUsageBaseline
cache purge - files usage baseline
Definition XrdPfc.hh:95
int m_dirStatsStoreDepth
maximum depth for statistics write out
Definition XrdPfc.hh:106
long long m_diskUsageHWM
cache purge - disk usage high water mark
Definition XrdPfc.hh:94
bool are_file_usage_limits_set() const
Definition XrdPfc.hh:67
long long m_fileUsageNominal
cache purge - files usage nominal
Definition XrdPfc.hh:96
int m_purgeAgeBasedPeriod
peform cold file / uvkeep purge every this many purge cycles
Definition XrdPfc.hh:100
long long m_diskUsageLWM
cache purge - disk usage low water mark
Definition XrdPfc.hh:93
int m_dirStatsInterval
time between resource monitor statistics dump in seconds
Definition XrdPfc.hh:105
bool is_age_based_purge_in_effect() const
Definition XrdPfc.hh:68
int m_purgeInterval
sleep interval between cache purges
Definition XrdPfc.hh:98
bool is_dir_stat_reporting_on() const
Definition XrdPfc.hh:70
std::vector< DirPurgeElement > m_dir_vec
void write_json_file(const std::string &fname, XrdOss &oss, bool include_preamble)
std::vector< DirStateElement > m_dir_states
DirUsage m_recursive_subdir_usage
int count_dirs_to_level(int max_depth) const
DirState * find_path(const std::string &path, int max_depth, bool parse_as_lfn, bool create_subdirs, DirState **last_existing_dir=nullptr)
void upward_propagate_initial_scan_usages()