XRootD
XrdPfcDirState.cc
Go to the documentation of this file.
1 #include "XrdPfcDirState.hh"
3 
4 #include <cstdlib>
5 #include <string>
6 
7 using namespace XrdPfc;
8 
9 //----------------------------------------------------------------------------
11 //----------------------------------------------------------------------------
12 DirState::DirState() : m_parent(0), m_depth(0)
13 {}
14 
15 //----------------------------------------------------------------------------
18 //----------------------------------------------------------------------------
20  m_parent(parent),
21  m_depth(m_parent->m_depth + 1)
22 {}
23 
24 //----------------------------------------------------------------------------
28 //----------------------------------------------------------------------------
29 DirState::DirState(DirState *parent, const std::string &dname) :
30  DirStateBase(dname),
31  m_parent(parent),
32  m_depth(m_parent->m_depth + 1)
33 {}
34 
35 //----------------------------------------------------------------------------
38 //----------------------------------------------------------------------------
39 DirState *DirState::create_child(const std::string &dir)
40 {
41  std::pair<DsMap_i, bool> ir = m_subdirs.insert(std::make_pair(dir, DirState(this, dir)));
42  return &ir.first->second;
43 }
44 
45 //----------------------------------------------------------------------------
48 //----------------------------------------------------------------------------
49 DirState *DirState::find_path_tok(PathTokenizer &pt, int pos, bool create_subdirs,
50  DirState **last_existing_dir)
51 {
52  if (pos == pt.get_n_dirs())
53  return this;
54 
55  DirState *ds = nullptr;
56 
57  DsMap_i i = m_subdirs.find(pt.m_dirs[pos]);
58 
59  if (i != m_subdirs.end())
60  {
61  ds = &i->second;
62  if (last_existing_dir)
63  *last_existing_dir = ds;
64  }
65  else if (create_subdirs)
66  {
67  ds = create_child(pt.m_dirs[pos]);
68  }
69 
70  if (ds)
71  return ds->find_path_tok(pt, pos + 1, create_subdirs, last_existing_dir);
72 
73  return nullptr;
74 }
75 
76 //----------------------------------------------------------------------------
82 DirState *DirState::find_path(const std::string &path, int max_depth, bool parse_as_lfn,
83  bool create_subdirs, DirState **last_existing_dir)
84 {
85  PathTokenizer pt(path, max_depth, parse_as_lfn);
86 
87  if (last_existing_dir)
88  *last_existing_dir = this;
89 
90  return find_path_tok(pt, 0, create_subdirs, last_existing_dir);
91 }
92 
93 //----------------------------------------------------------------------------
98 DirState *DirState::find_dir(const std::string &dir,
99  bool create_subdirs)
100 {
101  DsMap_i i = m_subdirs.find(dir);
102 
103  if (i != m_subdirs.end())
104  return &i->second;
105 
106  if (create_subdirs)
107  return create_child(dir);
108 
109  return nullptr;
110 }
111 
112 //----------------------------------------------------------------------------
116 int DirState::generate_dir_path(std::string &result)
117 {
118  int n_chars = 0;
119  if (m_parent) {
120  n_chars += m_parent->generate_dir_path(result) + 1 + m_dir_name.length();
121  result += '/';
122  result += m_dir_name;
123  }
124  return n_chars;
125 }
126 
127 //----------------------------------------------------------------------------
130 //----------------------------------------------------------------------------
132 {
133  DirUsage &here = m_here_usage;
135 
136  for (auto & [name, daughter] : m_subdirs)
137  {
138  daughter.upward_propagate_initial_scan_usages();
139 
140  DirUsage &dhere = daughter.m_here_usage;
141  DirUsage &dsubdirs = daughter.m_recursive_subdir_usage;
142 
143  here.m_NDirectories += 1;
144 
145  subdirs.m_StBlocks += dhere.m_StBlocks + dsubdirs.m_StBlocks;
146  subdirs.m_NFiles += dhere.m_NFiles + dsubdirs.m_NFiles;
147  subdirs.m_NDirectories += dhere.m_NDirectories + dsubdirs.m_NDirectories;
148 
149  // Carry the last-access times up, as update_stats_and_usages() does.
150  subdirs.update_last_times(dhere);
151  subdirs.update_last_times(dsubdirs);
152  }
153 }
154 
155 //----------------------------------------------------------------------------
159 
160 void DirState::update_stats_and_usages(std::vector<DirState*> *reap_candidates)
161 {
163  for (auto & [name, sub_ds] : m_subdirs)
164  {
165  sub_ds.update_stats_and_usages(reap_candidates);
166 
167  m_RecursiveStBlocksFuzz += sub_ds.m_StBlocksFuzz + sub_ds.m_RecursiveStBlocksFuzz;
168 
169  m_recursive_subdir_stats.AddUp(sub_ds.m_recursive_subdir_stats);
170  m_recursive_subdir_stats.AddUp(sub_ds.m_here_stats);
171  // nothing to do for m_here_stats.
172 
173  // Join timestamps from daugters and their sub-daugter trees.
174  m_recursive_subdir_usage.update_last_times(sub_ds.m_recursive_subdir_usage);
175  m_recursive_subdir_usage.update_last_times(sub_ds.m_here_usage);
176 
177  // The parent clears a child's deltas once absorbed; the root's are cleared by
178  // DataFsState::update_stats_and_usages().
179  sub_ds.m_here_stats.Reset();
180  sub_ds.m_recursive_subdir_stats.Reset();
181 
182  if (reap_candidates && sub_ds.is_empty_leaf())
183  reap_candidates->push_back(&sub_ds);
184  }
185 
186  // Apply current deltas to sshot_stats (and, if eventually needed, purge_stats).
189 
190  // Apply deltas / stats to usages.
193 
194  // Once we have m_report_stats and m_purge_stats, update them here.
195 }
196 
197 //----------------------------------------------------------------------------
200 
202 {
203  return m_subdirs.empty() &&
205  m_here_usage.m_NFilesOpen == 0 &&
207 }
208 
210 {
211  for (DsMap_i i = m_subdirs.begin(); i != m_subdirs.end(); ++i)
212  {
213  i->second.reset_sshot_stats();
214  }
216 }
217 
218 int DirState::count_dirs_to_level(int max_depth) const
219 {
220  int n_dirs = 1;
221  if (m_depth < max_depth)
222  {
223  for (auto & [name, ds] : m_subdirs)
224  {
225  n_dirs += ds.count_dirs_to_level(max_depth);
226  }
227  }
228  return n_dirs;
229 }
230 
231 //----------------------------------------------------------------------------
234 //----------------------------------------------------------------------------
235 void DirState::dump_recursively(const char *name, int max_depth) const
236 {
237  printf("%*d %s usage_here=%lld usage_sub=%lld usage_total=%lld num_ios=%d duration=%d b_hit=%lld b_miss=%lld b_byps=%lld b_wrtn=%lld\n",
238  2 + 2 * m_depth, m_depth, name,
241  // Since the last snapshot, this directory and below.
245 
246  if (m_depth < max_depth)
247  {
248  for (auto & [name, ds] : m_subdirs)
249  {
250  ds.dump_recursively(name.c_str(), max_depth);
251  }
252  }
253 }
254 
255 
256 //==============================================================================
257 // DataFsState
258 //==============================================================================
259 
260 void DataFsState::update_stats_and_usages(time_t last_update, bool collect_reap_candidates)
261 {
262  if (collect_reap_candidates)
263  m_reap_candidates.clear();
264  m_root.update_stats_and_usages(collect_reap_candidates ? &m_reap_candidates : nullptr);
267  m_usage_update_time = last_update;
268  m_stats_reset_time = last_update;
269 }
270 
271 //----------------------------------------------------------------------------
275 
277 {
278  long long total_residue = 0;
279  for (DirState *ds : m_reap_candidates)
280  {
281  // A pending close points at a DirState while m_NFilesOpen > 0.
282  if ( ! ds->is_empty_leaf())
283  continue;
284 
285  std::string dir_path;
286  dir_path.reserve(1024);
287  ds->generate_dir_path(dir_path);
288  if (unlink_foo(dir_path) != 0)
289  continue;
290 
291  // Return a leftover st_blocks through the recursive stats, where the ancestors
292  // counted it, rather than lose it with the node. Keep the correction, and the
293  // node's own fuzz, as the parent's.
294  DirState *parent = ds->m_parent;
295  long long residue = ds->m_here_usage.m_StBlocks + ds->m_recursive_subdir_usage.m_StBlocks;
296  parent->m_recursive_subdir_stats.m_StBlocksRemoved += residue;
297  parent->m_StBlocksFuzz += std::llabs(residue) + ds->m_StBlocksFuzz;
298  m_st_blocks_fuzz += std::llabs(residue);
299  total_residue += residue;
300 
301  ++parent->m_here_stats.m_NDirectoriesRemoved;
302  parent->m_subdirs.erase(parent->m_subdirs.find(ds->m_dir_name));
303  }
304  m_reap_candidates.clear();
305  return total_residue;
306 }
307 
308 void DataFsState::reset_sshot_stats(time_t last_update)
309 {
311  m_sshot_stats_reset_time = last_update;
312 }
313 
314 void DataFsState::dump_recursively(int max_depth) const
315 {
316  if (max_depth < 0)
317  max_depth = 4096;
318 
319  printf("DataFsState::dump_recursively delta_t = %lld, max_dump_depth = %d\n",
320  (long long)(m_usage_update_time - m_sshot_stats_reset_time), max_depth);
321 
322  m_root.dump_recursively("root", max_depth);
323 }
static void parent()
void AddUp(const DirStats &s)
Definition: XrdPfcStats.hh:192
long long m_BytesMissed
number of bytes served from remote and cached
Definition: XrdPfcStats.hh:40
long long m_BytesBypassed
number of bytes served directly through XrdCl
Definition: XrdPfcStats.hh:41
int m_Duration
total duration of all IOs attached
Definition: XrdPfcStats.hh:38
int m_NumIos
number of IO objects attached during this access
Definition: XrdPfcStats.hh:37
long long m_BytesHit
number of bytes served from disk
Definition: XrdPfcStats.hh:39
long long m_BytesWritten
number of bytes written to disk
Definition: XrdPfcStats.hh:42
Definition: XrdPfc.hh:44
std::function< int(const std::string &)> unlink_func
std::vector< DirState * > m_reap_candidates
void dump_recursively(int max_depth) const
void update_stats_and_usages(time_t last_update, bool collect_reap_candidates)
long long reap_empty_leaf_dirs(unlink_func unlink_foo)
void reset_sshot_stats(time_t last_update)
DirState * m_parent
DirUsage m_recursive_subdir_usage
DsMap_t::iterator DsMap_i
int count_dirs_to_level(int max_depth) const
DirState * create_child(const std::string &dir)
DirStats m_recursive_subdir_stats
DirState * find_dir(const std::string &dir, bool create_subdirs)
long long m_RecursiveStBlocksFuzz
sum over the subdirs, set by propagation
int generate_dir_path(std::string &result)
DirState()
Constructor.
DirState * find_path_tok(PathTokenizer &pt, int pos, bool create_subdirs, DirState **last_existing_dir=nullptr)
DirState * find_path(const std::string &path, int max_depth, bool parse_as_lfn, bool create_subdirs, DirState **last_existing_dir=nullptr)
void update_stats_and_usages(std::vector< DirState * > *reap_candidates=nullptr)
void dump_recursively(const char *name, int max_depth) const
void upward_propagate_initial_scan_usages()
DirStats m_sshot_stats
bool is_empty_leaf() const
void update_from_stats(const DirStats &s)
void update_last_times(const DirUsage &u)
std::vector< const char * > m_dirs