XRootD
XrdXrootdMonFile.cc
Go to the documentation of this file.
1 /******************************************************************************/
2 /* */
3 /* X r d X r o o t d M o n F i l e . c c */
4 /* */
5 /* (c) 2012 by the Board of Trustees of the Leland Stanford, Jr., University */
6 /* All Rights Reserved */
7 /* Produced by Andrew Hanushevsky for Stanford University under contract */
8 /* DE-AC02-76-SFO0515 with the Department of Energy */
9 /* */
10 /* This file is part of the XRootD software suite. */
11 /* */
12 /* XRootD is free software: you can redistribute it and/or modify it under */
13 /* the terms of the GNU Lesser General Public License as published by the */
14 /* Free Software Foundation, either version 3 of the License, or (at your */
15 /* option) any later version. */
16 /* */
17 /* XRootD is distributed in the hope that it will be useful, but WITHOUT */
18 /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */
19 /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */
20 /* License for more details. */
21 /* */
22 /* You should have received a copy of the GNU Lesser General Public License */
23 /* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */
24 /* COPYING (GPL license). If not, see <http://www.gnu.org/licenses/>. */
25 /* */
26 /* The copyright holder's institutional names and contributor's names may not */
27 /* be used to endorse or promote products derived from this software without */
28 /* specific prior written permission of the institution or contributor. */
29 /******************************************************************************/
30 
31 #include <algorithm>
32 #include <cstring>
33 
34 #include "Xrd/XrdScheduler.hh"
35 
36 #include "XrdSys/XrdSysError.hh"
37 #include "XrdSys/XrdSysPlatform.hh"
38 
41 
42 /******************************************************************************/
43 /* G l o b a l s */
44 /******************************************************************************/
45 
46 namespace XrdXrootdMonInfo
47 {
50 extern long long mySID;
51 extern int32_t startTime;
52 }
53 
54 /******************************************************************************/
55 /* C o n s t a n t s */
56 /******************************************************************************/
57 
58 namespace
59 {
60 // The fixed part of an open record: everything except the user dictid and the lfn.
61 //
62 const int minRecSz = sizeof(XrdXrootdMonFileOPN) - sizeof(XrdXrootdMonFileLFN);
63 
64 // MonEnt is a short, so -1 and the cookie of the last slot (0xffff) are the
65 // same value. Open() therefore never uses the last map. A file that has an
66 // f-stream open record but no slot gets the cookie noSlot from that map.
67 // Its value is not equal to -1, so the file still gets its close record.
68 //
69 const int mapUse = XrdXrootdMonFMap::mapNum - 1;
70 const short noSlot = static_cast<short>(mapUse << XrdXrootdMonFMap::fmShft);
71 }
72 
73 /******************************************************************************/
74 /* S t a t i c M e m b e r s */
75 /******************************************************************************/
76 
77 XrdSysMutex XrdXrootdMonFile::bfMutex;
78 XrdSysMutex XrdXrootdMonFile::fmMutex;
79 XrdXrootdMonFMap XrdXrootdMonFile::fmMap[XrdXrootdMonFMap::mapNum];
80 short XrdXrootdMonFile::fmUse[XrdXrootdMonFMap::mapNum] = {0};
81 char *XrdXrootdMonFile::repBuff = 0;
82 XrdXrootdMonHeader *XrdXrootdMonFile::repHdr = 0;
83 XrdXrootdMonFileTOD *XrdXrootdMonFile::repTOD = 0;
84 char *XrdXrootdMonFile::repNext = 0;
85 char *XrdXrootdMonFile::repFirst = 0;
86 char *XrdXrootdMonFile::repLast = 0;
87 int XrdXrootdMonFile::totRecs = 0;
88 int XrdXrootdMonFile::xfrRecs = 0;
89 int XrdXrootdMonFile::repSize = 0;
90 int XrdXrootdMonFile::repTime = 0;
91 int XrdXrootdMonFile::maxText = 0;
92 int XrdXrootdMonFile::fmHWM =-1;
93 int XrdXrootdMonFile::crecSize = 0;
94 int XrdXrootdMonFile::xfrCnt = 0;
95 int XrdXrootdMonFile::fBsz = 65472;
96 int XrdXrootdMonFile::xfrRem = 0;
97 XrdXrootdMonFileXFR XrdXrootdMonFile::xfrRec;
98 short XrdXrootdMonFile::crecNLen = 0;
99 short XrdXrootdMonFile::trecNLen = 0;
100 char XrdXrootdMonFile::fsLFN = 0;
101 char XrdXrootdMonFile::fsLVL = 0;
102 char XrdXrootdMonFile::fsOPS = 0;
103 char XrdXrootdMonFile::fsSSQ = 0;
104 char XrdXrootdMonFile::fsXFR = 0;
105 char XrdXrootdMonFile::crecFlag = 0;
106 
107 /******************************************************************************/
108 /* C l o s e */
109 /******************************************************************************/
110 
112 {
113  XrdXrootdMonFileCLS cRec;
114  char *cP;
115  int iEnt, iMap, iSlot;
116 
117 // If this object was registered for I/O reporting, deregister it. Then mark
118 // it as closed on the f-stream.
119 //
120  if (fsP->MonEnt != -1 && fsP->MonEnt != noSlot)
121  {iEnt = fsP->MonEnt & 0xffff;
122  iMap = iEnt >> XrdXrootdMonFMap::fmShft;
123  iSlot = iEnt & XrdXrootdMonFMap::fmMask;
124  fmMutex.Lock();
125  if (fmMap[iMap].Free(iSlot)) fmUse[iMap]--;
126  if (iMap == fmHWM) while(fmHWM >= 0 && !fmUse[fmHWM]) fmHWM--;
127  fmMutex.UnLock();
128  }
129  fsP->MonEnt = -1;
130 
131 // Insert a close record header (mostly precomputed)
132 //
134  cRec.Hdr.recFlag = crecFlag;
135  if (isDisc) cRec.Hdr.recFlag |= XrdXrootdMonFileHdr::forced;
136  cRec.Hdr.recSize = crecNLen;
137  cRec.Hdr.fileID = fsP->FileID;
138 
139 // Insert the I/O bytes
140 //
141  cRec.Xfr.read = htonll(fsP->xfr.read);
142  cRec.Xfr.readv = htonll(fsP->xfr.readv);
143  cRec.Xfr.write = htonll(fsP->xfr.write);
144 
145 // Insert ops if so wanted
146 //
147  if (fsOPS)
148  {cRec.Ops.read = htonl (fsP->ops.read);
149  if (fsP->ops.read)
150  {cRec.Ops.rdMin = htonl (fsP->ops.rdMin);
151  cRec.Ops.rdMax = htonl (fsP->ops.rdMax);
152  } else {
153  cRec.Ops.rdMin = cRec.Ops.rdMax = 0;
154  }
155  cRec.Ops.readv = htonl (fsP->ops.readv);
156  cRec.Ops.rsegs = htonll(fsP->ops.rsegs);
157  if (fsP->ops.readv)
158  {cRec.Ops.rsMin = htons (fsP->ops.rsMin);
159  cRec.Ops.rsMax = htons (fsP->ops.rsMax);
160  cRec.Ops.rvMin = htonl (fsP->ops.rvMin);
161  cRec.Ops.rvMax = htonl (fsP->ops.rvMax);
162  } else {
163  cRec.Ops.rsMin = cRec.Ops.rsMax = 0;
164  cRec.Ops.rvMin = cRec.Ops.rvMax = 0;
165  }
166  cRec.Ops.write = htonl (fsP->ops.write);
167  if (fsP->ops.write)
168  {cRec.Ops.wrMin = htonl (fsP->ops.wrMin);
169  cRec.Ops.wrMax = htonl (fsP->ops.wrMax);
170  } else {
171  cRec.Ops.wrMin = cRec.Ops.wrMax = 0;
172  }
173  }
174 
175 // Record sum of squares if so needed
176 //
177  if (fsSSQ)
178  {XrdXrootdMonDouble xval;
179  xval.dreal = fsP->ssq.read;
180  cRec.Ssq.read.dlong = htonll(xval.dlong);
181  xval.dreal = fsP->ssq.readv;
182  cRec.Ssq.readv.dlong = htonll(xval.dlong);
183  xval.dreal = fsP->ssq.rsegs;
184  cRec.Ssq.rsegs.dlong = htonll(xval.dlong);
185  xval.dreal = fsP->ssq.write;
186  cRec.Ssq.write.dlong = htonll(xval.dlong);
187  }
188 
189 // Get a pointer to the next slot (the buffer gets locked)
190 //
191  cP = GetSlot(crecSize);
192  memcpy(cP, &cRec, crecSize);
193  bfMutex.UnLock();
194 }
195 
196 /******************************************************************************/
197 /* D e f a u l t s */
198 /******************************************************************************/
199 
200 void XrdXrootdMonFile::Defaults(int intv, int opts, int xfrcnt, int fbsz)
201 {
202 
203 // Set the reporting interval and I/O counter
204 //
205  repTime = intv;
206  xfrCnt = xfrcnt;
207  xfrRem = xfrcnt;
208 
209 // Set the buffer size. The configuration accepts 1024 to 65535 bytes, but the
210 // buffer goes out as one datagram, so limit it to the largest UDP payload.
211 //
212  fBsz = (fbsz <= 0 ? 65472
213  : std::clamp(fbsz, 1024, XrdXrootdMonitor::maxDgram));
214 
215 // Expand out the options
216 //
217  fsXFR = (opts & XROOTD_MON_FSXFR) != 0;
218  fsLFN = (opts & XROOTD_MON_FSLFN) != 0;
219  fsOPS = (opts & (XROOTD_MON_FSOPS | XROOTD_MON_FSSSQ)) != 0;
220  fsSSQ = (opts & XROOTD_MON_FSSSQ) != 0;
221 
222 // Set monitoring level
223 //
224  if (fsSSQ) fsLVL = XrdXrootdFileStats::monSsq;
225  else if (fsOPS) fsLVL = XrdXrootdFileStats::monOps;
226  else if (intv) fsLVL = XrdXrootdFileStats::monOn;
227  else fsLVL = XrdXrootdFileStats::monOff;
228 }
229 
230 /******************************************************************************/
231 /* D i s c */
232 /******************************************************************************/
233 
234 void XrdXrootdMonFile::Disc(unsigned int usrID)
235 {
236  static short drecSize = htons(sizeof(XrdXrootdMonFileDSC));
238 
239 // Get a pointer to the next slot (the buffer gets locked)
240 //
241  dP = (XrdXrootdMonFileDSC *)GetSlot(sizeof(XrdXrootdMonFileDSC));
242 
243 // Fill out the record. It's pretty simple
244 //
246  dP->Hdr.recFlag = 0;
247  dP->Hdr.recSize = drecSize;
248  dP->Hdr.userID = usrID;
249  bfMutex.UnLock();
250 }
251 
252 /******************************************************************************/
253 /* D o I t */
254 /******************************************************************************/
255 
257 {
258 
259 // First check if we need to report all the I/O stats
260 //
261  xfrRem--;
262  if (!xfrRem) DoXFR();
263 
264 // Check if we should flush the buffer
265 //
266  bfMutex.Lock();
267  if (repNext) Flush();
268  bfMutex.UnLock();
269 
270 // Reschedule ourselves
271 //
272  XrdXrootdMonInfo::Sched->Schedule((XrdJob *)this, time(0)+repTime);
273 }
274 
275 /******************************************************************************/
276 /* Private: D o X F R */
277 /******************************************************************************/
278 
279 void XrdXrootdMonFile::DoXFR()
280 {
281  XrdXrootdFileStats *fsP;
282  int keep, i, n, hwm;
283 
284 // Reset interval counter
285 //
286  xfrRem = xfrCnt;
287 
288 // Grab the high watermark once
289 //
290  fmMutex.Lock();
291  hwm = fmHWM;
292  fmMutex.UnLock();
293 
294 // Report on all the files we have registered. This is a CPU burner as we
295 // periodically drop the lock to allow open/close requests to come through.
296 //
297  for (i = 0; i <= hwm; i++)
298  {fmMutex.Lock();
299  if (fmUse[i])
300  {n = 0;
302  while((fsP = fmMap[i].Next(n)))
303  {if (fsP->xfrXeq) DoXFR(fsP);
304  if (!keep--)
305  {fmMutex.UnLock();
307  fmMutex.Lock();
308  }
309  }
310  }
311  fmMutex.UnLock();
312  }
313 }
314 
315 /******************************************************************************/
316 
317 void XrdXrootdMonFile::DoXFR(XrdXrootdFileStats *fsP)
318 {
319  long long xfrRead, xfrReadv, xfrWrite;
320  char *cP;
321 
322 // Turn off the activity flag
323 //
324  fsP->xfrXeq = 0;
325 
326 // Grab the I/O bytes to get a somewhat consistent image here
327 //
328  xfrRead = fsP->xfr.read;
329  xfrReadv = fsP->xfr.readv;
330  xfrWrite = fsP->xfr.write;
331 
332 // Complete the record
333 //
334  xfrRec.Hdr.fileID = fsP->FileID;
335  xfrRec.Xfr.read = htonll(xfrRead);
336  xfrRec.Xfr.readv = htonll(xfrReadv);
337  xfrRec.Xfr.write = htonll(xfrWrite);
338 
339 // Get a pointer to the next slot (the buffer gets locked)
340 //
341  cP = GetSlot(sizeof(xfrRec));
342  memcpy(cP, &xfrRec, sizeof(xfrRec));
343  xfrRecs++;
344  bfMutex.UnLock();
345 }
346 
347 /******************************************************************************/
348 /* I n i t */
349 /******************************************************************************/
350 
352 {
353  XrdXrootdMonFile *mfP;
354  int alignment, pagsz = getpagesize();
355 
356 // Allocate a socket buffer
357 //
358  alignment = (fBsz < pagsz ? 1024 : pagsz);
359  if (posix_memalign((void **)&repBuff, alignment, fBsz))
360  {XrdXrootdMonInfo::eDest->Emsg("MonFile", "Unable to allocate monitor buffer.");
361  return false;
362  }
363 
364 // Set the header (always present)
365 //
366  repHdr = (XrdXrootdMonHeader *)repBuff;
367  repHdr->code = XROOTD_MON_MAPFSTA;
368  repHdr->pseq = 0;
370 
371 // Set the time record (always present)
372 //
373  repTOD = (XrdXrootdMonFileTOD *)(repBuff + sizeof(XrdXrootdMonHeader));
376  repTOD->Hdr.recSize = htons(sizeof(XrdXrootdMonFileTOD));
377  repTOD->sID = static_cast<kXR_int64>(XrdXrootdMonInfo::mySID);
378 
379 // Establish first real record in the buffer (always fixed)
380 //
381  repFirst = repBuff+sizeof(XrdXrootdMonHeader)+sizeof(XrdXrootdMonFileTOD);
382 
383 // Calculate the end nut the next slot always starts with a null pointer
384 //
385  repLast = repBuff+fBsz-1;
386  repNext = 0;
387 
388 // Compute the longest lfn that an open record can carry, not counting its
389 // null byte. The record must fit an empty buffer, and its size must fit the
390 // 16-bit recSize field. Records are a multiple of 8 bytes long, so use the
391 // largest multiple of 8 that fits. The fixed part is minRecSz plus the user
392 // dictid. Defaults() keeps fBsz at 1024 or more, so this is always positive.
393 //
394  int maxSlot = repLast - repFirst;
395  if (maxSlot > 32767) maxSlot = 32767;
396  maxText = (maxSlot & ~7) - (minRecSz + (int)sizeof(kXR_unt32)) - 1;
397 
398 // Calculate the close record size and the initial flags
399 //
400  crecSize = sizeof(XrdXrootdMonFileHdr) + sizeof(XrdXrootdMonStatXFR);
401  if (fsSSQ || fsOPS)
402  {crecSize += sizeof(XrdXrootdMonStatOPS);
403  crecFlag = XrdXrootdMonFileHdr::hasOPS;
404  } else crecFlag = 0;
405  if (fsSSQ)
406  {crecSize += sizeof(XrdXrootdMonStatSSQ);
407  crecFlag |= XrdXrootdMonFileHdr::hasSSQ;
408  }
409  crecNLen = htons(static_cast<short>(crecSize));
410 
411 // Preformat the i/o record
412 //
414  xfrRec.Hdr.recFlag = 0;
415  xfrRec.Hdr.recSize = htons(static_cast<short>(sizeof(xfrRec)));
416 
417 // Calculate the tod record size
418 //
419  trecNLen = htons(static_cast<short>(sizeof(XrdXrootdMonFileTOD)));
420 
421 // Allocate an instance of ourselves so we can schedule ourselves
422 //
423  mfP = new XrdXrootdMonFile();
424 
425 // Schedule an the flushes
426 //
427  XrdXrootdMonInfo::Sched->Schedule((XrdJob *)mfP, time(0)+repTime);
428  return true;
429 }
430 
431 /******************************************************************************/
432 /* Private: F l u s h */
433 /******************************************************************************/
434 
435 void XrdXrootdMonFile::Flush() // The bfMutex must be locked
436 {
437  static int seq = 0;
438  int bfSize;
439 
440 // Update the sequence number
441 //
442  repHdr->pseq = static_cast<char>(0x00ff & seq++);
443 
444 // Insert ending timestamp and record counts
445 //
446  repTOD->Hdr.nRecs[0] = htons(static_cast<short>(xfrRecs));
447  repTOD->Hdr.nRecs[1] = htons(static_cast<short>(totRecs));
448  repTOD->tEnd = htonl(static_cast<int>(time(0)));
449 
450 // Calculate buffer size and stick into the header
451 //
452  bfSize = (repNext - repBuff);
453  repHdr->plen = htons(static_cast<short>(bfSize));
454  repNext = 0;
455 
456 // Write this out
457 //
458  XrdXrootdMonitor::Send(XROOTD_MON_FSTA, repBuff, bfSize, false);
459  repTOD->tBeg = repTOD->tEnd;
460  xfrRecs = totRecs = 0;
461 }
462 
463 /******************************************************************************/
464 /* Private: G e t S l o t */
465 /******************************************************************************/
466 
467 char *XrdXrootdMonFile::GetSlot(int slotSZ)
468 {
469  char *myRec;
470 
471 // Lock this code to prevent interference (we should use double buffering)
472 // Note that the caller must do the unlock when finished with the slot.
473 //
474  bfMutex.Lock();
475 
476 // Check if we need to flush the buffer (sets repNext to zero). Otherwise,
477 // if this is the first record insert a timestamp.
478 //
479  if (repNext)
480  {if ((repNext + slotSZ) > repLast)
481  {Flush();
482  repNext = repFirst;
483  }
484  } else {
485  repTOD->tBeg = htonl(static_cast<int>(time(0)));
486  repNext = repFirst;
487  }
488 
489 // Return the slot
490 //
491  totRecs++;
492  myRec = repNext;
493  repNext += slotSZ;
494  return myRec;
495 }
496 
497 /******************************************************************************/
498 /* O p e n */
499 /******************************************************************************/
500 
502  unsigned int uDID, bool isRW)
503 {
505  int i = 0, sNum = -1, rLen, tLen = 0;
506 
507 // Assign the path a dictionary id if not assigned via file monitoring
508 //
509  if (fsP->FileID == 0) fsP->FileID = XrdXrootdMonitor::GetDictID();
510 
511 // Add this open to the map table if we are doing I/O stats.
512 //
513  if (fsXFR)
514  {fmMutex.Lock();
515  for (i = 0; i < mapUse; i++)
516  if (fmUse[i] < XrdXrootdMonFMap::fmSize)
517  {if ((sNum = fmMap[i].Insert(fsP)) >= 0)
518  {fmUse[i]++;
519  if (i > fmHWM) fmHWM = i;
520  break;
521  }
522  }
523  fmMutex.UnLock();
524  }
525 
526 // Generate the cookie (real or virtual) to find the entry in the map table.
527 // If the file has no entry (no xfr option, or the map is full), use noSlot.
528 // Supply the monitoring options for efficiency.
529 //
530  fsP->MonEnt = (sNum < 0 ? noSlot
531  : (sNum | (i << XrdXrootdMonFMap::fmShft)) & 0xffff);
532  fsP->monLvl = fsLVL;
533  fsP->xfrXeq = 0;
534 
535 // Compute the size of this record
536 //
537  rLen = minRecSz;
538  if (fsLFN)
539  {tLen = strlen(Path);
540  if (tLen > maxText) tLen = maxText;
541  rLen = (rLen + sizeof(kXR_unt32) + tLen + 1 + 7) & ~7;
542  }
543 
544 // Get a pointer to the next slot (the buffer gets locked)
545 //
546  oP = (XrdXrootdMonFileOPN *)GetSlot(rLen);
547 
548 // Fill out the record
549 //
551  oP->Hdr.recFlag = (isRW ? XrdXrootdMonFileHdr::hasRW : 0);
552  oP->Hdr.recSize = htons(static_cast<short>(rLen));
553  oP->Hdr.fileID = fsP->FileID;
554  oP->fsz = htonll(fsP->fSize);
555 
556 // Append user and path if so wanted (sizes have been verified).
557 // The lfn is null padded up to the end of the record.
558 //
559  if (fsLFN)
561  oP->ufn.user = uDID;
562  XrdXrootdMonitor::CopyText(oP->ufn.lfn, rLen - minRecSz - sizeof(kXR_unt32), Path, tLen);
563  }
564  bfMutex.UnLock();
565 }
long long kXR_int64
Definition: XPtypes.hh:98
unsigned int kXR_unt32
Definition: XPtypes.hh:90
XrdOucString Path
struct myOpts opts
XrdXrootdMonDouble write
XrdXrootdMonFileHdr Hdr
XrdXrootdMonFileLFN ufn
XrdXrootdMonDouble read
XrdXrootdMonStatXFR Xfr
XrdXrootdMonDouble readv
XrdXrootdMonStatXFR Xfr
XrdXrootdMonFileHdr Hdr
XrdXrootdMonStatOPS Ops
const kXR_char XROOTD_MON_MAPFSTA
XrdXrootdMonFileHdr Hdr
XrdXrootdMonFileHdr Hdr
XrdXrootdMonDouble rsegs
XrdXrootdMonStatSSQ Ssq
XrdXrootdMonFileHdr Hdr
#define XROOTD_MON_FSSSQ
#define XROOTD_MON_FSLFN
#define XROOTD_MON_FSTA
#define XROOTD_MON_FSOPS
#define XROOTD_MON_FSXFR
Definition: XrdJob.hh:43
void Schedule(XrdJob *jp)
int Emsg(const char *esfx, int ecode, const char *text1, const char *text2=0)
Definition: XrdSysError.cc:116
XrdXrootdMonStatXFR xfr
XrdXrootdMonStatOPS ops
struct XrdXrootdFileStats::@176 ssq
static const int fmShft
static const int fmSize
static const int mapNum
static const int fmHold
static const int fmMask
static void Disc(unsigned int usrID)
static void Defaults(int intv, int opts, int iocnt, int fbsz)
static void Open(XrdXrootdFileStats *fsP, const char *Path, unsigned int uDID, bool isRW)
static bool Init()
static void Close(XrdXrootdFileStats *fsP, bool isDisc=false)
static int CopyText(char *buff, int room, const char *text, int tLen)
static constexpr int maxDgram
static int Send(int mmode, void *buff, int size, bool setseq=true)
static kXR_unt32 GetDictID(bool hbo=false)
XrdScheduler * Sched
XrdSysError * eDest