XRootD
Loading...
Searching...
No Matches
XrdPfc.cc
Go to the documentation of this file.
1//----------------------------------------------------------------------------------
2// Copyright (c) 2014 by Board of Trustees of the Leland Stanford, Jr., University
3// Author: Alja Mrak-Tadel, Matevz Tadel, Brian Bockelman
4//----------------------------------------------------------------------------------
5// XRootD is free software: you can redistribute it and/or modify
6// it under the terms of the GNU Lesser General Public License as published by
7// the Free Software Foundation, either version 3 of the License, or
8// (at your option) any later version.
9//
10// XRootD is distributed in the hope that it will be useful,
11// but WITHOUT ANY WARRANTY; without even the implied warranty of
12// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13// GNU General Public License for more details.
14//
15// You should have received a copy of the GNU Lesser General Public License
16// along with XRootD. If not, see <http://www.gnu.org/licenses/>.
17//----------------------------------------------------------------------------------
18
19#include <fcntl.h>
20#include <sstream>
21#include <algorithm>
22#include <sys/statvfs.h>
23
24#include "XrdCl/XrdClURL.hh"
25
26#include "XrdOuc/XrdOucEnv.hh"
27#include "XrdOuc/XrdOucUtils.hh"
29
30#include "XrdSys/XrdSysTimer.hh"
31#include "XrdSys/XrdSysTrace.hh"
32#include "XrdSys/XrdSysXAttr.hh"
33
35
36#include "XrdOss/XrdOss.hh"
37
38#include "XrdPfc.hh"
39#include "XrdPfcTrace.hh"
40#include "XrdPfcFSctl.hh"
41#include "XrdPfcInfo.hh"
42#include "XrdPfcIOFile.hh"
43#include "XrdPfcIOFileBlock.hh"
45
47
48using namespace XrdPfc;
49
50Cache *Cache::m_instance = nullptr;
52
53
55{
57 return 0;
58}
59
61{
63 return 0;
64}
65
66void *PrefetchThread(void*)
67{
69 return 0;
70}
71
72//==============================================================================
73
74extern "C"
75{
77 const char *config_filename,
78 const char *parameters,
79 XrdOucEnv *env)
80{
81 XrdSysError err(logger, "");
82 err.Say("++++++ Proxy file cache initialization started.");
83
84 if ( ! env ||
85 ! (XrdPfc::Cache::schedP = (XrdScheduler*) env->GetPtr("XrdScheduler*")))
86 {
88 XrdPfc::Cache::schedP->Start();
89 }
90
91 Cache &instance = Cache::CreateInstance(logger, env);
92
93 if (! instance.Config(config_filename, parameters, env))
94 {
95 err.Say("Config Proxy file cache initialization failed.");
96 return 0;
97 }
98 err.Say("++++++ Proxy file cache initialization completed.");
99
100 {
101 pthread_t tid;
102
103 XrdSysThread::Run(&tid, ResourceMonitorThread, 0, 0, "XrdPfc ResourceMonitor");
104
105 for (int wti = 0; wti < instance.RefConfiguration().m_wqueue_threads; ++wti)
106 {
107 XrdSysThread::Run(&tid, ProcessWriteTaskThread, 0, 0, "XrdPfc WriteTasks ");
108 }
109
110 if (instance.is_prefetch_enabled())
111 {
112 XrdSysThread::Run(&tid, PrefetchThread, 0, 0, "XrdPfc Prefetch ");
113 }
114 }
115
116 XrdPfcFSctl* pfcFSctl = new XrdPfcFSctl(instance, logger);
117 env->PutPtr("XrdFSCtl_PC*", pfcFSctl);
118
119 return &instance;
120}
121}
122
123//==============================================================================
124
126{
127 assert (m_instance == 0);
128 m_instance = new Cache(logger, env);
129 return *m_instance;
130}
131
132 Cache& Cache::GetInstance() { return *m_instance; }
133const Cache& Cache::TheOne() { return *m_instance; }
134const Configuration& Cache::Conf() { return m_instance->RefConfiguration(); }
135 ResourceMonitor& Cache::ResMon() { return m_instance->RefResMon(); }
136
138{
139 if (! m_decisionpoints.empty())
140 {
141 XrdCl::URL url(io->Path());
142 std::string filename = url.GetPath();
143 std::vector<Decision*>::const_iterator it;
144 for (it = m_decisionpoints.begin(); it != m_decisionpoints.end(); ++it)
145 {
146 XrdPfc::Decision *d = *it;
147 if (! d) continue;
148 if (! d->Decide(filename, *m_oss))
149 {
150 return false;
151 }
152 }
153 }
154
155 return true;
156}
157
159 XrdOucCache("pfc"),
160 m_env(env),
161 m_log(logger, "XrdPfc_"),
162 m_trace(new XrdSysTrace("XrdPfc", logger)),
163 m_traceID("Cache"),
164 m_oss(0),
165 m_gstream(0),
166 m_purge_pin(0),
167 m_prefetch_condVar(0),
168 m_prefetch_enabled(false),
169 m_RAM_used(0),
170 m_RAM_write_queue(0),
171 m_RAM_std_size(0),
172 m_isClient(false),
173 m_active_cond(0)
174{
175 // Default log level is Warning.
176 m_trace->What = 2;
177}
178
180{
181 const char* tpfx = "Attach() ";
182
183 if (Options & XrdOucCache::optRW)
184 {
185 TRACE(Info, tpfx << "passing through write operation" << obfuscateAuth(io->Path()));
186 }
187 else if (Cache::GetInstance().Decide(io))
188 {
189 TRACE(Info, tpfx << obfuscateAuth(io->Path()));
190
191 IO *cio;
192
193 if (Cache::GetInstance().RefConfiguration().m_hdfsmode)
194 {
195 cio = new IOFileBlock(io, *this);
196 }
197 else
198 {
199 IOFile *iof = new IOFile(io, *this);
200
201 if ( ! iof->HasFile())
202 {
203 delete iof;
204 // TODO - redirect instead. But this is kind of an awkward place for it.
205 // errno is set during IOFile construction.
206 TRACE(Error, tpfx << "Failed opening local file, falling back to remote access " << io->Path());
207 return io;
208 }
209
210 cio = iof;
211 }
212
213 TRACE_PC(Debug, const char* loc = io->Location(), tpfx << io->Path() << " location: " <<
214 ((loc && loc[0] != 0) ? loc : "<deferred open>"));
215
216 return cio;
217 }
218 else
219 {
220 TRACE(Info, tpfx << "decision decline " << io->Path());
221 }
222 return io;
223}
224
225void Cache::AddWriteTask(Block* b, bool fromRead)
226{
227 TRACE(Dump, "AddWriteTask() offset=" << b->m_offset << ". file " << b->get_file()->GetLocalPath());
228
229 {
230 XrdSysMutexHelper lock(&m_RAM_mutex);
231 m_RAM_write_queue += b->get_size();
232 }
233
234 m_writeQ.condVar.Lock();
235 if (fromRead)
236 m_writeQ.queue.push_back(b);
237 else
238 m_writeQ.queue.push_front(b);
239 m_writeQ.size++;
240 m_writeQ.condVar.Signal();
241 m_writeQ.condVar.UnLock();
242}
243
245{
246 std::list<Block*> removed_blocks;
247 long long sum_size = 0;
248
249 m_writeQ.condVar.Lock();
250 std::list<Block*>::iterator i = m_writeQ.queue.begin();
251 while (i != m_writeQ.queue.end())
252 {
253 if ((*i)->m_file == file)
254 {
255 TRACE(Dump, "Remove entries for " << (void*)(*i) << " path " << file->lPath());
256 std::list<Block*>::iterator j = i++;
257 removed_blocks.push_back(*j);
258 sum_size += (*j)->get_size();
259 m_writeQ.queue.erase(j);
260 --m_writeQ.size;
261 }
262 else
263 {
264 ++i;
265 }
266 }
267 m_writeQ.condVar.UnLock();
268
269 {
270 XrdSysMutexHelper lock(&m_RAM_mutex);
271 m_RAM_write_queue -= sum_size;
272 }
273
274 file->BlocksRemovedFromWriteQ(removed_blocks);
275}
276
278{
279 std::vector<Block*> blks_to_write(m_configuration.m_wqueue_blocks);
280
281 while (true)
282 {
283 m_writeQ.condVar.Lock();
284 while (m_writeQ.size == 0)
285 {
286 m_writeQ.condVar.Wait();
287 }
288
289 // MT -- optimize to pop several blocks if they are available (or swap the list).
290 // This makes sense especially for smallish block sizes.
291
292 int n_pushed = std::min(m_writeQ.size, m_configuration.m_wqueue_blocks);
293 long long sum_size = 0;
294
295 for (int bi = 0; bi < n_pushed; ++bi)
296 {
297 Block* block = m_writeQ.queue.front();
298 m_writeQ.queue.pop_front();
299 m_writeQ.writes_between_purges += block->get_size();
300 sum_size += block->get_size();
301
302 blks_to_write[bi] = block;
303
304 TRACE(Dump, "ProcessWriteTasks for block " << (void*)(block) << " path " << block->m_file->lPath());
305 }
306 m_writeQ.size -= n_pushed;
307
308 m_writeQ.condVar.UnLock();
309
310 {
311 XrdSysMutexHelper lock(&m_RAM_mutex);
312 m_RAM_write_queue -= sum_size;
313 }
314
315 for (int bi = 0; bi < n_pushed; ++bi)
316 {
317 Block* block = blks_to_write[bi];
318
319 block->m_file->WriteBlockToDisk(block);
320 }
321 }
322}
323
325{
326 // Called from ResourceMonitor for an alternative estimation of disk writes.
327 XrdSysCondVarHelper lock(&m_writeQ.condVar);
328 long long ret = m_writeQ.writes_between_purges;
329 m_writeQ.writes_between_purges = 0;
330 return ret;
331}
332
333//==============================================================================
334
335char* Cache::RequestRAM(long long size)
336{
337 static const size_t s_block_align = sysconf(_SC_PAGESIZE);
338
339 bool std_size = (size == m_configuration.m_bufferSize);
340
341 m_RAM_mutex.Lock();
342
343 long long total = m_RAM_used + size;
344
345 if (total <= m_configuration.m_RamAbsAvailable)
346 {
347 m_RAM_used = total;
348 if (std_size && m_RAM_std_size > 0)
349 {
350 char *buf = m_RAM_std_blocks.back();
351 m_RAM_std_blocks.pop_back();
352 --m_RAM_std_size;
353
354 m_RAM_mutex.UnLock();
355
356 return buf;
357 }
358 else
359 {
360 m_RAM_mutex.UnLock();
361 char *buf;
362 if (posix_memalign((void**) &buf, s_block_align, (size_t) size))
363 {
364 // Report out of mem? Probably should report it at least the first time,
365 // then periodically.
366 return 0;
367 }
368 return buf;
369 }
370 }
371 m_RAM_mutex.UnLock();
372 return 0;
373}
374
375void Cache::ReleaseRAM(char* buf, long long size)
376{
377 bool std_size = (size == m_configuration.m_bufferSize);
378 {
379 XrdSysMutexHelper lock(&m_RAM_mutex);
380
381 m_RAM_used -= size;
382
383 if (std_size && m_RAM_std_size < m_configuration.m_RamKeepStdBlocks)
384 {
385 m_RAM_std_blocks.push_back(buf);
386 ++m_RAM_std_size;
387 return;
388 }
389 }
390 free(buf);
391}
392
393File* Cache::GetFile(const std::string& path, IO* io, long long off, long long filesize)
394{
395 // Called from virtual IOFile constructor.
396
397 TRACE(Debug, "GetFile " << path << ", io " << io);
398
399 ActiveMap_i it;
400
401 {
402 XrdSysCondVarHelper lock(&m_active_cond);
403
404 while (true)
405 {
406 it = m_active.find(path);
407
408 // File is not open or being opened. Mark it as being opened and
409 // proceed to opening it outside of while loop.
410 if (it == m_active.end())
411 {
412 it = m_active.insert(std::make_pair(path, (File*) 0)).first;
413 break;
414 }
415
416 if (it->second != 0)
417 {
418 it->second->AddIO(io);
419 inc_ref_cnt(it->second, false, true);
420
421 return it->second;
422 }
423 else
424 {
425 // Wait for some change in m_active, then recheck.
426 m_active_cond.Wait();
427 }
428 }
429 }
430
431 // This is always true, now that IOFileBlock is unsupported.
432 if (filesize == 0)
433 {
434 struct stat st;
435 int res = io->Fstat(st);
436 if (res < 0) {
437 errno = res;
438 filesize = -1;
439 TRACE(Error, "GetFile, could not get valid stat");
440 } else if (res > 0) {
441 errno = ENOTSUP;
442 filesize = -1;
443 TRACE(Error, "GetFile, stat returned positive value, this should NOT happen here");
444 } else {
445 filesize = st.st_size;
446 }
447 }
448
449 File *file = 0;
450
451 if (filesize >= 0)
452 {
453 file = File::FileOpen(path, off, filesize, io->GetInput());
454 }
455
456 {
457 XrdSysCondVarHelper lock(&m_active_cond);
458
459 if (file)
460 {
461 inc_ref_cnt(file, false, true);
462 it->second = file;
463
464 file->AddIO(io);
465 }
466 else
467 {
468 m_active.erase(it);
469 }
470
471 m_active_cond.Broadcast();
472 }
473
474 return file;
475}
476
478{
479 // Called from virtual IO::DetachFinalize.
480
481 TRACE(Debug, "ReleaseFile " << f->GetLocalPath() << ", io " << io);
482
483 {
484 XrdSysCondVarHelper lock(&m_active_cond);
485
486 f->RemoveIO(io);
487 }
488 dec_ref_cnt(f, true);
489}
490
491
492namespace
493{
494
495class DiskSyncer : public XrdJob
496{
497private:
498 File *m_file;
499 bool m_high_debug;
500
501public:
502 DiskSyncer(File *f, bool high_debug, const char *desc = "") :
503 XrdJob(desc),
504 m_file(f),
505 m_high_debug(high_debug)
506 {}
507
508 void DoIt()
509 {
510 m_file->Sync();
511 Cache::GetInstance().FileSyncDone(m_file, m_high_debug);
512 delete this;
513 }
514};
515
516
517class CommandExecutor : public XrdJob
518{
519private:
520 std::string m_command_url;
521
522public:
523 CommandExecutor(const std::string& command, const char *desc = "") :
524 XrdJob(desc),
525 m_command_url(command)
526 {}
527
528 void DoIt()
529 {
530 Cache::GetInstance().ExecuteCommandUrl(m_command_url);
531 delete this;
532 }
533};
534
535}
536
537//==============================================================================
538
539void Cache::schedule_file_sync(File* f, bool ref_cnt_already_set, bool high_debug)
540{
541 DiskSyncer* ds = new DiskSyncer(f, high_debug);
542
543 if ( ! ref_cnt_already_set) inc_ref_cnt(f, true, high_debug);
544
545 schedP->Schedule(ds);
546}
547
548void Cache::FileSyncDone(File* f, bool high_debug)
549{
550 dec_ref_cnt(f, high_debug);
551}
552
553void Cache::inc_ref_cnt(File* f, bool lock, bool high_debug)
554{
555 // called from GetFile() or SheduleFileSync();
556
557 int tlvl = high_debug ? TRACE_Debug : TRACE_Dump;
558
559 if (lock) m_active_cond.Lock();
560 int rc = f->inc_ref_cnt();
561 if (lock) m_active_cond.UnLock();
562
563 TRACE_INT(tlvl, "inc_ref_cnt " << f->GetLocalPath() << ", cnt at exit = " << rc);
564}
565
566void Cache::dec_ref_cnt(File* f, bool high_debug)
567{
568 // NOT under active lock.
569 // Called from ReleaseFile(), DiskSync callback and stat-like functions.
570
571 int tlvl = high_debug ? TRACE_Debug : TRACE_Dump;
572 int cnt;
573
574 bool emergency_close = false;
575 {
576 XrdSysCondVarHelper lock(&m_active_cond);
577
578 cnt = f->get_ref_cnt();
579 TRACE_INT(tlvl, "dec_ref_cnt " << f->GetLocalPath() << ", cnt at entry = " << cnt);
580
582 {
583 // In this case file has been already removed from m_active map and
584 // does not need to be synced.
585
586 if (cnt == 1)
587 {
588 TRACE_INT(tlvl, "dec_ref_cnt " << f->GetLocalPath() << " is in shutdown, ref_cnt = " << cnt
589 << " -- closing and deleting File object without further ado");
590 emergency_close = true;
591 }
592 else
593 {
594 TRACE_INT(tlvl, "dec_ref_cnt " << f->GetLocalPath() << " is in shutdown, ref_cnt = " << cnt
595 << " -- waiting");
596 f->dec_ref_cnt();
597 return;
598 }
599 }
600 if (cnt > 1)
601 {
602 f->dec_ref_cnt();
603 return;
604 }
605 }
606 if (emergency_close)
607 {
608 f->Close();
609 delete f;
610 return;
611 }
612
613 if (cnt == 1)
614 {
615 if (f->FinalizeSyncBeforeExit())
616 {
617 // Note, here we "reuse" the existing reference count for the
618 // final sync.
619
620 TRACE(Debug, "dec_ref_cnt " << f->GetLocalPath() << ", scheduling final sync");
621 schedule_file_sync(f, true, true);
622 return;
623 }
624 }
625
626 bool finished_p = false;
627 ActiveMap_i act_it;
628 {
629 XrdSysCondVarHelper lock(&m_active_cond);
630
631 cnt = f->dec_ref_cnt();
632 TRACE_INT(tlvl, "dec_ref_cnt " << f->GetLocalPath() << ", cnt after sync_check and dec_ref_cnt = " << cnt);
633 if (cnt == 0)
634 {
635 act_it = m_active.find(f->GetLocalPath());
636 act_it->second = 0;
637
638 finished_p = true;
639 }
640 }
641 if (finished_p)
642 {
643 f->Close();
644 {
645 XrdSysCondVarHelper lock(&m_active_cond);
646 m_active.erase(act_it);
647 m_active_cond.Broadcast();
648 }
649
650 if (m_gstream)
651 {
652 const Stats &st = f->RefStats();
653 const Info::AStat *as = f->GetLastAccessStats();
654
655 char buf[4096];
656 int len = snprintf(buf, 4096, "{\"event\":\"file_close\","
657 "\"lfn\":\"%s\",\"size\":%lld,\"blk_size\":%d,\"n_blks\":%d,\"n_blks_done\":%d,"
658 "\"access_cnt\":%lu,\"attach_t\":%lld,\"detach_t\":%lld,\"remotes\":%s,"
659 "\"b_hit\":%lld,\"b_miss\":%lld,\"b_bypass\":%lld,"
660 "\"b_todisk\":%lld,\"b_prefetch\":%lld,\"n_cks_errs\":%d}",
661 f->GetLocalPath().c_str(), f->GetFileSize(), f->GetBlockSize(),
663 (unsigned long) f->GetAccessCnt(), (long long) as->AttachTime, (long long) as->DetachTime,
664 f->GetRemoteLocations().c_str(),
665 as->BytesHit, as->BytesMissed, as->BytesBypassed,
667 );
668 bool suc = false;
669 if (len < 4096)
670 {
671 suc = m_gstream->Insert(buf, len + 1);
672 }
673 if ( ! suc)
674 {
675 TRACE(Error, "Failed g-stream insertion of file_close record, len=" << len);
676 }
677 }
678
679 delete f;
680 }
681}
682
683bool Cache::IsFileActiveOrPurgeProtected(const std::string& path) const
684{
685 XrdSysCondVarHelper lock(&m_active_cond);
686
687 return m_active.find(path) != m_active.end() ||
688 m_purge_delay_set.find(path) != m_purge_delay_set.end();
689}
690
692{
693 XrdSysCondVarHelper lock(&m_active_cond);
694 m_purge_delay_set.clear();
695}
696
697//==============================================================================
698//=== PREFETCH
699//==============================================================================
700
702{
703 // Can be called with other locks held.
704
705 if ( ! m_prefetch_enabled)
706 {
707 return;
708 }
709
710 m_prefetch_condVar.Lock();
711 m_prefetchList.push_back(file);
712 m_prefetch_condVar.Signal();
713 m_prefetch_condVar.UnLock();
714}
715
716
718{
719 // Can be called with other locks held.
720
721 if ( ! m_prefetch_enabled)
722 {
723 return;
724 }
725
726 m_prefetch_condVar.Lock();
727 for (PrefetchList::iterator it = m_prefetchList.begin(); it != m_prefetchList.end(); ++it)
728 {
729 if (*it == file)
730 {
731 m_prefetchList.erase(it);
732 break;
733 }
734 }
735 m_prefetch_condVar.UnLock();
736}
737
738
740{
741 m_prefetch_condVar.Lock();
742 while (m_prefetchList.empty())
743 {
744 m_prefetch_condVar.Wait();
745 }
746
747 // std::sort(m_prefetchList.begin(), m_prefetchList.end(), myobject);
748
749 size_t l = m_prefetchList.size();
750 int idx = rand() % l;
751 File* f = m_prefetchList[idx];
752
753 m_prefetch_condVar.UnLock();
754 return f;
755}
756
757
759{
760 const long long limit_RAM = m_configuration.m_RamAbsAvailable * 7 / 10;
761
762 while (true)
763 {
764 m_RAM_mutex.Lock();
765 bool doPrefetch = (m_RAM_used < limit_RAM);
766 m_RAM_mutex.UnLock();
767
768 if (doPrefetch)
769 {
771 f->Prefetch();
772 }
773 else
774 {
776 }
777 }
778}
779
780
781//==============================================================================
782//=== Virtuals from XrdOucCache
783//==============================================================================
784
785//------------------------------------------------------------------------------
799
800int Cache::LocalFilePath(const char *curl, char *buff, int blen,
801 LFP_Reason why, bool forall)
802{
803 static const mode_t groupReadable = S_IRUSR | S_IWUSR | S_IRGRP;
804 static const mode_t worldReadable = S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH;
805 static const char *lfpReason[] = { "ForAccess", "ForInfo", "ForPath" };
806
807 TRACE(Debug, "LocalFilePath '" << curl << "', why=" << lfpReason[why]);
808
809 if (buff && blen > 0) buff[0] = 0;
810
811 XrdCl::URL url(curl);
812 std::string f_name = url.GetPath();
813 std::string i_name = f_name + Info::s_infoExtension;
814
815 if (why == ForPath)
816 {
817 int ret = m_oss->Lfn2Pfn(f_name.c_str(), buff, blen);
818 TRACE(Info, "LocalFilePath '" << curl << "', why=" << lfpReason[why] << " -> " << ret);
819 return ret;
820 }
821
822 {
823 XrdSysCondVarHelper lock(&m_active_cond);
824 m_purge_delay_set.insert(f_name);
825 }
826
827 struct stat sbuff, sbuff2;
828 if (m_oss->Stat(f_name.c_str(), &sbuff) == XrdOssOK &&
829 m_oss->Stat(i_name.c_str(), &sbuff2) == XrdOssOK)
830 {
831 if (S_ISDIR(sbuff.st_mode))
832 {
833 TRACE(Info, "LocalFilePath '" << curl << "', why=" << lfpReason[why] << " -> EISDIR");
834 return -EISDIR;
835 }
836 else
837 {
838 bool read_ok = false;
839 bool is_complete = false;
840
841 // Lock and check if the file is active. If NOT, keep the lock
842 // and add dummy access after successful reading of info file.
843 // If it IS active, just release the lock, this ongoing access will
844 // assure the file continues to exist.
845
846 // XXXX How can I just loop over the cinfo file when active?
847 // Can I not get is_complete from the existing file?
848 // Do I still want to inject access record?
849 // Oh, it writes only if not active .... still let's try to use existing File.
850
851 m_active_cond.Lock();
852
853 bool is_active = m_active.find(f_name) != m_active.end();
854
855 if (is_active) m_active_cond.UnLock();
856
857 XrdOssDF* infoFile = m_oss->newFile(m_configuration.m_username.c_str());
858 XrdOucEnv myEnv;
859 int res = infoFile->Open(i_name.c_str(), O_RDWR, 0600, myEnv);
860 if (res >= 0)
861 {
862 Info info(m_trace, 0);
863 if (info.Read(infoFile, i_name.c_str()))
864 {
865 read_ok = true;
866
867 is_complete = info.IsComplete();
868
869 // Add full-size access if reason is for access.
870 if ( ! is_active && is_complete && why == ForAccess)
871 {
872 info.WriteIOStatSingle(info.GetFileSize());
873 info.Write(infoFile, i_name.c_str());
874 }
875 }
876 infoFile->Close();
877 }
878 delete infoFile;
879
880 if ( ! is_active) m_active_cond.UnLock();
881
882 if (read_ok)
883 {
884 if ((is_complete || why == ForInfo) && buff != 0)
885 {
886 int res2 = m_oss->Lfn2Pfn(f_name.c_str(), buff, blen);
887 if (res2 < 0)
888 return res2;
889
890 // Normally, files are owned by us but when direct cache access
891 // is wanted and possible, make sure the file is world readable.
892 if (why == ForAccess)
893 {mode_t mode = (forall ? worldReadable : groupReadable);
894 if (((sbuff.st_mode & worldReadable) != mode)
895 && (m_oss->Chmod(f_name.c_str(), mode) != XrdOssOK))
896 {is_complete = false;
897 *buff = 0;
898 }
899 }
900 }
901
902 TRACE(Info, "LocalFilePath '" << curl << "', why=" << lfpReason[why] <<
903 (is_complete ? " -> FILE_COMPLETE_IN_CACHE" : " -> EREMOTE"));
904
905 return is_complete ? 0 : -EREMOTE;
906 }
907 }
908 }
909
910 TRACE(Info, "LocalFilePath '" << curl << "', why=" << lfpReason[why] << " -> ENOENT");
911 return -ENOENT;
912}
913
914//______________________________________________________________________________
915// If supported, write file_size as xattr to cinfo file.
916//------------------------------------------------------------------------------
917void Cache::WriteFileSizeXAttr(int cinfo_fd, long long file_size)
918{
919 if (m_metaXattr) {
920 int res = XrdSysXAttrActive->Set("pfc.fsize", &file_size, sizeof(long long), 0, cinfo_fd, 0);
921 if (res != 0) {
922 TRACE(Debug, "WriteFileSizeXAttr error setting xattr " << res);
923 }
924 }
925}
926
927//______________________________________________________________________________
928// Determine full size of the data file from the corresponding cinfo-file name.
929// Attempts to read xattr first and falls back to reading of the cinfo file.
930// Returns -error on failure.
931//------------------------------------------------------------------------------
932long long Cache::DetermineFullFileSize(const std::string &cinfo_fname)
933{
934 if (m_metaXattr) {
935 char pfn[4096];
936 m_oss->Lfn2Pfn(cinfo_fname.c_str(), pfn, 4096);
937 long long fsize = -1ll;
938 int res = XrdSysXAttrActive->Get("pfc.fsize", &fsize, sizeof(long long), pfn);
939 if (res == sizeof(long long))
940 {
941 return fsize;
942 }
943 else
944 {
945 TRACE(Debug, "DetermineFullFileSize error getting xattr " << res);
946 }
947 }
948
949 XrdOssDF *infoFile = m_oss->newFile(m_configuration.m_username.c_str());
950 XrdOucEnv env;
951 long long ret;
952 int res = infoFile->Open(cinfo_fname.c_str(), O_RDONLY, 0600, env);
953 if (res < 0) {
954 ret = res;
955 } else {
956 Info info(m_trace, 0);
957 if ( ! info.Read(infoFile, cinfo_fname.c_str())) {
958 ret = -EBADF;
959 } else {
960 ret = info.GetFileSize();
961 }
962 infoFile->Close();
963 }
964 delete infoFile;
965 return ret;
966}
967
968//______________________________________________________________________________
969// Calculate if the file is to be considered cached for the purposes of
970// only-if-cached and setting of atime of the Stat() calls.
971// Returns true if the file is to be conidered cached.
972//------------------------------------------------------------------------------
973bool Cache::DecideIfConsideredCached(long long file_size, long long bytes_on_disk)
974{
975 if (file_size == 0 || bytes_on_disk >= file_size)
976 return true;
977
978 double frac_on_disk = (double) bytes_on_disk / file_size;
979
980 if (file_size <= m_configuration.m_onlyIfCachedMinSize)
981 {
982 if (frac_on_disk >= m_configuration.m_onlyIfCachedMinFrac)
983 return true;
984 }
985 else
986 {
987 if (bytes_on_disk >= m_configuration.m_onlyIfCachedMinSize &&
988 frac_on_disk >= m_configuration.m_onlyIfCachedMinFrac)
989 return true;
990 }
991 return false;
992}
993
994//______________________________________________________________________________
995// Check if the file is cached including m_onlyIfCachedMinSize and m_onlyIfCachedMinFrac
996// pfc configuration parameters. The logic of accessing the Info file is the same
997// as in Cache::LocalFilePath.
1005//------------------------------------------------------------------------------
1006int Cache::ConsiderCached(const char *curl)
1007{
1008 static const char* tpfx = "ConsiderCached ";
1009
1010 TRACE(Debug, tpfx << curl);
1011
1012 XrdCl::URL url(curl);
1013 std::string f_name = url.GetPath();
1014
1015 File *file = nullptr;
1016 {
1017 XrdSysCondVarHelper lock(&m_active_cond);
1018 auto it = m_active.find(f_name);
1019 if (it != m_active.end()) {
1020 file = it->second;
1021 // If the file-open is in progress, `file` is a nullptr
1022 // so we cannot increase the reference count. For now,
1023 // simply treat it as if the file open doesn't exist instead
1024 // of trying to wait and see if it succeeds.
1025 if (file) {
1026 inc_ref_cnt(file, false, false);
1027 }
1028 }
1029 }
1030 if (file) {
1031 struct stat sbuff;
1032 int res = file->Fstat(sbuff);
1033 dec_ref_cnt(file, false);
1034 if (res)
1035 return res;
1036 // DecideIfConsideredCached() already called in File::Fstat().
1037 return sbuff.st_atime > 0 ? 0 : -EREMOTE;
1038 }
1039
1040 struct stat sbuff;
1041 int res = m_oss->Stat(f_name.c_str(), &sbuff);
1042 if (res != XrdOssOK) {
1043 TRACE(Debug, tpfx << curl << " -> " << res);
1044 return res;
1045 }
1046 if (S_ISDIR(sbuff.st_mode))
1047 {
1048 TRACE(Debug, tpfx << curl << " -> EISDIR");
1049 return -EISDIR;
1050 }
1051
1052 long long file_size = DetermineFullFileSize(f_name + Info::s_infoExtension);
1053 if (file_size < 0) {
1054 TRACE(Debug, tpfx << curl << " -> " << file_size);
1055 return (int) file_size;
1056 }
1057 bool is_cached = DecideIfConsideredCached(file_size, sbuff.st_blocks * 512ll);
1058
1059 return is_cached ? 0 : -EREMOTE;
1060}
1061
1062//______________________________________________________________________________
1070//------------------------------------------------------------------------------
1071
1072int Cache::Prepare(const char *curl, int oflags, mode_t mode)
1073{
1074 XrdCl::URL url(curl);
1075 std::string f_name = url.GetPath();
1076 std::string i_name = f_name + Info::s_infoExtension;
1077
1078 // Do not allow write access.
1079 if ((oflags & O_ACCMODE) != O_RDONLY)
1080 {
1081 if (Cache::GetInstance().RefConfiguration().m_write_through)
1082 {
1083 return 0;
1084 }
1085 TRACE(Warning, "Prepare write access requested on file " << f_name << ". Denying access.");
1086 return -EROFS;
1087 }
1088
1089 // Intercept xrdpfc_command requests.
1090 if (m_configuration.m_allow_xrdpfc_command && strncmp("/xrdpfc_command/", f_name.c_str(), 16) == 0)
1091 {
1092 // Schedule a job to process command request.
1093 {
1094 CommandExecutor *ce = new CommandExecutor(f_name, "CommandExecutor");
1095
1096 schedP->Schedule(ce);
1097 }
1098
1099 return -EAGAIN;
1100 }
1101
1102 {
1103 XrdSysCondVarHelper lock(&m_active_cond);
1104 m_purge_delay_set.insert(f_name);
1105 }
1106
1107 struct stat sbuff;
1108 if (m_oss->Stat(i_name.c_str(), &sbuff) == XrdOssOK &&
1109 m_oss->Stat(f_name.c_str(), &sbuff) == XrdOssOK)
1110 {
1111 TRACE(Dump, "Prepare defer open " << f_name);
1112 return 1;
1113 }
1114 else
1115 {
1116 return 0;
1117 }
1118}
1119
1120//______________________________________________________________________________
1121// virtual method of XrdOucCache.
1126//------------------------------------------------------------------------------
1127
1128int Cache::Stat(const char *curl, struct stat &sbuff)
1129{
1130 const char *tpfx = "Stat ";
1131
1132 XrdCl::URL url(curl);
1133 std::string f_name = url.GetPath();
1134
1135 File *file = nullptr;
1136 {
1137 XrdSysCondVarHelper lock(&m_active_cond);
1138 auto it = m_active.find(f_name);
1139 if (it != m_active.end()) {
1140 file = it->second;
1141 // If `file` is nullptr, the file-open is in progress; instead
1142 // of waiting for the file-open to finish, simply treat it as if
1143 // the file-open doesn't exist.
1144 if (file) {
1145 inc_ref_cnt(file, false, false);
1146 }
1147 }
1148 }
1149 if (file) {
1150 int res = file->Fstat(sbuff);
1151 dec_ref_cnt(file, false);
1152 TRACE(Debug, tpfx << "from active file " << curl << " -> " << res);
1153 return res;
1154 }
1155
1156 int res = m_oss->Stat(f_name.c_str(), &sbuff);
1157 if (res != XrdOssOK) {
1158 TRACE(Debug, tpfx << curl << " -> " << res);
1159 return 1; // res; -- for only-if-cached
1160 }
1161 if (S_ISDIR(sbuff.st_mode))
1162 {
1163 TRACE(Debug, tpfx << curl << " -> EISDIR");
1164 return -EISDIR;
1165 }
1166
1167 long long file_size = DetermineFullFileSize(f_name + Info::s_infoExtension);
1168 if (file_size < 0) {
1169 TRACE(Debug, tpfx << curl << " -> " << file_size);
1170 return 1; // (int) file_size; -- for only-if-cached
1171 }
1172 sbuff.st_size = file_size;
1173 bool is_cached = DecideIfConsideredCached(file_size, sbuff.st_blocks * 512ll);
1174 if ( ! is_cached)
1175 sbuff.st_atime = 0;
1176
1177 TRACE(Debug, tpfx << "from disk " << curl << " -> " << res);
1178
1179 return 0;
1180}
1181
1182//______________________________________________________________________________
1183// virtual method of XrdOucCache.
1187//------------------------------------------------------------------------------
1188
1189int Cache::Unlink(const char *curl)
1190{
1191 XrdCl::URL url(curl);
1192 std::string f_name = url.GetPath();
1193
1194 // printf("Unlink url=%s\n\t fname=%s\n", curl, f_name.c_str());
1195
1196 return UnlinkFile(f_name, false);
1197}
1198
1199int Cache::UnlinkFile(const std::string& f_name, bool fail_if_open)
1200{
1201 static const char* trc_pfx = "UnlinkFile ";
1202 ActiveMap_i it;
1203 File *file = 0;
1204 long long st_blocks_to_purge = 0;
1205 bool file_existed = false;
1206 {
1207 XrdSysCondVarHelper lock(&m_active_cond);
1208
1209 it = m_active.find(f_name);
1210
1211 if (it != m_active.end())
1212 {
1213 if (fail_if_open)
1214 {
1215 TRACE(Info, trc_pfx << f_name << ", file currently open and force not requested - denying request");
1216 return -EBUSY;
1217 }
1218
1219 // Null File* in m_active map means an operation is ongoing, probably
1220 // Attach() with possible File::Open(). Ask for retry.
1221 if (it->second == 0)
1222 {
1223 TRACE(Info, trc_pfx << f_name << ", an operation on this file is ongoing - denying request");
1224 return -EAGAIN;
1225 }
1226
1227 file = it->second;
1228 st_blocks_to_purge = file->initiate_emergency_shutdown();
1229 it->second = 0;
1230 }
1231 else
1232 {
1233 it = m_active.insert(std::make_pair(f_name, (File*) 0)).first;
1234 }
1235 }
1236
1237 if (file) {
1238 file_existed = true;
1240 } else {
1241 struct stat f_stat;
1242 if (m_oss->Stat(f_name.c_str(), &f_stat) == XrdOssOK) {
1243 file_existed = true;
1244 st_blocks_to_purge = f_stat.st_blocks;
1245 }
1246 }
1247
1248 std::string i_name = f_name + Info::s_infoExtension;
1249
1250 // Unlink file & cinfo
1251 int f_ret = m_oss->Unlink(f_name.c_str());
1252 int i_ret = m_oss->Unlink(i_name.c_str());
1253
1254 // Count a removal for any file that existed, even with zero blocks, and never
1255 // for an absent one.
1256 if (file_existed)
1257 m_res_mon->register_file_purge(f_name, st_blocks_to_purge);
1258
1259 TRACE(Debug, trc_pfx << f_name << ", f_ret=" << f_ret << ", i_ret=" << i_ret);
1260
1261 {
1262 XrdSysCondVarHelper lock(&m_active_cond);
1263 m_active.erase(it);
1264 m_active_cond.Broadcast();
1265 }
1266
1267 return std::min(f_ret, i_ret);
1268}
int DoIt(int argpnt, int argc, char **argv, bool singleshot)
#define TRACE_Debug
#define XrdOssOK
Definition XrdOss.hh:50
std::string obfuscateAuth(const std::string &input)
#define TRACE_Dump
#define TRACE_PC(act, pre_code, x)
#define TRACE_INT(act, x)
void * ProcessWriteTaskThread(void *)
Definition XrdPfc.cc:60
XrdSysXAttr * XrdSysXAttrActive
void * ResourceMonitorThread(void *)
Definition XrdPfc.cc:54
void * PrefetchThread(void *)
Definition XrdPfc.cc:66
XrdOucCache * XrdOucGetCache(XrdSysLogger *logger, const char *config_filename, const char *parameters, XrdOucEnv *env)
Definition XrdPfc.cc:76
#define stat(a, b)
Definition XrdPosix.hh:101
bool Debug
#define TRACE(act, x)
Definition XrdTrace.hh:63
URL representation.
Definition XrdClURL.hh:31
const std::string & GetPath() const
Get the path.
Definition XrdClURL.hh:217
virtual int Close(long long *retsz=0)=0
virtual int Open(const char *path, int Oflag, mode_t Mode, XrdOucEnv &env)
Definition XrdOss.hh:200
virtual int Fstat(struct stat &sbuff)
virtual const char * Path()=0
virtual const char * Location(bool refresh=false)
static const int optRW
File is read/write (o/w read/only)
XrdOucCache(const char *ctype)
void * GetPtr(const char *varname)
Definition XrdOucEnv.cc:281
void PutPtr(const char *varname, void *value)
Definition XrdOucEnv.cc:316
int get_size() const
long long m_offset
File * get_file() const
Attaches/creates and detaches/deletes cache-io objects for disk based cache.
Definition XrdPfc.hh:165
long long DetermineFullFileSize(const std::string &cinfo_fname)
Definition XrdPfc.cc:932
void FileSyncDone(File *, bool high_debug)
Definition XrdPfc.cc:548
File * GetFile(const std::string &, IO *, long long off=0, long long filesize=0)
Definition XrdPfc.cc:393
static const Configuration & Conf()
Definition XrdPfc.cc:134
virtual int LocalFilePath(const char *url, char *buff=0, int blen=0, LFP_Reason why=ForAccess, bool forall=false)
Definition XrdPfc.cc:800
virtual int Stat(const char *url, struct stat &sbuff)
Definition XrdPfc.cc:1128
const Configuration & RefConfiguration() const
Reference XrdPfc configuration.
Definition XrdPfc.hh:217
static ResourceMonitor & ResMon()
Definition XrdPfc.cc:135
bool IsFileActiveOrPurgeProtected(const std::string &) const
Definition XrdPfc.cc:683
void ClearPurgeProtectedSet()
Definition XrdPfc.cc:691
void ReleaseRAM(char *buf, long long size)
Definition XrdPfc.cc:375
virtual int ConsiderCached(const char *url)
Definition XrdPfc.cc:1006
static Cache & GetInstance()
Singleton access.
Definition XrdPfc.cc:132
bool Config(const char *config_filename, const char *parameters, XrdOucEnv *env)
Parse configuration file.
void DeRegisterPrefetchFile(File *)
Definition XrdPfc.cc:717
void ExecuteCommandUrl(const std::string &command_url)
void RegisterPrefetchFile(File *)
Definition XrdPfc.cc:701
void WriteFileSizeXAttr(int cinfo_fd, long long file_size)
Definition XrdPfc.cc:917
void Prefetch()
Definition XrdPfc.cc:758
void ReleaseFile(File *, IO *)
Definition XrdPfc.cc:477
void AddWriteTask(Block *b, bool from_read)
Add downloaded block in write queue.
Definition XrdPfc.cc:225
Cache(XrdSysLogger *logger, XrdOucEnv *env)
Constructor.
Definition XrdPfc.cc:158
bool Decide(XrdOucCacheIO *)
Makes decision if the original XrdOucCacheIO should be cached.
Definition XrdPfc.cc:137
int UnlinkFile(const std::string &f_name, bool fail_if_open)
Remove cinfo and data files from cache.
Definition XrdPfc.cc:1199
static XrdScheduler * schedP
Definition XrdPfc.hh:304
File * GetNextFileToPrefetch()
Definition XrdPfc.cc:739
long long WritesSinceLastCall()
Definition XrdPfc.cc:324
void ProcessWriteTasks()
Separate task which writes blocks from ram to disk.
Definition XrdPfc.cc:277
virtual int Unlink(const char *url)
Definition XrdPfc.cc:1189
void RemoveWriteQEntriesFor(File *f)
Remove blocks from write queue which belong to given prefetch. This method is used at the time of Fil...
Definition XrdPfc.cc:244
virtual XrdOucCacheIO * Attach(XrdOucCacheIO *, int Options=0)
Definition XrdPfc.cc:179
static const Cache & TheOne()
Definition XrdPfc.cc:133
char * RequestRAM(long long size)
Definition XrdPfc.cc:335
virtual int Prepare(const char *url, int oflags, mode_t mode)
Definition XrdPfc.cc:1072
bool DecideIfConsideredCached(long long file_size, long long bytes_on_disk)
Definition XrdPfc.cc:973
static Cache & CreateInstance(XrdSysLogger *logger, XrdOucEnv *env)
Singleton creation.
Definition XrdPfc.cc:125
bool is_prefetch_enabled() const
Definition XrdPfc.hh:309
Base class for selecting which files should be cached.
virtual bool Decide(const std::string &, XrdOss &) const =0
bool FinalizeSyncBeforeExit()
Returns true if any of blocks need sync. Called from Cache::dec_ref_cnt on zero ref cnt.
const char * lPath() const
Log path.
void WriteBlockToDisk(Block *b)
int GetNBlocks() const
std::string GetRemoteLocations() const
size_t GetAccessCnt() const
int Fstat(struct stat &sbuff)
void AddIO(IO *io)
static File * FileOpen(const std::string &path, long long offset, long long fileSize, XrdOucCacheIO *inputIO)
Static constructor that also does Open. Returns null ptr if Open fails.
long long GetPrefetchedBytes() const
int GetBlockSize() const
int GetNDownloadedBlocks() const
const Info::AStat * GetLastAccessStats() const
void BlocksRemovedFromWriteQ(std::list< Block * > &)
Handle removal of a set of blocks from Cache's write queue.
int inc_ref_cnt()
const Stats & RefStats() const
void Sync()
Sync file cache inf o and output data with disk.
int dec_ref_cnt()
int get_ref_cnt()
long long initiate_emergency_shutdown()
long long GetFileSize() const
const std::string & GetLocalPath() const
void RemoveIO(IO *io)
bool is_in_emergency_shutdown()
Downloads original file into multiple files, chunked into blocks. Only blocks that are asked for are ...
Downloads original file into a single file on local disk. Handles read requests as they come along.
bool HasFile() const
Check if File was opened successfully.
Base cache-io class that implements some XrdOucCacheIO abstract methods.
Definition XrdPfcIO.hh:16
XrdOucCacheIO * GetInput()
Definition XrdPfcIO.cc:31
Status of cached file. Can be read from and written into a binary file.
Definition XrdPfcInfo.hh:41
static const char * s_infoExtension
void WriteIOStatSingle(long long bytes_disk)
Write single open/close time for given bytes read from disk.
bool Write(XrdOssDF *fp, const char *dname, const char *fname=0)
bool IsComplete() const
Get complete status.
long long GetFileSize() const
Get file size.
bool Read(XrdOssDF *fp, const char *dname, const char *fname=0)
Read content of cinfo file into this object.
int m_NCksumErrors
number of checksum errors while getting data from remote
long long m_BytesWritten
number of bytes written to disk
void Say(const char *text1, const char *text2=0, const char *txt3=0, const char *text4=0, const char *text5=0, const char *txt6=0)
static int Run(pthread_t *, void *(*proc)(void *), void *arg, int opts=0, const char *desc=0)
static void Wait(int milliseconds)
XrdPosixStats Stats
Contains parameters configurable from the xrootd config file.
Definition XrdPfc.hh:64
long long BytesHit
read from cache
Definition XrdPfcInfo.hh:64
long long BytesBypassed
read from remote and dropped
Definition XrdPfcInfo.hh:66
time_t DetachTime
close time
Definition XrdPfcInfo.hh:59
long long BytesMissed
read from remote and cached
Definition XrdPfcInfo.hh:65
time_t AttachTime
open time
Definition XrdPfcInfo.hh:58