XRootD
XrdClCopy.cc
Go to the documentation of this file.
1 //------------------------------------------------------------------------------
2 // Copyright (c) 2011-2014 by European Organization for Nuclear Research (CERN)
3 // Author: Lukasz Janyst <ljanyst@cern.ch>
4 //------------------------------------------------------------------------------
5 // This file is part of the XRootD software suite.
6 //
7 // XRootD is free software: you can redistribute it and/or modify
8 // it under the terms of the GNU Lesser General Public License as published by
9 // the Free Software Foundation, either version 3 of the License, or
10 // (at your option) any later version.
11 //
12 // XRootD is distributed in the hope that it will be useful,
13 // but WITHOUT ANY WARRANTY; without even the implied warranty of
14 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 // GNU General Public License for more details.
16 //
17 // You should have received a copy of the GNU Lesser General Public License
18 // along with XRootD. If not, see <http://www.gnu.org/licenses/>.
19 //
20 // In applying this licence, CERN does not waive the privileges and immunities
21 // granted to it by virtue of its status as an Intergovernmental Organization
22 // or submit itself to any jurisdiction.
23 //------------------------------------------------------------------------------
24 
25 #include "XrdApps/XrdCpConfig.hh"
26 #include "XrdApps/XrdCpFile.hh"
27 #include "XrdCl/XrdClConstants.hh"
29 #include "XrdCl/XrdClDefaultEnv.hh"
30 #include "XrdCl/XrdClLog.hh"
31 #include "XrdCl/XrdClFileSystem.hh"
32 #include "XrdCl/XrdClUtils.hh"
33 #include "XrdCl/XrdClDlgEnv.hh"
34 #include "XrdCl/XrdClOptimizers.hh"
35 #include "XrdSys/XrdSysE2T.hh"
36 #include "XrdSys/XrdSysPthread.hh"
38 
39 #include <cstdio>
40 #include <iostream>
41 #include <iomanip>
42 #include <limits>
43 #include <string>
44 
45 using namespace std::string_literals;
46 
47 //------------------------------------------------------------------------------
48 // Progress notifier
49 //------------------------------------------------------------------------------
51 {
52  public:
53  //--------------------------------------------------------------------------
55  //--------------------------------------------------------------------------
56  ProgressDisplay(): pPrevious(0), pPrintProgressBar(true),
57  pPrintSourceCheckSum(false), pPrintTargetCheckSum(false),
58  pPrintAdditionalCheckSum(false)
59  {}
60 
61  //--------------------------------------------------------------------------
63  //--------------------------------------------------------------------------
64  virtual void BeginJob( uint32_t jobNum,
65  uint32_t jobTotal,
66  const XrdCl::URL *source,
67  const XrdCl::URL *destination )
68  {
69  XrdSysMutexHelper scopedLock( pMutex );
70  if( pPrintProgressBar )
71  {
72  if( jobTotal > 1 )
73  {
74  std::cerr << "Job: " << jobNum << "/" << jobTotal << std::endl;
75  std::cerr << "Source: " << source->GetURL() << std::endl;
76  std::cerr << "Target: " << destination->GetURL() << std::endl;
77  }
78  }
79  pPrevious = 0;
80 
81  JobData d;
82  d.started = time(0);
83  d.source = source;
84  d.target = destination;
85  pOngoingJobs[jobNum] = d;
86  }
87 
88  //--------------------------------------------------------------------------
90  //--------------------------------------------------------------------------
91  virtual void EndJob( uint32_t jobNum, const XrdCl::PropertyList *results )
92  {
93  XrdSysMutexHelper scopedLock( pMutex );
94 
95  std::map<uint32_t, JobData>::iterator it = pOngoingJobs.find( jobNum );
96  if( it == pOngoingJobs.end() )
97  return;
98 
99  JobData &d = it->second;
100 
101  // make sure the last available status was printed, which may not be
102  // the case when processing stdio since we throttle printing and don't
103  // know the total size
104  JobProgress( jobNum, d.bytesProcessed, d.bytesTotal );
105 
106  if( pPrintProgressBar )
107  {
108  if( pOngoingJobs.size() > 1 )
109  std::cerr << "\r" << std::string(70, ' ') << "\r";
110  else
111  std::cerr << std::endl;
112  }
113 
115  results->Get( "status", st );
116  if( !st.IsOK() )
117  {
118  pOngoingJobs.erase(it);
119  return;
120  }
121 
122  std::string checkSum;
123  uint64_t size;
124  results->Get( "size", size );
125  if( pPrintSourceCheckSum )
126  {
127  results->Get( "sourceCheckSum", checkSum );
128  PrintCheckSum( d.source, checkSum, size );
129  }
130 
131  if( pPrintTargetCheckSum )
132  {
133  results->Get( "targetCheckSum", checkSum );
134  PrintCheckSum( d.target, checkSum, size );
135  }
136 
137  if( pPrintAdditionalCheckSum )
138  {
139  std::vector<std::string> addcksums;
140  results->Get( "additionalCkeckSum", addcksums );
141  for( auto &cks : addcksums )
142  PrintCheckSum( d.source, cks, size );
143  }
144 
145  pOngoingJobs.erase(it);
146  }
147 
148  //--------------------------------------------------------------------------
150  //--------------------------------------------------------------------------
151  std::string GetProgressBar( time_t now )
152  {
153  JobData &d = pOngoingJobs.begin()->second;
154 
155  uint64_t speed = 0;
156  if( now-d.started )
157  speed = d.bytesProcessed/(now-d.started);
158  else
159  speed = d.bytesProcessed;
160 
161  std::string bar;
162  int prog = 0;
163  int proc = 0;
164 
165  if( d.bytesTotal )
166  {
167  prog = (int)((double)d.bytesProcessed/d.bytesTotal*50);
168  proc = (int)((double)d.bytesProcessed/d.bytesTotal*100);
169  }
170  else
171  {
172  prog = 50;
173  proc = 100;
174  }
175  bar.append( prog, '=' );
176  if( prog < 50 )
177  bar += ">";
178 
179  std::ostringstream o;
180  o << "[" << XrdCl::Utils::BytesToString(d.bytesProcessed) << "B/";
181  o << XrdCl::Utils::BytesToString(d.bytesTotal) << "B]";
182  o << "[" << std::setw(3) << std::right << proc << "%]";
183  o << "[" << std::setw(50) << std::left;
184  o << bar;
185  o << "]";
186  o << "[" << XrdCl::Utils::BytesToString(speed) << "B/s] ";
187  return o.str();
188  }
189 
190  //--------------------------------------------------------------------------
192  //--------------------------------------------------------------------------
193  std::string GetSummaryBar( time_t now )
194  {
195  std::map<uint32_t, JobData>::iterator it;
196  std::ostringstream o;
197 
198  for( it = pOngoingJobs.begin(); it != pOngoingJobs.end(); ++it )
199  {
200  JobData &d = it->second;
201  uint32_t jobNum = it->first;
202 
203  uint64_t speed = 0;
204  if( now-d.started )
205  speed = d.bytesProcessed/(now-d.started);
206 
207  int proc = 0;
208  if( d.bytesTotal )
209  proc = (int)((double)d.bytesProcessed/d.bytesTotal*100);
210  else
211  proc = 100;
212 
213  o << "[#" << jobNum << ": ";
214  o << proc << "% ";
215  o << XrdCl::Utils::BytesToString(speed) << "B/s] ";
216  }
217  o << " ";
218  return o.str();
219  }
220 
221  //--------------------------------------------------------------------------
223  //--------------------------------------------------------------------------
224  virtual void JobProgress( uint32_t jobNum,
225  uint64_t bytesProcessed,
226  uint64_t bytesTotal )
227  {
228  XrdSysMutexHelper scopedLock( pMutex );
229 
230  if( pPrintProgressBar )
231  {
232  time_t now = time(0);
233  if( (now - pPrevious < 1) && (bytesProcessed != bytesTotal) )
234  return;
235  pPrevious = now;
236 
237  std::map<uint32_t, JobData>::iterator it = pOngoingJobs.find( jobNum );
238  if( it == pOngoingJobs.end() )
239  return;
240 
241  JobData &d = it->second;
242 
243  d.bytesProcessed = bytesProcessed;
244  d.bytesTotal = bytesTotal;
245 
246  std::string progress;
247  if( pOngoingJobs.size() == 1 )
248  progress = GetProgressBar( now );
249  else
250  progress = GetSummaryBar( now );
251 
252  std::cerr << "\r" << progress << std::flush;
253  }
254  }
255 
256  //--------------------------------------------------------------------------
258  //--------------------------------------------------------------------------
259  void PrintCheckSum( const XrdCl::URL *url,
260  const std::string &checkSum,
261  uint64_t size )
262  {
263  if( checkSum.empty() )
264  return;
265  std::string::size_type i = checkSum.find( ':' );
266  std::string type = checkSum.substr( 0, i );
267  std::string value = checkSum.substr( i+1, checkSum.length()-i );
268 
269  //------------------------------------------------------------------------
270  // The comparison uses adler32 and crc32 without leading zeros (see
271  // Utils::NormalizeChecksum), but print all 8 digits, as xrdadler32 and
272  // the servers do
273  //------------------------------------------------------------------------
274  if( ( type == "adler32" || type == "crc32" ) && value.length() < 8 )
275  value.insert( 0, 8 - value.length(), '0' );
276 
277  std::cerr << type << ": " << value << " ";
278 
279  if( url->IsLocalFile() )
280  std::cerr << url->GetPath() << " ";
281  else
282  {
283  std::cerr << url->GetProtocol() << "://" << url->GetHostId();
284  std::cerr << url->GetPath() << " ";
285  }
286 
287  std::cerr << size;
288  std::cerr << std::endl;
289  }
290 
291  //--------------------------------------------------------------------------
292  // Printing flags
293  //--------------------------------------------------------------------------
294  void PrintProgressBar( bool print ) { pPrintProgressBar = print; }
295  void PrintSourceCheckSum( bool print ) { pPrintSourceCheckSum = print; }
296  void PrintTargetCheckSum( bool print ) { pPrintTargetCheckSum = print; }
297  void PrintAdditionalCheckSum( bool print ) { pPrintAdditionalCheckSum = print; }
298 
299  private:
300  struct JobData
301  {
302  JobData(): bytesProcessed(0), bytesTotal(0),
303  started(0), source(0), target(0) {}
304  uint64_t bytesProcessed;
305  uint64_t bytesTotal;
306  time_t started;
307  const XrdCl::URL *source;
308  const XrdCl::URL *target;
309  };
310 
311  time_t pPrevious;
312  bool pPrintProgressBar;
313  bool pPrintSourceCheckSum;
314  bool pPrintTargetCheckSum;
315  bool pPrintAdditionalCheckSum;
316  std::map<uint32_t, JobData> pOngoingJobs;
317  XrdSysRecMutex pMutex;
318 };
319 
320 //------------------------------------------------------------------------------
321 // Check if we support all the specified user options
322 //------------------------------------------------------------------------------
324 {
325  if( config->pHost )
326  {
327  std::cerr << "SOCKS Proxies are not yet supported" << std::endl;
328  return false;
329  }
330 
331  return true;
332 }
333 
334 //------------------------------------------------------------------------------
335 // Append extra cgi info to existing URL
336 //------------------------------------------------------------------------------
337 void AppendCGI( std::string &url, const char *newCGI )
338 {
339  if( !newCGI || !(*newCGI) )
340  return;
341 
342  if( *newCGI == '&' )
343  ++newCGI;
344 
345  if( url.find( '?' ) == std::string::npos )
346  url += "?";
347 
348  if( url.find( '&' ) == std::string::npos )
349  url += "&";
350 
351  url += newCGI;
352 }
353 
354 //------------------------------------------------------------------------------
355 // Process commandline environment settings
356 //------------------------------------------------------------------------------
358 {
360 
361  XrdCpConfig::defVar *cursor = config->intDefs;
362  while( cursor )
363  {
364  env->PutInt( cursor->vName, cursor->intVal );
365  cursor = cursor->Next;
366  }
367 
368  cursor = config->strDefs;
369  while( cursor )
370  {
371  env->PutString( cursor->vName, cursor->strVal );
372  cursor = cursor->Next;
373  }
374 }
375 
376 //------------------------------------------------------------------------------
377 // Translate file type to a string for diagnostics purposes
378 //------------------------------------------------------------------------------
380 {
381  switch( type )
382  {
383  case XrdCpFile::isDir: return "directory";
384  case XrdCpFile::isFile: return "local file";
385  case XrdCpFile::isXroot: return "xroot";
386  case XrdCpFile::isXroots: return "xroots";
387  case XrdCpFile::isHttp: return "http";
388  case XrdCpFile::isHttps: return "https";
389  case XrdCpFile::isStdIO: return "stdio";
390  default: return "other";
391  };
392 }
393 
394 //------------------------------------------------------------------------------
395 // Count the sources
396 //------------------------------------------------------------------------------
397 uint32_t CountSources( XrdCpFile *file )
398 {
399  uint32_t count;
400  for( count = 0; file; file = file->Next, ++count ) {};
401  return count;
402 }
403 
404 //------------------------------------------------------------------------------
405 // Adjust file information for the cases when XrdCpConfig cannot do this
406 //------------------------------------------------------------------------------
408 {
409  //----------------------------------------------------------------------------
410  // If the file is url and the directory offset is not set we set it
411  // to the last slash
412  //----------------------------------------------------------------------------
413  if( file->Doff == 0 )
414  {
415  char *slash = file->Path;
416  for( ; *slash; ++slash ) {};
417  for( ; *slash != '/' && slash > file->Path; --slash ) {};
418  file->Doff = slash - file->Path;
419  }
420 };
421 
422 //------------------------------------------------------------------------------
423 // Recursively index all files and directories inside a remote directory
424 //------------------------------------------------------------------------------
426  std::string basePath,
427  long dirOffset )
428 {
429  using namespace XrdCl;
430 
431  Log *log = DefaultEnv::GetLog();
432  log->Debug( AppMsg, "Indexing %s", basePath.c_str() );
433 
434  DirectoryList *dirList = 0;
435  XRootDStatus st = fs->DirList( URL( basePath ).GetPath(), DirListFlags::Recursive
436  | DirListFlags::Locate | DirListFlags::Merge, dirList );
437  if( !st.IsOK() )
438  {
439  log->Info( AppMsg, "Failed to get directory listing for %s: %s",
440  basePath.c_str(),
441  st.GetErrorMessage().c_str() );
442  return 0;
443  }
444 
445  XrdCpFile start, *current = 0;
446  XrdCpFile *end = &start;
447  int badUrl = 0;
448  for( auto itr = dirList->Begin(); itr != dirList->End(); ++itr )
449  {
450  DirectoryList::ListEntry *e = *itr;
451  if( e->GetStatInfo()->TestFlags( StatInfo::IsDir ) )
452  continue;
453  std::string path = basePath + '/' + e->GetName();
454  current = new XrdCpFile( path.c_str(), badUrl );
455  if( badUrl )
456  {
457  log->Error( AppMsg, "Bad URL: %s", current->Path );
458  delete current;
459  return 0;
460  }
461 
462  current->Doff = dirOffset;
463  end->Next = current;
464  end = current;
465  }
466 
467  delete dirList;
468 
469  return start.Next;
470 }
471 
472 //------------------------------------------------------------------------------
473 // Clean up the copy job descriptors
474 //------------------------------------------------------------------------------
475 void CleanUpResults( std::vector<XrdCl::PropertyList *> &results )
476 {
477  std::vector<XrdCl::PropertyList *>::iterator it;
478  for( it = results.begin(); it != results.end(); ++it )
479  delete *it;
480 }
481 
482 //--------------------------------------------------------------------------
483 // Let the show begin
484 //------------------------------------------------------------------------------
485 int main( int argc, char **argv )
486 {
487  using namespace XrdCl;
488 
489  //----------------------------------------------------------------------------
490  // Configure the copy command, if it returns then everything went well, ugly
491  //----------------------------------------------------------------------------
492  XrdCpConfig config( argv[0] );
493  config.Config( argc, argv, XrdCpConfig::optRmtRec );
494  if( !AllOptionsSupported( &config ) )
495  return 50; // generic error
496  ProcessCommandLineEnv( &config );
497 
498  //----------------------------------------------------------------------------
499  // Set options
500  //----------------------------------------------------------------------------
501  CopyProcess process;
502  Log *log = DefaultEnv::GetLog();
503  if( config.Dlvl )
504  {
505  if( config.Dlvl == 1 ) log->SetLevel( Log::InfoMsg );
506  else if( config.Dlvl == 2 ) log->SetLevel( Log::DebugMsg );
507  else if( config.Dlvl == 3 ) log->SetLevel( Log::DumpMsg );
508  }
509 
510  ProgressDisplay progress;
511  if( config.Want(XrdCpConfig::DoNoPbar) || !isatty( fileno( stdout ) ) )
512  progress.PrintProgressBar( false );
513 
514  bool posc = false;
515  bool force = false;
516  bool coerce = false;
517  bool makedir = false;
518  bool dynSrc = false;
519  bool delegate = false;
520  bool preserveXAttr = false;
521  bool rmOnBadCksum = false;
522  bool continue_ = false;
523  bool zipappend = false;
524  bool doserver = false;
525  std::string thirdParty = "none"s;
526  std::string thirdPartyMode = "pull"s;
527 
528  const std::string thirdPartyTokenFile = config.tcpTokenFile ? config.tcpTokenFile : ""s;
529 
530  if( config.Want( XrdCpConfig::DoPosc ) ) posc = true;
531  if( config.Want( XrdCpConfig::DoForce ) ) force = true;
532  if( config.Want( XrdCpConfig::DoCoerce ) ) coerce = true;
533  if( config.Want( XrdCpConfig::DoTpc ) ) thirdParty = "first"s;
534  if( config.Want( XrdCpConfig::DoTpcOnly ) ) thirdParty = "only"s;
535  if( config.Want( XrdCpConfig::DoTpcModePush ) ) thirdPartyMode = "push"s;
536  if( config.Want( XrdCpConfig::DoZipAppend ) ) zipappend = true;
537  if( config.Want( XrdCpConfig::DoServer ) ) doserver = true;
538  if( config.Want( XrdCpConfig::DoTpcDlgt ) )
539  {
540  // the env var is being set already here (we are issuing a stat
541  // inhere and we need the env var when we are establishing the
542  // connection and authenticating), but we are also setting a delegate
543  // parameter for CopyJob so it can be used on its own.
544  DlgEnv::Instance().Enable();
545  delegate = true;
546  }
547  else
548  DlgEnv::Instance().Disable();
549 
550  if( config.Want( XrdCpConfig::DoRecurse ) ) makedir = true;
551  if( config.Want( XrdCpConfig::DoPath ) ) makedir = true;
552  if( config.Want( XrdCpConfig::DoDynaSrc ) ) dynSrc = true;
553  if( config.Want( XrdCpConfig::DoXAttr ) ) preserveXAttr = true;
554  if( config.Want( XrdCpConfig::DoRmOnBadCksum ) ) rmOnBadCksum = true;
555  if( config.Want( XrdCpConfig::DoContinue ) ) continue_ = true;
556 
557  if( force && continue_ )
558  {
559  std::cerr << "Invalid argument combination: continue + force." << std::endl;
560  return 50;
561  }
562 
563  //----------------------------------------------------------------------------
564  // Checksums
565  //----------------------------------------------------------------------------
566  std::string checkSumType;
567  std::string checkSumPreset;
568  std::string checkSumMode = "none";
569  if( config.Want( XrdCpConfig::DoCksum ) )
570  {
571  checkSumMode = "end2end";
572  std::vector<std::string> ckSumParams;
573  Utils::splitString( ckSumParams, config.CksVal, ":" );
574  if( ckSumParams.size() > 1 )
575  {
576  if( ckSumParams[1] == "print" )
577  {
578  checkSumMode = "target";
579  progress.PrintTargetCheckSum( true );
580  }
581  else
582  checkSumPreset = ckSumParams[1];
583  }
584  checkSumType = ckSumParams[0];
585  }
586 
587  if( config.Want( XrdCpConfig::DoCksrc ) )
588  {
589  checkSumMode = "source";
590  std::vector<std::string> ckSumParams;
591  Utils::splitString( ckSumParams, config.CksVal, ":" );
592  if( ckSumParams.size() == 2 )
593  {
594  checkSumMode = "source";
595  checkSumType = ckSumParams[0];
596  progress.PrintSourceCheckSum( true );
597  }
598  else
599  {
600  std::cerr << "Invalid parameter: " << config.CksVal << std::endl;
601  return 50; // generic error
602  }
603  }
604 
605  if( !config.AddCksVal.empty() )
606  progress.PrintAdditionalCheckSum( true );
607 
608  //----------------------------------------------------------------------------
609  // ZIP archive
610  //----------------------------------------------------------------------------
611  std::string zipFile;
612  bool zip = false;
613  if( config.Want( XrdCpConfig::DoZip ) )
614  {
615  zipFile = config.zipFile;
616  zip = true;
617  }
618 
619  //----------------------------------------------------------------------------
620  // Extreme Copy
621  //----------------------------------------------------------------------------
622  int nbSources = 0;
623  bool xcp = false;
624  if( config.Want( XrdCpConfig::DoSources ) )
625  {
626  nbSources = config.nSrcs;
627  xcp = true;
628  }
629 
630  //----------------------------------------------------------------------------
631  // Environment settings
632  //----------------------------------------------------------------------------
634 
635  /* Stop PostMaster when exiting main() to ensure proper shutdown */
636  struct scope_exit {
637  ~scope_exit() { XrdCl::DefaultEnv::GetPostMaster()->Stop(); }
638  } stopPostMaster;
639 
640  if( config.nStrm != 0 )
641  env->PutInt( "SubStreamsPerChannel", config.nStrm + 1 /*stands for the control stream*/ );
642 
643  if( config.Retry != -1 )
644  {
645  env->PutInt( "CpRetry", config.Retry );
646  env->PutString( "CpRetryPolicy", config.RetryPolicy );
647  }
648 
649  if( config.Want( XrdCpConfig::DoNoTlsOK ) )
650  env->PutInt( "NoTlsOK", 1 );
651 
652  if( config.Want( XrdCpConfig::DoTlsNoData ) )
653  env->PutInt( "TlsNoData", 1 );
654 
655  if( config.Want( XrdCpConfig::DoTlsMLF ) )
656  env->PutInt( "TlsMetalink", 1 );
657 
658  if( config.Want( XrdCpConfig::DoZipMtlnCksum ) )
659  env->PutInt( "ZipMtlnCksum", 1 );
660 
661  //----------------------------------------------------------------------------
662  // Headers to inject into the HTTP requests. These are handed to the XrdClHttp
663  // plug-in verbatim, as a newline separated list; that plug-in owns all of the
664  // parsing and validation of their contents.
665  //----------------------------------------------------------------------------
666  if( !config.HttpHeaders.empty() )
667  {
668  std::string headers;
669  for( auto &header : config.HttpHeaders )
670  {
671  if( !headers.empty() ) headers += '\n';
672  headers += header;
673  }
674  env->PutString( "HttpHeaders", headers );
675  }
676 
677  int chunkSize = DefaultCPChunkSize;
678  env->GetInt( "CPChunkSize", chunkSize );
679 
680  int blockSize = DefaultXCpBlockSize;
681  env->GetInt( "XCpBlockSize", blockSize );
682 
683  int parallelChunks = DefaultCPParallelChunks;
684  env->GetInt( "CPParallelChunks", parallelChunks );
685  if( parallelChunks < 1 ||
686  parallelChunks > std::numeric_limits<uint8_t>::max() )
687  {
688  std::cerr << "Can only handle between 1 and ";
689  std::cerr << (int)std::numeric_limits<uint8_t>::max();
690  std::cerr << " chunks in parallel. You asked for " << parallelChunks;
691  std::cerr << "." << std::endl;
692  return 50; // generic error
693  }
694 
695  if( !preserveXAttr )
696  {
697  int val = DefaultPreserveXAttrs;
698  env->GetInt( "PreserveXAttrs", val );
699  if( val ) preserveXAttr = true;
700  }
701 
702  log->Dump( AppMsg, "Chunk size: %d, parallel chunks %d, streams: %d",
703  chunkSize, parallelChunks, config.nStrm + 1 );
704 
705  //----------------------------------------------------------------------------
706  // Build the URLs
707  //----------------------------------------------------------------------------
708  std::vector<XrdCl::PropertyList*> resultVect;
709 
710  std::string dest;
711  if( config.dstFile->Protocol == XrdCpFile::isDir ||
712  config.dstFile->Protocol == XrdCpFile::isFile )
713  {
714  dest = "file://";
715 
716  // if it is not an absolute path append cwd
717  if( config.dstFile->Path[0] != '/' )
718  {
719  char buf[FILENAME_MAX];
720  char *cwd = getcwd( buf, FILENAME_MAX );
721  if( !cwd )
722  {
723  XRootDStatus st( stError, XProtocol::mapError( errno ), errno );
724  std::cerr << st.GetErrorMessage() << std::endl;
725  return st.GetShellCode();
726  }
727  dest += cwd;
728  dest += '/';
729  }
730  }
731  dest += config.dstFile->Path;
732 
733  //----------------------------------------------------------------------------
734  // A destination that ends in / is a directory. Several sources are allowed,
735  // and the source file name is added. root://host/ loses that slash when the
736  // address is parsed, so the slash is read from the text the user typed.
737  //----------------------------------------------------------------------------
738  bool targetIsDir = config.dstFile->Protocol == XrdCpFile::isDir;
739  if( !targetIsDir &&
740  ( config.dstFile->Protocol == XrdCpFile::isXroot ||
741  config.dstFile->Protocol == XrdCpFile::isXroots ||
742  config.dstFile->Protocol == XrdCpFile::isHttp ||
743  config.dstFile->Protocol == XrdCpFile::isHttps ||
744  config.dstFile->Protocol == XrdCpFile::isS3 ) )
745  {
746  URL target( dest );
747  const std::string path = target.GetPath();
748  if( !path.empty() )
749  targetIsDir = path.back() == '/';
750  else
751  {
752  const std::string spec = config.dstFile->Path;
753  const size_t query = spec.find( '?' );
754  const size_t end = query == std::string::npos ? spec.size() : query;
755  targetIsDir = end > 0 && spec[end - 1] == '/';
756  }
757  }
758 
759  // A local directory has already been replaced by the files inside it.
760  // The destination of that copy is a directory.
761  if( !targetIsDir && config.Want( XrdCpConfig::DoRecurse ) )
762  {
763  for( XrdCpFile *file = config.srcFile; file; file = file->Next )
764  {
765  if( strncmp( file->ProtName, "file", 4 ) == 0 &&
766  std::string( file->Path ).size() == size_t( file->Doff + file->Dlen ) )
767  {
768  targetIsDir = true;
769  break;
770  }
771  }
772  }
773 
774  // root:// and roots:// may already be a directory even when the path does
775  // not end in /. Ask the server, and add the source file name if it is.
776  // http://, https://, and s3:// are directories only when the path ends in /.
777  if( !targetIsDir &&
778  ( config.dstFile->Protocol == XrdCpFile::isXroot ||
779  config.dstFile->Protocol == XrdCpFile::isXroots ) )
780  {
781  URL target( dest );
782  FileSystem fs( target );
783  StatInfo *statInfo = 0;
784  XRootDStatus st = fs.Stat( target.GetPathWithParams(), statInfo );
785  if( st.IsOK() && statInfo && statInfo->TestFlags( StatInfo::IsDir ) )
786  targetIsDir = true;
787  else if( st.errNo == kXR_NotAuthorized )
788  {
789  log->Error( AppMsg, "%s (destination)", st.ToString().c_str() );
790  std::cerr << st.ToStr() << std::endl;
791  delete statInfo;
792  return st.GetShellCode();
793  }
794  delete statInfo;
795  }
796 
797  //----------------------------------------------------------------------------
798  // If we have multiple sources and target is neither a directory nor stdout
799  // then we cannot proceed
800  //----------------------------------------------------------------------------
801  if( CountSources(config.srcFile) > 1 && !targetIsDir &&
802  config.dstFile->Protocol != XrdCpFile::isStdIO )
803  {
804  std::cerr << "Multiple sources were given but target is not a directory.";
805  std::cerr << std::endl;
806  return 50; // generic error
807  }
808 
809  //----------------------------------------------------------------------------
810  // If we're doing remote recursive copy, chain all the files (if it's a
811  // directory)
812  //----------------------------------------------------------------------------
813  bool remoteSrcIsDir = false;
814  if( config.Want( XrdCpConfig::DoRecurse ) &&
815  (config.srcFile->Protocol == XrdCpFile::isXroot ||
816  config.srcFile->Protocol == XrdCpFile::isXroots) )
817  {
818  URL source( config.srcFile->Path );
819  FileSystem *fs = new FileSystem( source );
820  StatInfo *statInfo = 0;
821 
822  XRootDStatus st = fs->Stat( source.GetPath(), statInfo );
823  if( st.IsOK() && statInfo->TestFlags( StatInfo::IsDir ) )
824  {
825  remoteSrcIsDir = true;
826  // Each listed file name is added to this destination.
827  targetIsDir = true;
828  //------------------------------------------------------------------------
829  // Recursively index the remote directory
830  //------------------------------------------------------------------------
831  delete config.srcFile;
832  std::string url = source.GetURL();
833  config.srcFile = IndexRemote( fs, url, url.size() );
834  if ( !config.srcFile )
835  {
836  std::cerr << "Error indexing remote directory.";
837  return 50; // generic error
838  }
839  }
840 
841  delete fs;
842  delete statInfo;
843  }
844 
845  XrdCpFile *sourceFile = config.srcFile;
846  //----------------------------------------------------------------------------
847  // Process the sources
848  //----------------------------------------------------------------------------
849  while( sourceFile )
850  {
851  AdjustFileInfo( sourceFile );
852 
853  //--------------------------------------------------------------------------
854  // Create a job for every source
855  //--------------------------------------------------------------------------
856  PropertyList properties;
857  PropertyList *results = new PropertyList;
858  std::string source = sourceFile->Path;
859  if( sourceFile->Protocol == XrdCpFile::isFile )
860  {
861  // make sure it is an absolute path
862  if( source[0] == '/' )
863  source = "file://" + source;
864  else
865  {
866  char buf[FILENAME_MAX];
867  char *cwd = getcwd( buf, FILENAME_MAX );
868  if( !cwd )
869  {
870  XRootDStatus st( stError, XProtocol::mapError( errno ), errno );
871  std::cerr << st.GetErrorMessage() << std::endl;
872  return st.GetShellCode();
873  }
874  source = "file://" + std::string( cwd ) + '/' + source;
875  }
876  }
877 
878  AppendCGI( source, config.srcOpq );
879 
880  std::string sourcePathObf = sourceFile->Path;
881  std::string destPathObf = dest;
882  if( unlikely(log->GetLevel() >= Log::DumpMsg) ) {
883  sourcePathObf = obfuscateAuth(sourcePathObf);
884  destPathObf = obfuscateAuth(destPathObf);
885  }
886  log->Dump( AppMsg, "Processing source entry: %s, type %s, target file: %s, logLevel = %d",
887  sourcePathObf.c_str(), FileType2String( sourceFile->Protocol ),
888  destPathObf.c_str(), log->GetLevel() );
889 
890  //--------------------------------------------------------------------------
891  // Set up the job
892  //--------------------------------------------------------------------------
893  std::string target = dest;
894 
895 
896  bool srcIsDir = false;
897  // if this is local file, for a directory Dlen + Doff will overlap with path size
898  if( strncmp( sourceFile->ProtName, "file", 4 ) == 0 )
899  srcIsDir = std::string( sourceFile->Path ).size() == size_t( sourceFile->Doff + sourceFile->Dlen );
900  // otherwise we are handling a remote file
901  else
902  srcIsDir = remoteSrcIsDir;
903  // if this is a recursive copy make sure we preserve the directory structure
904  if( config.Want( XrdCpConfig::DoRecurse ) && srcIsDir )
905  {
906  // get the source directory
907  std::string srcDir( sourceFile->Path, sourceFile->Doff );
908  // remove the trailing slash
909  if( !srcDir.empty() && srcDir.back() == '/' )
910  srcDir.pop_back();
911  size_t diroff = srcDir.rfind( '/' );
912  // A relative name has no slash, so the whole name is kept. Otherwise move
913  // past the slash in front of the directory name. Keeping it would write
914  // two slashes into the destination.
915  if( diroff == std::string::npos )
916  diroff = 0;
917  else
918  ++diroff;
919  if( target.empty() || target.back() != '/' )
920  target += '/';
921  target += sourceFile->Path + diroff;
922  // remove the filename from destination path as it will be appended later anyway
923  target = target.substr( 0 , target.rfind('/') );
924  }
925  AppendCGI( target, config.dstOpq );
926 
927  properties.Set( "source", source );
928  properties.Set( "target", target );
929  properties.Set( "force", force );
930  properties.Set( "posc", posc );
931  properties.Set( "coerce", coerce );
932  properties.Set( "makeDir", makedir );
933  properties.Set( "dynamicSource", dynSrc );
934  properties.Set( "thirdParty", thirdParty );
935  properties.Set( "thirdPartyMode", thirdPartyMode );
936  properties.Set( "checkSumMode", checkSumMode );
937  properties.Set( "checkSumType", checkSumType );
938  properties.Set( "checkSumPreset", checkSumPreset );
939  properties.Set( "chunkSize", chunkSize );
940  properties.Set( "parallelChunks", parallelChunks );
941  properties.Set( "zipArchive", zip );
942  properties.Set( "xcp", xcp );
943  properties.Set( "xcpBlockSize", blockSize );
944  properties.Set( "delegate", delegate );
945  properties.Set( "targetIsDir", targetIsDir );
946  properties.Set( "preserveXAttr", preserveXAttr );
947  properties.Set( "xrate", config.xRate );
948  properties.Set( "xrateThreshold", config.xRateThreshold );
949  properties.Set( "rmOnBadCksum", rmOnBadCksum );
950  properties.Set( "continue", continue_ );
951  properties.Set( "zipAppend", zipappend );
952  properties.Set( "addcksums", config.AddCksVal );
953  properties.Set( "doServer", doserver );
954 
955  if( zip )
956  properties.Set( "zipSource", zipFile );
957 
958  if( xcp )
959  properties.Set( "nbXcpSources", nbSources );
960 
961  if( !thirdPartyTokenFile.empty() )
962  properties.Set( "thirdPartyTokenFile", thirdPartyTokenFile );
963 
964  XRootDStatus st = process.AddJob( properties, results );
965  if( !st.IsOK() )
966  {
967  std::cerr << "AddJob " << source << " -> " << target << ": ";
968  std::cerr << st.ToStr() << std::endl;
969  }
970  resultVect.push_back( results );
971  sourceFile = sourceFile->Next;
972  }
973 
974  //----------------------------------------------------------------------------
975  // Configure the copy process
976  //----------------------------------------------------------------------------
977  PropertyList processConfig;
978  processConfig.Set( "jobType", "configuration" );
979  processConfig.Set( "parallel", config.Parallel );
980  process.AddJob( processConfig, 0 );
981 
982  //----------------------------------------------------------------------------
983  // Prepare and run the copy process
984  //----------------------------------------------------------------------------
985  XRootDStatus st = process.Prepare();
986  if( !st.IsOK() )
987  {
988  CleanUpResults( resultVect );
989  std::cerr << "Prepare: " << st.ToStr() << std::endl;
990  return st.GetShellCode();
991  }
992 
993  st = process.Run( &progress );
994  if( !st.IsOK() )
995  {
996  if( resultVect.size() == 1 )
997  std::cerr << "Run: " << st.ToStr() << std::endl;
998  else
999  {
1000  std::vector<XrdCl::PropertyList*>::iterator it;
1001  uint32_t i = 1;
1002  uint32_t jobsRun = 0;
1003  uint32_t errors = 0;
1004  for( it = resultVect.begin(); it != resultVect.end(); ++it, ++i )
1005  {
1006  if( !(*it)->HasProperty( "status" ) )
1007  continue;
1008 
1009  XRootDStatus st = (*it)->Get<XRootDStatus>("status");
1010  if( !st.IsOK() )
1011  {
1012  std::cerr << "Job #" << i << ": " << st.ToStr();
1013  ++errors;
1014  }
1015  ++jobsRun;
1016  }
1017  std::cerr << "Jobs total: " << resultVect.size();
1018  std::cerr << ", run: " << jobsRun;
1019  std::cerr << ", errors: " << errors << std::endl;
1020  }
1021  CleanUpResults( resultVect );
1022  return st.GetShellCode();
1023  }
1024  CleanUpResults( resultVect );
1025  return 0;
1026 }
1027 
@ kXR_NotAuthorized
Definition: XProtocol.hh:1042
bool AllOptionsSupported(XrdCpConfig *config)
Definition: XrdClCopy.cc:323
const char * FileType2String(XrdCpFile::PType type)
Definition: XrdClCopy.cc:379
int main(int argc, char **argv)
Definition: XrdClCopy.cc:485
XrdCpFile * IndexRemote(XrdCl::FileSystem *fs, std::string basePath, long dirOffset)
Definition: XrdClCopy.cc:425
void ProcessCommandLineEnv(XrdCpConfig *config)
Definition: XrdClCopy.cc:357
void CleanUpResults(std::vector< XrdCl::PropertyList * > &results)
Definition: XrdClCopy.cc:475
void AdjustFileInfo(XrdCpFile *file)
Definition: XrdClCopy.cc:407
uint32_t CountSources(XrdCpFile *file)
Definition: XrdClCopy.cc:397
void AppendCGI(std::string &url, const char *newCGI)
Definition: XrdClCopy.cc:337
#define unlikely(x)
std::string obfuscateAuth(const std::string &input)
void PrintAdditionalCheckSum(bool print)
Definition: XrdClCopy.cc:297
void PrintSourceCheckSum(bool print)
Definition: XrdClCopy.cc:295
void PrintProgressBar(bool print)
Definition: XrdClCopy.cc:294
void PrintCheckSum(const XrdCl::URL *url, const std::string &checkSum, uint64_t size)
Print the checksum.
Definition: XrdClCopy.cc:259
virtual void BeginJob(uint32_t jobNum, uint32_t jobTotal, const XrdCl::URL *source, const XrdCl::URL *destination)
Begin job.
Definition: XrdClCopy.cc:64
std::string GetProgressBar(time_t now)
Get progress bar.
Definition: XrdClCopy.cc:151
std::string GetSummaryBar(time_t now)
Get sumary bar.
Definition: XrdClCopy.cc:193
ProgressDisplay()
Constructor.
Definition: XrdClCopy.cc:56
virtual void EndJob(uint32_t jobNum, const XrdCl::PropertyList *results)
End job.
Definition: XrdClCopy.cc:91
void PrintTargetCheckSum(bool print)
Definition: XrdClCopy.cc:296
virtual void JobProgress(uint32_t jobNum, uint64_t bytesProcessed, uint64_t bytesTotal)
Job progress.
Definition: XrdClCopy.cc:224
static int mapError(int rc)
Definition: XProtocol.hh:1404
Copy the data from one point to another.
XRootDStatus Run(CopyProgressHandler *handler)
Run the copy jobs.
XRootDStatus Prepare()
XRootDStatus AddJob(const PropertyList &properties, PropertyList *results)
Interface for copy progress notification.
static PostMaster * GetPostMaster()
Get default post master.
static Env * GetEnv()
Get default client environment.
const std::string & GetName() const
Get file name.
StatInfo * GetStatInfo()
Get the stat info object.
Iterator End()
Get the end iterator.
Iterator Begin()
Get the begin iterator.
bool PutInt(const std::string &key, int value)
Definition: XrdClEnv.cc:136
bool PutString(const std::string &key, const std::string &value)
Definition: XrdClEnv.cc:52
bool GetInt(const std::string &key, int &value)
Definition: XrdClEnv.cc:115
Send file/filesystem queries to an XRootD cluster.
XRootDStatus DirList(const std::string &path, DirListFlags::Flags flags, ResponseHandler *handler, time_t timeout=0) XRD_WARN_UNUSED_RESULT
XRootDStatus Stat(const std::string &path, ResponseHandler *handler, time_t timeout=0) XRD_WARN_UNUSED_RESULT
Handle diagnostics.
Definition: XrdClLog.hh:101
void SetLevel(LogLevel level)
Set the level of the messages that should be sent to the destination.
Definition: XrdClLog.hh:193
void Error(uint64_t topic, const char *format,...)
Report an error.
Definition: XrdClLog.cc:231
LogLevel GetLevel() const
Get the log level.
Definition: XrdClLog.hh:258
void Dump(uint64_t topic, const char *format,...)
Print a dump message.
Definition: XrdClLog.cc:299
void Info(uint64_t topic, const char *format,...)
Print an info.
Definition: XrdClLog.cc:265
void Debug(uint64_t topic, const char *format,...)
Print a debug message.
Definition: XrdClLog.cc:282
bool Stop()
Stop the postmaster.
A key-value pair map storing both keys and values as strings.
void Set(const std::string &name, const Item &value)
bool Get(const std::string &name, Item &item) const
Object stat info.
bool TestFlags(uint32_t flags) const
Test flags.
URL representation.
Definition: XrdClURL.hh:31
std::string GetHostId() const
Get the host part of the URL (user:password@host:port)
Definition: XrdClURL.hh:99
const std::string & GetProtocol() const
Get the protocol.
Definition: XrdClURL.hh:118
std::string GetPathWithParams() const
Get the path with params.
Definition: XrdClURL.cc:294
std::string GetURL() const
Get the URL.
Definition: XrdClURL.hh:86
bool IsLocalFile() const
Definition: XrdClURL.cc:450
const std::string & GetPath() const
Get the path.
Definition: XrdClURL.hh:217
static std::string BytesToString(uint64_t bytes)
Convert bytes to a human readable string.
Definition: XrdClUtils.cc:367
const std::string & GetErrorMessage() const
Get error message.
std::string ToStr() const
Convert to string.
short Doff
Definition: XrdCpFile.hh:46
PType Protocol
Definition: XrdCpFile.hh:49
char * Path
Definition: XrdCpFile.hh:45
char ProtName[8]
Definition: XrdCpFile.hh:50
XrdCpFile * Next
Definition: XrdCpFile.hh:44
short Dlen
Definition: XrdCpFile.hh:47
const int DefaultCPChunkSize
const uint16_t stError
An error occurred that could potentially be retried.
Definition: XrdClStatus.hh:32
const uint64_t AppMsg
LocateImpl< false > Locate
const int DefaultCPParallelChunks
const int DefaultXCpBlockSize
const int DefaultPreserveXAttrs
bool IsOK() const
We're fine.
Definition: XrdClStatus.hh:124
std::string ToString() const
Create a string representation.
Definition: XrdClStatus.cc:97
uint32_t errNo
Errno, if any.
Definition: XrdClStatus.hh:148
int GetShellCode() const
Get the status code that may be returned to the shell.
Definition: XrdClStatus.hh:129
const char * vName
Definition: XrdCpConfig.hh:53
defVar * intDefs
Definition: XrdCpConfig.hh:63
void Config(int argc, char **argv, int Opts=0)
Definition: XrdCpConfig.cc:210
char * tcpTokenFile
Definition: XrdCpConfig.hh:95
std::vector< std::string > AddCksVal
Definition: XrdCpConfig.hh:99
const char * dstOpq
Definition: XrdCpConfig.hh:65
static const uint64_t DoZipMtlnCksum
Definition: XrdCpConfig.hh:196
XrdCpFile * srcFile
Definition: XrdCpConfig.hh:90
XrdCpFile * dstFile
Definition: XrdCpConfig.hh:91
char * zipFile
Definition: XrdCpConfig.hh:93
static const uint64_t DoNoPbar
Definition: XrdCpConfig.hh:129
static const uint64_t DoCoerce
Definition: XrdCpConfig.hh:112
static const uint64_t DoForce
Definition: XrdCpConfig.hh:118
static const uint64_t DoRmOnBadCksum
Definition: XrdCpConfig.hh:199
static const uint64_t DoNoTlsOK
Definition: XrdCpConfig.hh:184
static const uint64_t DoTpc
Definition: XrdCpConfig.hh:157
char * pHost
Definition: XrdCpConfig.hh:71
std::vector< std::string > HttpHeaders
Definition: XrdCpConfig.hh:101
static const uint64_t DoCksum
Definition: XrdCpConfig.hh:108
defVar * strDefs
Definition: XrdCpConfig.hh:64
static const uint64_t DoCksrc
Definition: XrdCpConfig.hh:107
static const uint64_t DoTpcModePush
Definition: XrdCpConfig.hh:218
static const uint64_t DoTpcDlgt
Definition: XrdCpConfig.hh:159
static const uint64_t DoZip
Definition: XrdCpConfig.hh:178
static const uint64_t DoContinue
Definition: XrdCpConfig.hh:202
const char * CksVal
Definition: XrdCpConfig.hh:88
static const uint64_t DoRecurse
Definition: XrdCpConfig.hh:139
const char * srcOpq
Definition: XrdCpConfig.hh:66
static const uint64_t DoZipAppend
Definition: XrdCpConfig.hh:211
static const uint64_t DoDynaSrc
Definition: XrdCpConfig.hh:173
int Want(uint64_t What)
Definition: XrdCpConfig.hh:234
long long xRate
Definition: XrdCpConfig.hh:68
static const uint64_t DoSources
Definition: XrdCpConfig.hh:151
static const uint64_t DoXAttr
Definition: XrdCpConfig.hh:193
static const uint64_t DoTlsMLF
Definition: XrdCpConfig.hh:187
static const int optRmtRec
Definition: XrdCpConfig.hh:226
std::string RetryPolicy
Definition: XrdCpConfig.hh:78
static const uint64_t DoPath
Definition: XrdCpConfig.hh:190
static const uint64_t DoPosc
Definition: XrdCpConfig.hh:132
long long xRateThreshold
Definition: XrdCpConfig.hh:69
static const uint64_t DoTpcOnly
Definition: XrdCpConfig.hh:158
static const uint64_t DoTlsNoData
Definition: XrdCpConfig.hh:181
static const uint64_t DoServer
Definition: XrdCpConfig.hh:145