XRootD
XrdClS3Filesystem.cc
Go to the documentation of this file.
1 /******************************************************************************/
2 /* Copyright (C) 2025, Pelican Project, Morgridge Institute for Research */
3 /* */
4 /* This file is part of the XrdClS3 client plugin for XRootD. */
5 /* */
6 /* XRootD is free software: you can redistribute it and/or modify it under */
7 /* the terms of the GNU Lesser General Public License as published by the */
8 /* Free Software Foundation, either version 3 of the License, or (at your */
9 /* option) any later version. */
10 /* */
11 /* XRootD is distributed in the hope that it will be useful, but WITHOUT */
12 /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */
13 /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */
14 /* License for more details. */
15 /* */
16 /* The copyright holder's institutional names and contributor's names may not */
17 /* be used to endorse or promote products derived from this software without */
18 /* specific prior written permission of the institution or contributor. */
19 /******************************************************************************/
20 
22 #include "XrdClS3Factory.hh"
23 #include "XrdClS3Filesystem.hh"
24 
25 #include <tinyxml.h>
26 #include <XrdCl/XrdClConstants.hh>
27 #include <XrdCl/XrdClDefaultEnv.hh>
28 #include <XrdCl/XrdClURL.hh>
29 #include <XrdCl/XrdClLog.hh>
30 #include <XrdOuc/XrdOucUtils.hh>
31 
32 #include <charconv>
33 
34 using namespace XrdClS3;
35 
36 namespace {
37 
38 // Helper function to URL-quote a string.
39 std::string urlquote(const std::string input) {
40  std::string output;
41  output.reserve(3 * input.size());
42  for (char val : input) {
43  if ((val >= 48 && val <= 57) || // Digits 0-9
44  (val >= 65 && val <= 90) || // Uppercase A-Z
45  (val >= 97 && val <= 122) || // Lowercase a-z
46  (val == 95 || val == 46 || val == 45 || val == 126 ||
47  val == 47)) // '_.-~/'
48  {
49  output += val;
50  } else {
51  output += "%" + std::to_string(val);
52  }
53  }
54  return output;
55 }
56 
57 class StatHandler : public XrdCl::ResponseHandler {
58 public:
59  StatHandler(const std::string &path, const std::string &s3_url, XrdClHttp::HeaderCallout *header_callout, XrdCl::ResponseHandler *handler, time_t timeout, XrdCl::Log &log) :
60  m_timeout(timeout),
61  m_handler(handler),
62  m_header_callout(header_callout),
63  m_path(path),
64  m_s3_url(s3_url),
65  m_logger(log)
66  {}
67 
68  virtual void HandleResponse(XrdCl::XRootDStatus *status, XrdCl::AnyObject *response) override;
69 
70 private:
71  time_t m_timeout;
72  XrdCl::ResponseHandler *m_handler{nullptr};
73  XrdClHttp::HeaderCallout *m_header_callout{nullptr};
74  std::string m_path;
75  std::string m_s3_url;
76  XrdCl::Log &m_logger;
77 };
78 
79 // If the stat request returns a "file not found" error, then there is definitely not
80 // an object at the given path. However, it could be a directory, so we
81 // issue a directory listing request to see if it is a directory.
82 // This is the response handler for that directory listing request.
83 class StatHandlerDirectory : public XrdCl::ResponseHandler {
84 public:
85  StatHandlerDirectory(XrdCl::ResponseHandler *handler) :
86  m_handler(handler)
87  {}
88 
89  virtual void HandleResponse(XrdCl::XRootDStatus *status, XrdCl::AnyObject *response) override;
90 
91 private:
92  XrdCl::ResponseHandler *m_handler{nullptr};
93 };
94 
95 // Response handler for the S3 directory listing GET operation.
96 class DirListResponseHandler : public XrdCl::ResponseHandler {
97 public:
98  DirListResponseHandler(bool existence_check, const std::string &url, const std::string &prefix, XrdClHttp::HeaderCallout *header_callout, XrdCl::ResponseHandler *handler, time_t expiry, XrdCl::Log &log) :
99  m_existence_check(existence_check),
100  m_expiry(expiry),
101  m_header_callout(header_callout),
102  m_url(url),
103  m_prefix(prefix),
104  m_host(Factory::ExtractHostname(url)),
105  m_handler(handler),
106  m_logger(log)
107  {}
108 
109  virtual void HandleResponse(XrdCl::XRootDStatus *status, XrdCl::AnyObject *response) override;
110 
111 private:
112  // Sometimes we are simply looking to see if a "directory" exists; in such a case, we
113  // don't need to enumerate all the entries in the bucket and can exit early after the first subdir
114  // or file is found
115  bool m_existence_check;
116 
117  time_t m_expiry; // Expiration time for the directory listing request
118  XrdClHttp::HeaderCallout *m_header_callout{nullptr}; // Header callout for S3 signing
119  std::string m_url; // The URL of the S3 directory listing
120  std::string m_prefix; // ListObjects prefix. Empty for the bucket root.
121  std::string m_host; // The host address of the S3 endpoint
122 
123  std::unique_ptr<XrdCl::DirectoryList> dirlist{new XrdCl::DirectoryList()}; // Directory listing object to hold the results
124 
125  XrdCl::ResponseHandler *m_handler{nullptr};
126  XrdCl::Log &m_logger;
127 };
128 
129 // Handle the creation of a zero-sized file that indicates a "directory"
130 class MkdirHandler : public XrdCl::ResponseHandler {
131 public:
132  MkdirHandler(XrdCl::File *file, XrdCl::ResponseHandler *handler, time_t timeout) :
133  m_expiry(time(NULL) + (timeout ? timeout : 30)),
134  m_file(file),
135  m_handler(handler)
136  {}
137 
138  virtual void HandleResponse(XrdCl::XRootDStatus *status, XrdCl::AnyObject *response) override;
139 
140 private:
141  time_t m_expiry;
142  bool m_started_close{false};
143  std::unique_ptr<XrdCl::File> m_file;
144  XrdCl::ResponseHandler *m_handler{nullptr};
145 };
146 
147 void
148 StatHandler::HandleResponse(XrdCl::XRootDStatus *status_raw, XrdCl::AnyObject *response_raw) {
149  std::unique_ptr<StatHandler> self(this);
150  std::unique_ptr<XrdCl::AnyObject> response_holder(response_raw);
151  std::unique_ptr<XrdCl::XRootDStatus> status(status_raw);
152 
153  if (!status) {
154  if (m_handler) {return m_handler->HandleResponse(status.release(), response_holder.release());}
155  else return;
156  }
157 
158  if (status->IsOK() || status->errNo != kXR_NotFound) {
159  if (m_handler) {return m_handler->HandleResponse(status.release(), response_holder.release());}
160  else return;
161  }
162 
163  // We got a "file not found" type of response. In this case, we could interpret
164  // this as a directory.
165  std::string https_url, err_msg;
166  const auto s3_url = XrdOucUtils::JoinUrl(m_s3_url, m_path);
167  std::string obj;
168  if (!Factory::GenerateHttpUrl(s3_url, https_url, &obj, err_msg)) {
169  if (m_handler) return m_handler->HandleResponse(new XrdCl::XRootDStatus(XrdCl::stError, XrdCl::errInvalidAddr, 0, err_msg), nullptr);
170  else return;
171  }
172  obj = obj.substr(0, obj.find('?'));
173  if (!obj.empty() && obj.back() != '/') {
174  obj += "/";
175  }
176  auto query_loc = https_url.find('?');
177  https_url += (query_loc == std::string::npos) ? "?" : "&";
178  https_url += "list-type=2&delimiter=/&encoding-type=url";
179  // An empty prefix= is dropped from the signature and the request is rejected.
180  if (!obj.empty()) {
181  https_url += "&prefix=" + urlquote(obj);
182  }
183 
184  auto expiry = time(NULL) + m_timeout;
185 
186  auto st = DownloadUrl(
187  https_url,
188  m_header_callout,
189  new DirListResponseHandler(
190  true, https_url, obj, m_header_callout, new StatHandlerDirectory(m_handler), expiry, m_logger
191  ),
192  m_timeout
193  );
194  if (!st.IsOK()) {
195  if (m_handler) return m_handler->HandleResponse(new XrdCl::XRootDStatus(st), response_holder.release());
196  else return;
197  }
198 }
199 
200 void
201 StatHandlerDirectory::HandleResponse(XrdCl::XRootDStatus *status, XrdCl::AnyObject *response) {
202  std::unique_ptr<StatHandlerDirectory> self(this);
203  if (!m_handler) {
204  delete response;
205  delete status;
206  return;
207  }
208  if (!status || !status->IsOK()) {
209  return m_handler->HandleResponse(status, response);
210  }
211  time_t mtime = 0;
212  if (response) {
213  XrdCl::DirectoryList *list = nullptr;
214  response->Get(list);
215  const auto &sentinel = Factory::GetMkdirSentinel();
216  if (list && !sentinel.empty()) {
217  for (auto it = list->Begin(); it != list->End(); ++it) {
218  if ((*it)->GetName() != sentinel) {
219  continue;
220  }
221  auto *info = (*it)->GetStatInfo();
222  if (info) {
223  mtime = static_cast<time_t>(info->GetModTime());
224  }
225  break;
226  }
227  }
228  }
229  auto stat_info = new XrdCl::StatInfo("nobody", 0, XrdCl::StatInfo::IsDir, mtime);
230  auto obj = new XrdCl::AnyObject();
231  obj->Set(stat_info);
232  delete response;
233  m_handler->HandleResponse(status, obj);
234 }
235 
236 void
237 DirListResponseHandler::HandleResponse(XrdCl::XRootDStatus *status_raw, XrdCl::AnyObject *response_raw) {
238  std::unique_ptr<DirListResponseHandler> self(this);
239  std::unique_ptr<XrdCl::AnyObject> response(response_raw);
240  std::unique_ptr<XrdCl::XRootDStatus> status(status_raw);
241  if (!m_handler) {
242  return;
243  }
244  if (!status || !status->IsOK()) {
245  return m_handler->HandleResponse(status.release(), response.release());
246  }
247 
248  if (!response) {
249  m_logger.Error(kLogXrdClS3, "Directory listing returned without any response object.");
250  return m_handler->HandleResponse(new XrdCl::XRootDStatus(XrdCl::stError, XrdCl::errInvalidResponse, 0, "No response object provided"), nullptr);
251  }
252 
253  XrdCl::Buffer *buffer = nullptr;
254  response->Get(buffer);
255  if (!buffer) {
256  m_logger.Error(kLogXrdClS3, "Directory listing response object was not a buffer.");
257  return m_handler->HandleResponse(new XrdCl::XRootDStatus(XrdCl::stError, XrdCl::errInvalidResponse, 0, "No buffer in response object"), nullptr);
258  }
259 
260  // Parse the XML response from the S3 service
261  TiXmlDocument doc;
262  std::string buffer_str(buffer->GetBuffer(), buffer->GetSize());
263  doc.Parse(buffer_str.c_str());
264  if (doc.Error()) {
265  std::string errMsg = "Error when parsing S3 endpoint's listing response: " + std::string(doc.ErrorDesc());
266  m_handler->HandleResponse(new XrdCl::XRootDStatus(XrdCl::stError, XrdCl::errInvalidResponse, 0, errMsg), nullptr);
267  return;
268  }
269 
270  auto elem = doc.RootElement();
271  if (strcmp(elem->Value(), "ListBucketResult")) {
272  m_handler->HandleResponse(new XrdCl::XRootDStatus(XrdCl::stError, XrdCl::errInvalidResponse, 0,
273  "S3 ListBucket response is not rooted with ListBucketResult element"), nullptr);
274  return;
275  }
276 
277  // Example response from S3:
278  // <?xml version="1.0" encoding="utf-8"?>
279  // <ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
280  // <Name>genome-browser</Name>
281  // <Prefix>cells/muscle-ibm/endothelial-stromal-cells</Prefix>
282  // <KeyCount>40</KeyCount>
283  // <MaxKeys>40</MaxKeys>
284  // <NextContinuationToken>1PnsptbFFpBSb6UBNN4F/RrxtBvIHjNpdXNYlX8E7IyqXRK26w2y36KViUAbyPPsjzikVY0Zj4jMvQHRhsGWZbcKKrEVvaR0HaZDtfUXUwnc=</NextContinuationToken>
285  // <IsTruncated>false</IsTruncated>
286  // <Contents>
287  // <Key>cells/muscle-ibm/endothelial-stromal-cells/UMAP.coords.tsv.gz</Key>
288  // <LastModified>2023-08-21T11:02:53.000Z</LastModified>
289  // <ETag>"b9b0065f10cbd91c9d341acc235c63b0"</ETag>
290  // <Size>360012</Size>
291  // <StorageClass>STANDARD</StorageClass>
292  // </Contents>
293  // <Contents>
294  // <Key>cells/muscle-ibm/endothelial-stromal-cells/barcodes.tsv.gz</Key>
295  // <LastModified>2023-07-17T11:02:19.000Z</LastModified>
296  // <ETag>"048feef5d340e2dd4d2d2d495c24ad7e"</ETag>
297  // <Size>118061</Size>
298  // <StorageClass>STANDARD</StorageClass>
299  // </Contents>
300  // ... (truncated some entries for readability) ...
301  // <CommonPrefixes>
302  // <Prefix>cells/muscle-ibm/endothelial-stromal-cells/coords/</Prefix>
303  // </CommonPrefixes>
304  // <CommonPrefixes>
305  // <Prefix>cells/muscle-ibm/endothelial-stromal-cells/markers/</Prefix>
306  // </CommonPrefixes>
307  // <CommonPrefixes>
308  // <Prefix>cells/muscle-ibm/endothelial-stromal-cells/metaFields/</Prefix>
309  // </CommonPrefixes>
310  // </ListBucketResult>
311  bool isTruncated = false;
312  std::string ct;
313  bool found_prefix_marker = false;
314  auto relativeName = [&](std::string_view encoded) {
315  std::string name = XrdOucUtils::UrlDecode(std::string(encoded));
316  if (!m_prefix.empty() &&
317  name.compare(0, m_prefix.size(), m_prefix) == 0) {
318  name.erase(0, m_prefix.size());
319  }
320  if (!name.empty() && name.back() == '/') {
321  name.pop_back();
322  }
323  return name;
324  };
325  for (auto child = elem->FirstChildElement(); child != nullptr;
326  child = child->NextSiblingElement()) {
327  if (!strcmp(child->Value(), "IsTruncated")) {
328  auto text = child->GetText();
329  if (!strcasecmp(text, "true")) {
330  isTruncated = true;
331  } else if (!strcasecmp(text, "false")) {
332  isTruncated = false;
333  }
334  } else if (!strcmp(child->Value(), "CommonPrefixes")) {
335  auto prefix = child->FirstChildElement("Prefix");
336  if (prefix != nullptr) {
337  auto prefixChar = prefix->GetText();
338  if (prefixChar != nullptr) {
339  auto prefixStr = Factory::TrimView(prefixChar);
340  if (!prefixStr.empty()) {
341  auto name = relativeName(prefixStr);
342  if (name.empty()) {
343  found_prefix_marker = true;
344  } else {
345  uint32_t flags = XrdCl::StatInfo::Flags::IsReadable |
346  XrdCl::StatInfo::Flags::IsDir |
347  XrdCl::StatInfo::Flags::XBitSet;
348  dirlist->Add(
350  m_host, std::move(name), new XrdCl::StatInfo(
351  "nobody", 4096, flags, 0)));
352  }
353  }
354  }
355  }
356  } else if (!strcmp(child->Value(), "Contents")) {
357  std::string_view keyStr;
358  auto key = child->FirstChildElement("Key");
359  if (key != nullptr) {
360  auto keyChar = key->GetText();
361  if (keyChar != nullptr) {
362  keyStr = Factory::TrimView(keyChar);
363  }
364  }
365  if (keyStr.empty()) {
366  continue;
367  }
368  auto name = relativeName(keyStr);
369  if (name.empty()) {
370  found_prefix_marker = true;
371  if (m_existence_check) break;
372  else continue;
373  }
374  int64_t size = -1;
375  bool goodSize = false;
376  auto sizeElem = child->FirstChildElement("Size");
377  if (sizeElem != nullptr) {
378  auto sizeChar = sizeElem->GetText();
379  if (sizeChar != nullptr && *sizeChar) {
380  auto res = std::from_chars(sizeChar, sizeChar + strlen(sizeChar), size);
381  if (res.ec == std::errc()) {
382  goodSize = true;
383  }
384  }
385  }
386  auto lastModifiedElem = child->FirstChildElement("LastModified");
387  time_t lastModified = 0;
388  if (lastModifiedElem != nullptr) {
389  auto lastModifiedChar = lastModifiedElem->GetText();
390  if (lastModifiedChar != nullptr) {
391  struct tm tm;
392  // LastModified is UTC, for example "2023-08-21T11:02:53.000Z".
393  if (strptime(lastModifiedChar, "%Y-%m-%dT%H:%M:%S", &tm) != nullptr) {
394  tm.tm_isdst = 0;
395  lastModified = timegm(&tm);
396  }
397  }
398  }
399  if (goodSize) {
400  uint32_t flags = XrdCl::StatInfo::Flags::IsReadable;
401  dirlist->Add(
403  m_host, std::move(name), new XrdCl::StatInfo(
404  "nobody", size, flags, lastModified)));
405  }
406  } else if (!strcmp(child->Value(), "NextContinuationToken")) {
407  auto ctChar = child->GetText();
408  if (ctChar) {
409  ct = Factory::TrimView(ctChar);
410  }
411  }
412  }
413  const bool directory_exists = found_prefix_marker || dirlist->GetSize();
414  // - !isTruncated indicates all object listings have been consumed.
415  // - If m_existence_check mode is set, then the caller only cares to know that this is a
416  // directory; as soon as the directory has any "contents", then it officially exists and
417  // we can return. The bucket root exists when its list succeeds.
418  if (!isTruncated || (m_existence_check && (directory_exists || m_prefix.empty()))) {
419  // An empty listing is not a directory unless a prefix placeholder
420  // object was present. That object is not itself a child entry.
421  // The bucket root is a directory even when the list is empty.
422  if (!directory_exists && !m_prefix.empty()) {
423  m_handler->HandleResponse(
425  nullptr
426  );
427  return;
428  }
429  auto object = new XrdCl::AnyObject();
430  object->Set(dirlist.release());
431  m_handler->HandleResponse(
432  new XrdCl::XRootDStatus{},
433  object
434  );
435  return;
436  }
437 
438  // The signer encodes this query value. Encoding the token first makes
439  // the signature disagree with the one the service computes.
440  auto url = m_url + "&continuation-token=" + ct;
441 
442  // Calculate the timeout based on the current time and the expiry time
443  time_t now = time(NULL);
444  if (now >= m_expiry) {
445  m_handler->HandleResponse(
446  new XrdCl::XRootDStatus(XrdCl::stError, XrdCl::errOperationExpired, 0, "Request timed out"),
447  nullptr
448  );
449  return;
450  }
451 
452  auto st = DownloadUrl(url, m_header_callout, this, m_expiry - now);
453  if (!st.IsOK()) {
454  m_handler->HandleResponse(new XrdCl::XRootDStatus(st), nullptr);
455  return;
456  }
457  // DownloadUrl calls this handler when the next page arrives. The
458  // unique_ptr must not delete it before then.
459  self.release();
460 }
461 
462 void
463 MkdirHandler::HandleResponse(XrdCl::XRootDStatus *status_raw, XrdCl::AnyObject *response_raw)
464 {
465  std::unique_ptr<MkdirHandler> self(this);
466  std::unique_ptr<XrdCl::XRootDStatus> status(status_raw);
467  std::unique_ptr<XrdCl::AnyObject> response(response_raw);
468 
469  if (!status || !status->IsOK() || m_started_close) {
470  if (m_handler) m_handler->HandleResponse(status.release(), response.release());
471  return;
472  }
473 
474  time_t now = time(NULL);
475  if (now >= m_expiry) {
476  m_handler->HandleResponse(
477  new XrdCl::XRootDStatus(XrdCl::stError, XrdCl::errOperationExpired, 0, "Request timed out"),
478  nullptr
479  );
480  return;
481  }
482 
483  self.release();
484  m_started_close = true;
485  auto st = m_file->Close(this, m_expiry - now);
486  if (!st.IsOK()) {
487  if (m_handler) m_handler->HandleResponse(status.release(), response.release());
488  return;
489  }
490 }
491 
492 
493 } // namespace
494 
495 Filesystem::Filesystem(const std::string &url, XrdCl::Log *log) :
496  m_logger(log),
497  m_url(url)
498 {
499  m_url.SetPath("");
501  m_url.SetParams(map);
502 
503  m_logger->Debug(kLogXrdClS3, "S3 filesystem constructed with URL: %s.",
504  m_url.GetURL().c_str());
505 }
506 
508 
510 Filesystem::DirList(const std::string &path,
512  XrdCl::ResponseHandler *handler,
513  time_t timeout)
514 {
515  std::string https_url, err_msg;
516  const auto s3_url = XrdOucUtils::JoinUrl(m_url.GetURL(), path);
517  std::string obj;
518  if (!Factory::GenerateHttpUrl(s3_url, https_url, &obj, err_msg)) {
520  }
521  obj = obj.substr(0, obj.find('?'));
522  if (!obj.empty() && obj.back() != '/') {
523  obj += "/";
524  }
525  auto query_loc = https_url.find('?');
526  https_url += (query_loc == std::string::npos) ? "?" : "&";
527  https_url += "list-type=2&delimiter=/&encoding-type=url";
528  // An empty prefix= is dropped from the signature and the request is rejected.
529  if (!obj.empty()) {
530  https_url += "&prefix=" + urlquote(obj);
531  }
532 
533  // File::Open turns a zero timeout into RequestTimeout for the first page.
534  // The next page is fetched only while this deadline is still ahead, and
535  // a zero timeout makes that deadline the current time.
536  if (!timeout) {
538  XrdCl::DefaultEnv::GetEnv()->GetInt("RequestTimeout", val);
539  timeout = val;
540  }
541  auto expiry = time(NULL) + timeout;
542 
543  return DownloadUrl(
544  https_url,
545  &m_header_callout,
546  new DirListResponseHandler(
547  false, https_url, obj, &m_header_callout, handler, expiry, *m_logger
548  ),
549  timeout
550  );
551 }
552 
553 std::pair<XrdCl::XRootDStatus, XrdCl::FileSystem*>
554 Filesystem::GetFSHandle(const std::string &path) {
555  const auto s3_url = XrdOucUtils::JoinUrl(m_url.GetURL(), path);
556  std::string https_url, err_msg;
557  if (!Factory::GenerateHttpUrl(s3_url, https_url, nullptr, err_msg)) {
558  return std::make_pair(XrdCl::XRootDStatus(XrdCl::stError, XrdCl::errInvalidAddr, 0, err_msg), nullptr);
559  }
560  auto loc = https_url.find('/', 8); // strlen("https://") -> 8
561  if (loc == std::string::npos) {
562  return std::make_pair(XrdCl::XRootDStatus(XrdCl::stError, XrdCl::errInvalidAddr, 0, "Invalid generated URL"), nullptr);
563  }
564  auto endpoint = https_url.substr(0, loc);
565  {
566  std::shared_lock lock(m_handles_mutex);
567  auto iter = m_handles.find(endpoint);
568  if (iter != m_handles.end()) {
569  return std::make_pair(XrdCl::XRootDStatus{}, iter->second);
570  }
571  }
572  XrdCl::URL url;
573  if (!url.FromString(https_url)) {
574  return std::make_pair(XrdCl::XRootDStatus(XrdCl::stError, XrdCl::errInvalidAddr, 0, "Invalid generated XrdCl URL"), nullptr);
575  }
576  std::unique_lock lock(m_handles_mutex);
577  auto iter = m_handles.find(endpoint);
578  if (iter != m_handles.end()) {
579  return std::make_pair(XrdCl::XRootDStatus{}, iter->second);
580  }
581  auto fs = new XrdCl::FileSystem(url);
582  std::stringstream ss;
583  ss << std::hex << reinterpret_cast<long long>(&m_header_callout);
584  if (!fs->SetProperty("XrdClHttpHeaderCallout", ss.str())) {
585  delete fs;
586  return std::make_pair(XrdCl::XRootDStatus(XrdCl::stError, XrdCl::errInvalidAddr, 0, "Failed to setup header callout"), nullptr);
587  }
588  m_handles[endpoint] = fs;
589 
590  return std::make_pair(XrdCl::XRootDStatus{}, fs);
591 }
592 
593 bool
594 Filesystem::GetProperty(const std::string &name,
595  std::string &value) const
596 {
597  std::unique_lock lock(m_properties_mutex);
598  const auto p = m_properties.find(name);
599  if (p == std::end(m_properties)) {
600  return false;
601  }
602 
603  value = p->second;
604  return true;
605 }
606 
608 Filesystem::Locate(const std::string &path,
610  XrdCl::ResponseHandler *handler,
611  time_t timeout)
612 {
613  auto cleaned_path = Factory::CleanObjectName(path);
614  auto [st, fs] = GetFSHandle(cleaned_path);
615  if (!st.IsOK()) {
616  return st;
617  }
618  return fs->Locate(cleaned_path, flags, handler, timeout);
619 }
620 
622 Filesystem::MkDir(const std::string &input_path,
624  XrdCl::Access::Mode mode,
625  XrdCl::ResponseHandler *handler,
626  time_t timeout)
627 {
628  auto sentinel = Factory::GetMkdirSentinel();
629  if (sentinel.empty()) {
630  if (handler) handler->HandleResponse(new XrdCl::XRootDStatus{}, nullptr);
631  return {};
632  }
633  auto loc = input_path.find('?');
634  auto path = input_path.substr(0, loc);
635  if (!path.empty() && path[path.size() - 1] != '/') path += "/";
636  path += sentinel;
637  if (loc != std::string::npos) {
638  path += input_path.substr(loc);
639  }
640 
641  // Try creating a zero-sized sentinel.
642  std::string https_url, err_msg;
643  const auto s3_url = XrdOucUtils::JoinUrl(m_url.GetURL(), path);
644  if (!Factory::GenerateHttpUrl(s3_url, https_url, nullptr, err_msg)) {
646  }
647 
648  XrdCl::File *http_file(new XrdCl::File(https_url));
649 
650  auto callout_loc = reinterpret_cast<long long>(&m_header_callout);
651  size_t buf_size = 16;
652  char callout_buf[buf_size];
653  std::to_chars_result result = std::to_chars(callout_buf, callout_buf + buf_size - 1, callout_loc, 16);
654  if (result.ec == std::errc{}) {
655  std::string callout_str(callout_buf, result.ptr - callout_buf);
656  http_file->SetProperty("XrdClHttpHeaderCallout", callout_str);
657  }
658 
659  MkdirHandler *mkdirHandler = new MkdirHandler(http_file, handler, timeout);
660 
661  return http_file->Open(https_url, XrdCl::OpenFlags::Write, XrdCl::Access::None, mkdirHandler, timeout);
662 }
663 
666  const XrdCl::Buffer &arg,
667  XrdCl::ResponseHandler *handler,
668  time_t timeout)
669 {
670  if (queryCode != XrdCl::QueryCode::Checksum && queryCode != XrdCl::QueryCode::XAttr) {
672  }
673  auto cleaned_path = Factory::CleanObjectName(arg.ToString());
674  auto [st, fs] = GetFSHandle(cleaned_path);
675  if (!st.IsOK()) {
676  return st;
677  }
678  XrdCl::Buffer cleanedArg;
679  cleanedArg.FromString(cleaned_path);
680  return fs->Query(queryCode, cleanedArg, handler, timeout);
681 }
682 
683 
685 Filesystem::Rm(const std::string &path,
686  XrdCl::ResponseHandler *handler,
687  time_t timeout)
688 {
689  auto cleaned_path = Factory::CleanObjectName(path);
690  auto [st, fs] = GetFSHandle(cleaned_path);
691  if (!st.IsOK()) {
692  return st;
693  }
694  return fs->Rm(cleaned_path, handler, timeout);
695 }
696 
698 Filesystem::RmDir(const std::string &input_path,
699  XrdCl::ResponseHandler *handler,
700  time_t timeout)
701 {
702  auto sentinel = Factory::GetMkdirSentinel();
703  if (sentinel.empty()) {
704  if (handler) handler->HandleResponse(new XrdCl::XRootDStatus{}, nullptr);
705  return {};
706  }
707  auto loc = input_path.find('?');
708  auto path = input_path.substr(0, loc);
709  if (!path.empty() && path[path.size() - 1] != '/') path += "/";
710  path += sentinel;
711  if (loc != std::string::npos) {
712  path += input_path.substr(loc);
713  }
714  return Rm(path, handler, timeout);
715 }
716 
717 
718 bool
719 Filesystem::SetProperty(const std::string &name,
720  const std::string &value)
721 {
722  std::unique_lock lock(m_properties_mutex);
723  m_properties[name] = value;
724  return true;
725 }
726 
728 Filesystem::Stat(const std::string &path,
729  XrdCl::ResponseHandler *handler,
730  time_t timeout)
731 {
732  auto cleaned_path = Factory::CleanObjectName(path);
733  std::string https_url, err_msg, obj;
734  const auto s3_url = XrdOucUtils::JoinUrl(m_url.GetURL(), cleaned_path);
735  if (!Factory::GenerateHttpUrl(s3_url, https_url, &obj, err_msg)) {
737  }
738  auto query = obj.find('?');
739  if (query != std::string::npos) {
740  obj.erase(query);
741  }
742  // HEAD of the bucket returns a zero-byte success. List it instead.
743  if (obj.empty()) {
744  auto query_loc = https_url.find('?');
745  https_url += (query_loc == std::string::npos) ? "?" : "&";
746  https_url += "list-type=2&delimiter=/&encoding-type=url";
747  auto expiry = time(NULL) + timeout;
748  return DownloadUrl(
749  https_url,
750  &m_header_callout,
751  new DirListResponseHandler(
752  true, https_url, "", &m_header_callout, new StatHandlerDirectory(handler), expiry, *m_logger
753  ),
754  timeout
755  );
756  }
757  auto [st, fs] = GetFSHandle(cleaned_path);
758  if (!st.IsOK()) {
759  return st;
760  }
761  return fs->Stat(cleaned_path, new StatHandler(cleaned_path, m_url.GetURL(), &m_header_callout, handler, timeout, *m_logger), timeout);
762 }
763 
764 std::shared_ptr<XrdClHttp::HeaderCallout::HeaderList>
765 Filesystem::S3HeaderCallout::GetHeaders(const std::string &verb,
766  const std::string &url,
768 {
769  std::string auth_token, err_msg;
770  std::shared_ptr<HeaderList> header_list(new HeaderList(headers));
771  if (Factory::GenerateV4Signature(url, verb, *header_list, auth_token, err_msg)) {
772  header_list->emplace_back("Authorization", auth_token);
773  } else {
774  m_parent.m_logger->Error(kLogXrdClS3, "Failed to generate V4 signature: %s", err_msg.c_str());
775  return nullptr;
776  }
777  return header_list;
778 }
@ kXR_NotFound
Definition: XProtocol.hh:1043
static void child()
std::vector< std::pair< std::string, std::string > > HeaderList
static const std::string & GetMkdirSentinel()
static std::string CleanObjectName(const std::string &object)
static bool GenerateHttpUrl(const std::string &s3_url, std::string &https_url, std::string *obj_result, std::string &err_msg)
static bool GenerateV4Signature(const std::string &url, const std::string &verb, std::vector< std::pair< std::string, std::string >> &headers, std::string &auth_token, std::string &err_msg)
static std::string_view TrimView(const std::string_view str)
virtual XrdCl::XRootDStatus MkDir(const std::string &path, XrdCl::MkDirFlags::Flags flags, XrdCl::Access::Mode mode, XrdCl::ResponseHandler *handler, time_t timeout) override
Filesystem(const std::string &, XrdCl::Log *log)
virtual bool SetProperty(const std::string &name, const std::string &value) override
virtual XrdCl::XRootDStatus DirList(const std::string &path, XrdCl::DirListFlags::Flags flags, XrdCl::ResponseHandler *handler, time_t timeout) override
virtual XrdCl::XRootDStatus Rm(const std::string &path, XrdCl::ResponseHandler *handler, time_t timeout) override
virtual XrdCl::XRootDStatus Stat(const std::string &path, XrdCl::ResponseHandler *handler, time_t timeout) override
virtual XrdCl::XRootDStatus Locate(const std::string &path, XrdCl::OpenFlags::Flags flags, XrdCl::ResponseHandler *handler, time_t timeout) override
virtual ~Filesystem() noexcept
virtual XrdCl::XRootDStatus RmDir(const std::string &path, XrdCl::ResponseHandler *handler, time_t timeout) override
virtual bool GetProperty(const std::string &name, std::string &value) const override
virtual XrdCl::XRootDStatus Query(XrdCl::QueryCode::Code queryCode, const XrdCl::Buffer &arg, XrdCl::ResponseHandler *handler, time_t timeout) override
void Get(Type &object)
Retrieve the object being held.
Binary blob representation.
Definition: XrdClBuffer.hh:34
void FromString(const std::string str)
Fill the buffer from a string.
Definition: XrdClBuffer.hh:205
const char * GetBuffer(uint32_t offset=0) const
Get the message buffer.
Definition: XrdClBuffer.hh:72
uint32_t GetSize() const
Get the size of the message.
Definition: XrdClBuffer.hh:132
std::string ToString() const
Convert the buffer to a string.
Definition: XrdClBuffer.hh:215
static Env * GetEnv()
Get default client environment.
Iterator End()
Get the end iterator.
Iterator Begin()
Get the begin iterator.
bool GetInt(const std::string &key, int &value)
Definition: XrdClEnv.cc:115
Send file/filesystem queries to an XRootD cluster.
A file.
Definition: XrdClFile.hh:52
XRootDStatus Open(const std::string &url, OpenFlags::Flags flags, Access::Mode mode, ResponseHandler *handler, time_t timeout=0) XRD_WARN_UNUSED_RESULT
Definition: XrdClFile.cc:125
bool SetProperty(const std::string &name, const std::string &value)
Definition: XrdClFile.cc:983
Handle diagnostics.
Definition: XrdClLog.hh:101
void Debug(uint64_t topic, const char *format,...)
Print a debug message.
Definition: XrdClLog.cc:282
Handle an async response.
virtual void HandleResponse(XRootDStatus *status, AnyObject *response)
Object stat info.
@ IsDir
This is a directory.
URL representation.
Definition: XrdClURL.hh:31
std::map< std::string, std::string > ParamsMap
Definition: XrdClURL.hh:33
bool FromString(const std::string &url)
Parse a string and fill the URL fields.
Definition: XrdClURL.cc:59
void SetParams(const std::string &params)
Set params.
Definition: XrdClURL.cc:378
std::string GetURL() const
Get the URL.
Definition: XrdClURL.hh:86
void SetPath(const std::string &path)
Set the path.
Definition: XrdClURL.hh:225
static std::string UrlDecode(const std::string &input)
static std::string JoinUrl(const std::string &base, const std::string &path)
std::vector< std::pair< std::string, std::string > > HeaderList
XrdCl::XRootDStatus DownloadUrl(const std::string &url, XrdClHttp::HeaderCallout *header_callout, XrdCl::ResponseHandler *handler, time_t timeout)
const uint64_t kLogXrdClS3
const uint16_t errInvalidAddr
Definition: XrdClStatus.hh:71
const uint16_t errErrorResponse
Definition: XrdClStatus.hh:105
const uint16_t errOperationExpired
Definition: XrdClStatus.hh:90
const uint16_t errNotImplemented
Operation is not implemented.
Definition: XrdClStatus.hh:64
const uint16_t stError
An error occurred that could potentially be retried.
Definition: XrdClStatus.hh:32
const uint16_t errInvalidResponse
Definition: XrdClStatus.hh:99
const int DefaultRequestTimeout
Mode
Access mode.
Flags
Open flags, may be or'd when appropriate.
@ Write
Open only for writing.
Code
XRootD query request codes.
@ XAttr
Query file extended attributes.
@ Checksum
Query file checksum.
bool IsOK() const
We're fine.
Definition: XrdClStatus.hh:124