XRootD
XrdClS3Factory.cc
Go to the documentation of this file.
1 /******************************************************************************/
2 /* Copyright (C) 2025, Pelican Project, Morgridge Institute for Research */
3 /* */
4 /* This file is part of the XrdClS3 client plugin for XRootD. */
5 /* */
6 /* XRootD is free software: you can redistribute it and/or modify it under */
7 /* the terms of the GNU Lesser General Public License as published by the */
8 /* Free Software Foundation, either version 3 of the License, or (at your */
9 /* option) any later version. */
10 /* */
11 /* XRootD is distributed in the hope that it will be useful, but WITHOUT */
12 /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */
13 /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */
14 /* License for more details. */
15 /* */
16 /* The copyright holder's institutional names and contributor's names may not */
17 /* be used to endorse or promote products derived from this software without */
18 /* specific prior written permission of the institution or contributor. */
19 /******************************************************************************/
20 
21 #include "XrdVersion.hh"
22 
23 #include "XrdClS3Factory.hh"
24 #include "XrdClS3File.hh"
25 #include "XrdClS3Filesystem.hh"
26 
27 #include <openssl/evp.h>
28 #include <openssl/hmac.h>
29 #include <XrdCl/XrdClDefaultEnv.hh>
30 #include <XrdCl/XrdClLog.hh>
31 
32 #include <fcntl.h>
33 
35 
36 using namespace XrdClS3;
37 
38 std::shared_mutex Factory::m_bucket_auth_map_mutex;
39 bool Factory::m_initialized = false;
40 XrdCl::Log *Factory::m_log{nullptr};
41 std::once_flag Factory::m_init_once;
42 std::string Factory::m_endpoint = "";
43 std::string Factory::m_service = "s3";
44 std::string Factory::m_region = "";
45 std::string Factory::m_url_style = "path";
46 std::string Factory::m_mkdir_sentinel;
47 Factory::Credentials Factory::m_default_creds;
48 std::unordered_map<std::string, Factory::Credentials> Factory::m_bucket_location_map;
49 std::unordered_map<std::string, std::pair<Factory::Credentials, std::chrono::steady_clock::time_point>> Factory::m_bucket_auth_map;
50 
51 
52 namespace {
53 
54 std::string
55 AmazonURLEncode(const std::string &input) {
56  /*
57  * See
58  * http://docs.amazonwebservices.com/AWSEC2/2010-11-15/DeveloperGuide/using-query-api.html
59  *
60  */
61  std::string output;
62  output.reserve(input.size());
63  for (const auto & val : input) {
64  // "Do not URL encode ... A-Z, a-z, 0-9, hyphen ( - ),
65  // underscore ( _ ), period ( . ), and tilde ( ~ ). Percent
66  // encode all other characters with %XY, where X and Y are hex
67  // characters 0-9 and uppercase A-F. Percent encode extended
68  // UTF-8 characters in the form %XY%ZA..."
69  if (('A' <= val && val <= 'Z') ||
70  ('a' <= val && val <= 'z') ||
71  ('0' <= val && val <= '9') || val == '-' ||
72  val == '_' || val == '.' || val == '~') {
73  output.append(1, val);
74  } else {
75  char percentEncode[4];
76  snprintf(percentEncode, 4, "%%%.2hhX", val);
77  output.append(percentEncode);
78  }
79  }
80  return output;
81 }
82 
83 }
84 
86  std::call_once(m_init_once, [&] {
87  m_log = XrdCl::DefaultEnv::GetLog();
88  if (!m_log) {
89  return;
90  }
91  m_log->SetTopicName(kLogXrdClS3, "XrdClS3");
92 
93  auto env = XrdCl::DefaultEnv::GetEnv();
94  if (!env) {
95  return;
96  }
97  InitS3Config();
98  m_initialized = true;
99  });
100 }
101 
102 std::string
103 Factory::CanonicalizeQueryString(const std::string &url) {
104  auto loc = url.find("://");
105  if (loc == std::string::npos) {
106  return "";
107  }
108  loc += 3; // Skip the "://"
109  loc = url.find('?', loc);
110  if (loc == std::string::npos) {
111  return "";
112  }
113  std::vector<std::pair<std::string, std::string>> query_parameters;
114  auto param_end = url.find('&', loc);
115  while (loc != std::string::npos) {
116  auto param_start = loc + 1; // Skip the '?' / '&'
117  loc = url.find('=', param_start);
118  if (loc == param_start) {
119  // Empty parameter name, skip
120  }
121  else if (loc >= param_end) {
122  auto param = url.substr(param_start, param_end - param_start);
123  if (!param.empty()) {
124  // No '=' found, treat as a parameter without value
125  query_parameters.emplace_back(AmazonURLEncode(param), "");
126  }
127  } else {
128  std::string name = url.substr(param_start, loc - param_start);
129  loc++; // Move past '='
130  auto value_start = loc;
131  std::string value;
132  if (param_end == std::string::npos) {
133  value = url.substr(value_start);
134  } else {
135  value = url.substr(value_start, param_end - value_start);
136  }
137  if (!value.empty()) {
138  query_parameters.emplace_back(AmazonURLEncode(name), AmazonURLEncode(value));
139  }
140  }
141  loc = param_end;
142  if (loc != std::string::npos) {
143  param_end = url.find('&', loc + 1);
144  }
145  }
146  std::sort(query_parameters.begin(), query_parameters.end(),
147  [](const auto &a, const auto &b) { return a.first < b.first; });
148 
149  size_t string_size = 0;
150  for (const auto &param : query_parameters) {
151  string_size += param.first.size() + param.second.size() + 2;
152  }
153  std::string canonicalQueryString;
154  if (string_size) {
155  canonicalQueryString.reserve(string_size);
156  }
157  for (const auto &param : query_parameters) {
158 
159  // Step 1C: Separate parameter names from values with '='.
160  canonicalQueryString += param.first + '=' + param.second;
161 
162  // Step 1D: Separate name-value pairs with '&';
163  canonicalQueryString += '&';
164  }
165  // We'll always have a superflous trailing ampersand.
166  if (!canonicalQueryString.empty()) {
167  canonicalQueryString.erase(canonicalQueryString.end() - 1);
168  }
169  return canonicalQueryString;
170 }
171 
173 Factory::CreateFile(const std::string & /*url*/) {
174  if (!m_initialized) {return nullptr;}
175  return new File(m_log);
176 }
177 
179 Factory::CreateFileSystem(const std::string & url) {
180  if (!m_initialized) {return nullptr;}
181  return new Filesystem(url, m_log);
182 }
183 
184 namespace {
185 
186 void SetDefault(XrdCl::Env *env, const std::string &optName, const std::string &envName, std::string &location, const std::string &def) {
187  std::string val;
188  if (!env->GetString(optName, val) || val.empty()) {
189  env->PutString(optName, "");
190  env->ImportString(optName, envName);
191  }
192  if (env->GetString(optName, val) && !val.empty()) {
193  location = val;
194  } else {
195  location = def;
196  }
197 }
198 
199 // Trim the left side of a string_view for space
200 std::string_view ltrim_view(const std::string_view input_view) {
201  for (size_t idx = 0; idx < input_view.size(); idx++) {
202  if (!isspace(input_view[idx])) {
203  return input_view.substr(idx);
204  }
205  }
206  return "";
207 }
208 
209 bool ComputeSHA256(const std::string_view payload, std::vector<unsigned char> &messageDigest) {
210  EVP_MD_CTX *mdctx = EVP_MD_CTX_create();
211  if (mdctx == NULL) {
212  return false;
213  }
214 
215  if (!EVP_DigestInit_ex(mdctx, EVP_sha256(), NULL)) {
216  EVP_MD_CTX_destroy(mdctx);
217  return false;
218  }
219 
220  if (!EVP_DigestUpdate(mdctx, payload.data(), payload.length())) {
221  EVP_MD_CTX_destroy(mdctx);
222  return false;
223  }
224 
225  unsigned int mdLength;
226  if (!EVP_DigestFinal_ex(mdctx, messageDigest.data(), &mdLength)) {
227  EVP_MD_CTX_destroy(mdctx);
228  return false;
229  }
230  messageDigest.resize(mdLength);
231 
232  EVP_MD_CTX_destroy(mdctx);
233  return true;
234 }
235 
236 void MessageDigestAsHex(const std::vector<unsigned char> messageDigest,
237  std::string &hexEncoded) {
238  hexEncoded.resize(messageDigest.size() * 2);
239  char *ptr = hexEncoded.data();
240  for (unsigned int idx = 0; idx < messageDigest.size(); ++idx, ptr += 2) {
241  snprintf(ptr, 3, "%02x", messageDigest[idx]);
242  }
243 }
244 
245 // Helper function to read a file descriptor until EOF or
246 // `nbytes` bytes have been read.
247 // Includes appropriate handling of EINTR.
248 ssize_t FullRead(int fd, void *ptr, size_t nbytes) {
249  ssize_t nleft, nread;
250 
251  nleft = nbytes;
252  while (nleft > 0) {
253  REISSUE_READ:
254  nread = read(fd, ptr, nleft);
255  if (nread < 0) {
256  if (errno == EINTR) {
257  goto REISSUE_READ;
258  }
259  return -1;
260  } else if (nread == 0) {
261  break;
262  }
263  nleft -= nread;
264  ptr = static_cast<char *>(ptr) + nread;
265  }
266  return (nbytes - nleft);
267 }
268 
269 // Read a file into a string.
270 // If the file is larger than 32k, it will return false.
271 bool
272 ReadShortFile(const std::string &fileName, std::string &contents, std::string &err_msg) {
273  int fd = open(fileName.c_str(), O_RDONLY, 0600);
274  if (fd < 0) {
275  err_msg = "Failed to open file '" + fileName + "': " + std::string(strerror(errno));
276  return false;
277  }
278  contents.resize(32*1024);
279 
280  auto totalRead = FullRead(fd, contents.data(), contents.size());
281  close(fd);
282  if (totalRead == -1) {
283  err_msg = "Failed to read file '" + fileName + "': " + std::string(strerror(errno));
284  return false;
285  }
286  contents.resize(totalRead);
287  return true;
288 }
289 
290 } // namespace
291 
292 std::string
293 Factory::CleanObjectName(const std::string & input_obj) {
294  std::string obj = input_obj;
295  auto loc = input_obj.find('?');
296  if (loc != std::string::npos) {
297  auto query = std::string_view(input_obj).substr(loc + 1);
298  obj = obj.substr(0, loc);
299  bool added_query = false;
300  while (!query.empty()) {
301  auto next_query_loc = query.find('&');
302  auto current_query = (next_query_loc == std::string::npos) ? query : query.substr(0, next_query_loc);
303  query = (next_query_loc == std::string::npos) ? "" : query.substr(next_query_loc + 1);
304  if (current_query.empty()) {
305  continue;
306  }
307  auto equal_loc = current_query.find('=');
308  if (equal_loc != std::string::npos) {
309  auto key = current_query.substr(0, equal_loc);
310  if (key != "authz") {
311  obj += (added_query ? "&" : "?") + std::string(current_query);
312  added_query = true;
313  }
314  } else if (current_query != "authz") {
315  obj += (added_query ? "&" : "?") + std::string(current_query);
316  added_query = true;
317  }
318  }
319  } else {
320  obj = input_obj;
321  }
322  return obj;
323 }
324 
325 std::string_view
326 Factory::ExtractHostname(const std::string_view url) {
327  auto loc = url.find("://");
328  if (loc == std::string_view::npos) {
329  return {};
330  }
331  loc += 3; // Move past "://"
332  auto slash_loc = url.find('/', loc);
333  auto query_loc = url.find('?', loc);
334  if (query_loc != std::string_view::npos && (slash_loc == std::string_view::npos || query_loc < slash_loc)) {
335  slash_loc = query_loc; // If there's a query, we stop at it
336  }
337  auto authority = url.substr(loc, slash_loc - loc);
338  if (authority.empty()) {
339  return {};
340  }
341  auto at_loc = authority.find('@');
342  if (at_loc != std::string_view::npos) {
343  // If there's an '@', we have user info, so we skip it
344  authority = authority.substr(at_loc + 1);
345  }
346  // If the authority contains a port, we need to strip it
347  auto colon_loc = authority.find(':');
348  if (colon_loc != std::string_view::npos) {
349  authority = authority.substr(0, colon_loc);
350  }
351  return authority;
352 }
353 
354 void
355 Factory::InitS3Config()
356 {
357  auto env = XrdCl::DefaultEnv::GetEnv();
358  SetDefault(env, "XrdClS3MkdirSentinel", "XRDCLS3_MKDIRSENTINEL", m_mkdir_sentinel, ".xrdcls3.dirsentinel");
359  SetDefault(env, "XrdClS3Endpoint", "XRDCLS3_ENDPOINT", m_endpoint, "");
360  SetDefault(env, "XrdClS3UrlStyle", "XRDCLS3_URLSTYLE", m_url_style, "path");
361  SetDefault(env, "XrdClS3Region", "XRDCLS3_REGION", m_region, "");
362  std::string access_key;
363  SetDefault(env, "XrdClS3AccessKeyLocation", "XRDCLS3_ACCESSKEYLOCATION", access_key, "");
364  std::string secret_key;
365  SetDefault(env, "XrdClS3SecretKeyLocation", "XRDCLS3_SECRETKEYLOCATION", secret_key, "");
366  if (!access_key.empty() && !secret_key.empty()) {
367  m_default_creds = {access_key, secret_key};
368  } else if (access_key.empty() && secret_key.empty()) {
369  m_log->Info(kLogXrdClS3, "Defaulting to public bucket access");
370  } else if (access_key.empty() && !secret_key.empty()) {
371  m_log->Warning(kLogXrdClS3, "Secret key location set (%s) but access key location is empty; authorization will not work.", secret_key.c_str());
372  } else if (!access_key.empty() && secret_key.empty()) {
373  m_log->Warning(kLogXrdClS3, "Access key location set (%s) but secret key location is empty; authorization will not work.", access_key.c_str());
374  }
375 
376  // Parse the per-bucket configuration of credentials.
377  std::string bucket_configs;
378  SetDefault(env, "XrdClS3BucketConfigs", "XRDCLS3_BUCKETCONFIGS", bucket_configs, "");
379  if (!bucket_configs.empty()) {
380  std::stringstream ss(bucket_configs);
381  std::string config_name;
382  while (std::getline(ss, config_name)) {
383  auto name = TrimView(config_name);
384  auto bucket_name_key = std::string("XrdClS3") + std::string(name) + "BucketName";
385  std::string bucket_name_val;
386  if (!env->GetString(bucket_name_key, bucket_name_val) || bucket_name_val.empty()) {
387  m_log->Warning(kLogXrdClS3, "Per-bucket config includes entry '%s' but XrdClS3%sBucketName is not set", std::string(name).c_str(), std::string(name).c_str());
388  continue;
389  }
390  auto access_key_location_key = std::string("XrdClS3") + std::string(name) + "AccessKeyLocation";
391  std::string access_key_location_val;
392  auto has_access_key = env->GetString(access_key_location_key, access_key_location_val) && !access_key_location_val.empty();
393 
394  auto secret_key_location_key = std::string("XrdClS3") + std::string(name) + "SecretKeyLocation";
395  std::string secret_key_location_val;
396  auto has_secret_key = env->GetString(secret_key_location_key, secret_key_location_val) && !secret_key_location_val.empty();
397 
398  if (has_access_key && has_secret_key) {
399  m_bucket_location_map[bucket_name_val] = {access_key_location_val, secret_key_location_val};
400  } else if (!has_access_key && !has_secret_key) {
401  // If both are empty, then it is implicitly a public bucket.
402  m_bucket_location_map[bucket_name_val] = {"", ""};
403  } else if (has_access_key && !has_secret_key) {
404  m_log->Warning(kLogXrdClS3, "Per-bucket config for entry '%s' has an access key location set (%s) but no secret key", std::string(name).c_str(), access_key_location_val.c_str());
405  } else {
406  m_log->Warning(kLogXrdClS3, "Per-bucket config for entry '%s' has an secret key location set (%s) but no access key", std::string(name).c_str(), secret_key_location_val.c_str());
407  }
408  }
409  }
410 }
411 
412 bool
413 Factory::GenerateHttpUrl(const std::string &s3_url, std::string &https_url, std::string *obj_result, std::string &err_msg) {
414  if (s3_url.substr(0, 5) != "s3://") {
415  err_msg = "Provided URL does not start with s3://";
416  return false;
417  }
418  auto loc = s3_url.find('/', 5);
419  auto bucket = s3_url.substr(5, loc - 5);
420  auto at_loc = bucket.find('@');
421  if (at_loc != std::string::npos) {
422  std::string login = "";
423  login = bucket.substr(0, at_loc);
424  bucket = bucket.substr(at_loc + 1);
425  }
426  std::string endpoint = m_endpoint;
427  std::string region = m_region;
428  if ((bucket == m_endpoint) || m_endpoint.empty()) {
429  endpoint = bucket;
430  if (loc == std::string::npos) {
431  err_msg = "Provided S3 URL does not contain a bucket in path";
432  return false;
433  }
434  auto old_loc = loc + 1;
435  loc = s3_url.find('/', old_loc);
436  if (loc == std::string::npos) {
437  bucket = s3_url.substr(old_loc);
438  auto query = bucket.find('?');
439  if (query != std::string::npos) {
440  bucket.erase(query);
441  }
442  if (bucket.empty()) {
443  err_msg = "Provided S3 URL does not contain a bucket in path";
444  return false;
445  }
446  } else {
447  bucket = s3_url.substr(old_loc, loc - old_loc);
448  }
449  } else {
450  auto authority = ExtractHostname(s3_url);
451  std::string test_endpoint = "." + endpoint;
452  if (!m_region.empty()) {
453  auto bucket_loc = authority.rfind("." + m_region + test_endpoint);
454  if (bucket_loc != std::string::npos) {
455  bucket = authority.substr(0, bucket_loc);
456  } else {
457  auto bucket_loc = authority.rfind(test_endpoint);
458  if (bucket_loc != std::string::npos) {
459  bucket = authority.substr(0, bucket_loc);
460  }
461  }
462  } else {
463  auto bucket_loc = authority.rfind(test_endpoint);
464  if (bucket_loc != std::string::npos) {
465  bucket = authority.substr(0, bucket_loc);
466  }
467  }
468  }
469  std::string obj;
470  if (loc != std::string::npos) {
471  obj = s3_url.substr(loc + 1);
472  }
473  // Strip out "authz" query parameters; those are internal to XRootD.
474  obj = CleanObjectName(obj);
475  if (obj_result) {
476  *obj_result = obj;
477  }
478  if (m_url_style == "virtual" || m_url_style.empty()) {
479  https_url = "https://" + bucket + "." + m_region + "." + endpoint + (obj_result ? "" : ("/" + obj));
480  return true;
481  } else if (m_url_style == "path") {
482  if (!m_region.empty()) {
483  https_url = "https://" + m_region + "." + endpoint + "/" + bucket + (obj_result ? "" : ("/" + obj));
484  } else {
485  https_url = "https://" + endpoint + "/" + bucket + (obj_result ? "" : ("/" + obj));
486  }
487  return true;
488  } else {
489  err_msg = "Server configuration has invalid setting for URL style";
490  return false;
491  }
492 }
493 
494 bool
495 Factory::GenerateV4Signature(const std::string &url, const std::string &verb, std::vector<std::pair<std::string, std::string>> &headers, std::string &auth_token, std::string &err_msg) {
496  auto bucket = GetBucketFromHttpsUrl(url);
497 
498  // If we're using temporary credentials, we need to add the token
499  // header here as well. We set saKey and keyID here (well before
500  // necessary) since we'll get them for free when we get the token.
501  auto [keyId, secretKey, ok] = GetCredentialsForBucket(bucket, err_msg);
502  if (!ok) {
503  return false;
504  }
505 
506  if (secretKey.empty()) {
507  auth_token = "";
508  return true;
509  }
510 
511  //
512  // Create task 1's inputs.
513  //
514 
515  auto canonicalURI = PathEncode(url);
516 
517  // The canonical query string is the alphabetically sorted list of
518  // URI-encoded parameter names '=' values, separated by '&'s.
519  auto canonicalQueryString = CanonicalizeQueryString(url);
520 
521  // The canonical headers must include the Host header, so add that
522  // now if we don't have it.
523  if (std::find_if(headers.begin(), headers.end(),
524  [](const auto &pair) { return pair.first == "Host"; }) == headers.end()) {
525  auto host = ExtractHostname(url);
526  if (host.empty()) {
527  err_msg = "Unable to extract hostname from URL: " + url;
528  return false;
529  }
530  headers.emplace_back("Host", host);
531  }
532 
533  // S3 complains if x-amz-date isn't signed, so do this early.
534  auto iter = std::find_if(headers.begin(), headers.end(),
535  [](const auto &pair) { return !strcasecmp(pair.first.c_str(), "X-Amz-Date"); });
536  std::string date_time;
537  char date_char[] = "YYYYMMDD";
538  if (iter == headers.end()) {
539  time_t now;
540  time(&now);
541  struct tm brokenDownTime;
542  gmtime_r(&now, &brokenDownTime);
543 
544  date_time = "YYYYMMDDThhmmssZ";
545  strftime(date_time.data(), date_time.size(), "%Y%m%dT%H%M%SZ", &brokenDownTime);
546  headers.emplace_back("X-Amz-Date", date_time);
547  strftime(date_char, sizeof(date_char), "%Y%m%d", &brokenDownTime);
548  } else {
549  date_time = iter->second;
550  auto loc = date_time.find('T', 0);
551  if (loc != 8) {
552  err_msg = "Invalid value for X-Amz-Date";
553  return false;
554  }
555  memcpy(date_char, date_time.c_str(), 8);
556  }
557 
558  // The canonical payload hash is the lowercase hexadecimal string of the
559  // (SHA256) hash value of the payload or "UNSIGNED-PAYLOAD" if
560  // we are not signing the payload.
561  std::string payload_hash = "UNSIGNED-PAYLOAD";
562  iter = std::find_if(headers.begin(), headers.end(),
563  [](const auto &pair) { return !strcasecmp(pair.first.c_str(), "X-Amz-Content-Sha256"); });
564  if (iter == headers.end()) {
565  headers.emplace_back("X-Amz-Content-Sha256", payload_hash);
566  } else {
567  payload_hash = iter->second;
568  }
569 
570  // The canonical list of headers is a sorted list of lowercase header
571  // names paired via ':' with the trimmed header value, each pair
572  // terminated with a newline.
573  std::vector<std::pair<std::string, std::string>> transformed_headers;
574  transformed_headers.reserve(headers.size());
575  for (const auto &info : headers) {
576  std::string header = info.first;
577  std::transform(header.begin(), header.end(), header.begin(), &tolower);
578 
579  std::string value = info.second;
580  if (value.empty()) {
581  continue;
582  }
583  auto value_trimmed = std::string(TrimView(value));
584 
585  // Convert internal runs of spaces into single spaces.
586  unsigned left = 1;
587  unsigned right = 1;
588  bool inSpaces = false;
589  while (right < value_trimmed.length()) {
590  if (!inSpaces) {
591  if (value_trimmed[right] == ' ') {
592  inSpaces = true;
593  left = right;
594  ++right;
595  } else {
596  ++right;
597  }
598  } else {
599  if (value_trimmed[right] == ' ') {
600  ++right;
601  } else {
602  inSpaces = false;
603  value_trimmed.erase(left, right - left - 1);
604  right = left + 1;
605  }
606  }
607  }
608 
609  transformed_headers.emplace_back(header, value);
610  }
611  std::sort(transformed_headers.begin(), transformed_headers.end(),
612  [](const auto &a, const auto &b) { return a.first < b.first; });
613 
614  // The canonical list of signed headers is trivial to generate while
615  // generating the list of headers.
616  std::string signedHeaders, canonicalHeaders;
617  for (const auto &info : transformed_headers) {
618  canonicalHeaders += info.first + ":" + info.second + "\n";
619  signedHeaders += info.first + ";";
620  }
621  signedHeaders.erase(signedHeaders.end() - 1);
622 
623  // Task 1: create the canonical request.
624  auto canonicalRequest =
625  verb + "\n" + canonicalURI + "\n" + canonicalQueryString + "\n" +
626  canonicalHeaders + "\n" + signedHeaders + "\n" + payload_hash;
627 
628  //
629  // Create task 2's inputs.
630  //
631 
632  // Hash the canonical request the way we did the payload.
633  std::string canonicalRequestHash;
634  std::vector<unsigned char> messageDigest;
635  messageDigest.resize(EVP_MAX_MD_SIZE);
636  if (!ComputeSHA256(canonicalRequest, messageDigest)) {
637  err_msg = "Unable to hash canonical request.";
638  return false;
639  }
640  MessageDigestAsHex(messageDigest, canonicalRequestHash);
641 
642  // Task 2: create the string to sign.
643  auto credentialScope = std::string(date_char) + "/" + m_region + "/" + m_service + "/aws4_request";
644  auto stringToSign = std::string("AWS4-HMAC-SHA256\n") + date_time + "\n" + credentialScope + "\n" + canonicalRequestHash;
645 
646  //
647  // Creating task 3's inputs was done when we checked to see if we needed
648  // to get the security token, since they come along for free when we do.
649  //
650 
651  // Task 3: calculate the signature.
652  auto saKey = std::string("AWS4") + secretKey;
653  unsigned int mdLength = 0;
654  const unsigned char *hmac =
655  HMAC(EVP_sha256(), saKey.c_str(), saKey.length(), (unsigned char *)date_char,
656  sizeof(date_char) - 1, messageDigest.data(), &mdLength);
657  if (hmac == NULL) {
658  err_msg = "Unable to calculate HMAC for date.";
659  return false;
660  }
661 
662  unsigned int md2Length = 0;
663  unsigned char messageDigest2[EVP_MAX_MD_SIZE];
664  hmac = HMAC(EVP_sha256(), messageDigest.data(), mdLength,
665  reinterpret_cast<unsigned char *>(m_region.data()), m_region.size(), messageDigest2,
666  &md2Length);
667  if (hmac == NULL) {
668  err_msg = "Unable to calculate HMAC for region.";
669  return false;
670  }
671 
672  hmac = HMAC(EVP_sha256(), messageDigest2, md2Length,
673  reinterpret_cast<unsigned char *>(m_service.data()), m_service.size(), messageDigest.data(),
674  &mdLength);
675  if (hmac == NULL) {
676  err_msg = "Unable to calculate HMAC for service.";
677  return false;
678  }
679 
680  const char request_char[] = "aws4_request";
681  hmac = HMAC(EVP_sha256(), messageDigest.data(), messageDigest.size(), reinterpret_cast<const unsigned char *>(request_char),
682  sizeof(request_char) - 1, messageDigest2, &md2Length);
683  if (hmac == NULL) {
684  err_msg = "Unable to calculate HMAC for request.";
685  return false;
686  }
687 
688  hmac = HMAC(EVP_sha256(), messageDigest2, md2Length,
689  reinterpret_cast<unsigned char *>(stringToSign.data()),
690  stringToSign.size(), messageDigest.data(), &mdLength);
691  if (hmac == NULL) {
692  err_msg = "Unable to calculate HMAC for request string.";
693  return false;
694  }
695 
696  std::string signature;
697  MessageDigestAsHex(messageDigest, signature);
698 
699  auth_token =
700  std::string("AWS4-HMAC-SHA256 Credential=") + keyId + "/" + credentialScope +
701  ",SignedHeaders=" + signedHeaders + ",Signature=" + signature;
702  return true;
703 }
704 
705 std::string
706 Factory::GetBucketFromHttpsUrl(const std::string &url) {
707  if (m_url_style == "virtual" || m_url_style.empty()) {
708  // Virtual-hosted-style URLs are of the form https://bucket.region.endpoint/object
709  auto hostname = ExtractHostname(url);
710  if (hostname.empty()) {
711  return {};
712  }
713  auto test_endpoint = "." + m_endpoint;
714  if (!m_region.empty()) test_endpoint = "." + m_region + test_endpoint;
715  auto loc = hostname.rfind(test_endpoint);
716  if (loc == std::string::npos) {
717  if (!m_region.empty()) {
718  loc = hostname.rfind("." + m_endpoint);
719  if (loc != std::string::npos) {
720  return std::string(hostname.substr(0, loc));
721  }
722  }
723  return {};
724  }
725  return std::string(hostname.substr(0, loc));
726  } else if (m_url_style == "path") {
727  // Path style URLs are of the form https://region.endpoint/bucket/object
728  auto loc = url.find("://");
729  if (loc == std::string::npos) {
730  return {};
731  }
732  loc += 3; // Move past "://"
733  auto slash_loc = url.find('/', loc);
734  if (slash_loc == std::string::npos) {
735  return {};
736  }
737  auto bucket_start = slash_loc + 1;
738  auto bucket_end = url.find('/', bucket_start);
739  if (bucket_end == std::string::npos) {
740  return url.substr(bucket_start);
741  }
742  return url.substr(bucket_start, bucket_end - bucket_start);
743  } else {
744  // Invalid URL style
745  return {};
746  }
747 }
748 
749 std::tuple<std::string, std::string, bool>
750 Factory::GetCredentialsForBucket(const std::string &bucket, std::string &err_msg)
751 {
752  auto now = std::chrono::steady_clock::now();
753  {
754  std::shared_lock lock(m_bucket_auth_map_mutex);
755  auto iter = m_bucket_auth_map.find(bucket);
756  if (iter != m_bucket_auth_map.end()) {
757  // If we have credentials for this bucket, check if they are still valid.
758  auto &creds = iter->second.first;
759  auto &expiration = iter->second.second;
760  if (now < expiration) {
761  // Credentials are still valid, return them.
762  return {creds.m_accesskey, creds.m_secretkey, true};
763  }
764  }
765  }
766 
767  std::unique_lock lock(m_bucket_auth_map_mutex);
768  auto iter = m_bucket_location_map.find(bucket);
769  std::string access_key_location, secret_key_location;
770  if (iter == m_bucket_location_map.end()) {
771  // If we don't have credentials for this bucket, use the default.
772  if (m_default_creds.m_accesskey.empty() || m_default_creds.m_secretkey.empty()) {
773  // No credentials at all, so we assume public access.
774  m_bucket_auth_map[bucket] = {{"", ""}, now + std::chrono::minutes(1)};
775  return {"", "", true};
776  }
777  access_key_location = m_default_creds.m_accesskey;
778  secret_key_location = m_default_creds.m_secretkey;
779  } else {
780  access_key_location = iter->second.m_accesskey;
781  secret_key_location = iter->second.m_secretkey;
782  }
783  if (access_key_location.empty() && secret_key_location.empty()) {
784  // If both are empty, we assume public access.
785  m_bucket_auth_map[bucket] = {{"", ""}, now + std::chrono::minutes(1)};
786  return {"", "", true};
787  }
788  if (access_key_location.empty() || secret_key_location.empty()) {
789  err_msg = "No credentials available for bucket: " + bucket;
790  m_bucket_auth_map[bucket] = {{"", ""}, now + std::chrono::seconds(10)};
791  return {"", "", false};
792  }
793 
794  std::string access_key, secret_key;
795  if (!ReadShortFile(access_key_location, access_key, err_msg)) {
796  m_bucket_auth_map[bucket] = {{"", ""}, now + std::chrono::seconds(10)};
797  return {"", "", false};
798  }
799  access_key = TrimView(access_key);
800 
801  if (!ReadShortFile(secret_key_location, secret_key, err_msg)) {
802  m_bucket_auth_map[bucket] = {{"", ""}, now + std::chrono::seconds(10)};
803  return {"", "", false};
804  }
805  secret_key = TrimView(secret_key);
806 
807  if (access_key.empty() || secret_key.empty()) {
808  err_msg = "Credentials for bucket '" + bucket + "' are empty.";
809  m_bucket_auth_map[bucket] = {{"", ""}, now + std::chrono::seconds(10)};
810  return {"", "", false};
811  }
812  m_bucket_auth_map[bucket] = {{access_key, secret_key}, now + std::chrono::minutes(1)};
813  return {access_key, secret_key, true};
814 }
815 
816 std::string
817 Factory::PathEncode(const std::string_view url) {
818  auto loc = url.find("://");
819  if (loc == std::string_view::npos) {
820  return "";
821  }
822  auto path_loc = url.find("/", loc + 3);
823  auto query_loc = url.find("?", loc + 3);
824  if (query_loc != std::string_view::npos && (path_loc == std::string_view::npos || query_loc < path_loc)) {
825  // No path, just a query string
826  return "/";
827  }
828  auto path = url.substr(path_loc, query_loc - path_loc);
829  std::string segment;
830  std::string encoded;
831 
832  size_t next = 0;
833  size_t offset = 0;
834  const auto length = path.size();
835  while (offset < length) {
836  next = strcspn(path.data() + offset, "/");
837  if (next == 0) {
838  encoded += "/";
839  offset += 1;
840  continue;
841  }
842  if (offset + next >= length) {
843  next = length - offset;
844  }
845 
846  segment = std::string(path.data() + offset, next);
847  encoded += AmazonURLEncode(segment);
848 
849  offset += next;
850  }
851  return encoded;
852 }
853 
854 // Trim left and right side of a string_view for space characters
855 std::string_view
856 Factory::TrimView(const std::string_view input_view) {
857  auto view = ltrim_view(input_view);
858  for (size_t idx = 0; idx < input_view.size(); idx++) {
859  if (!isspace(view[view.size() - 1 - idx])) {
860  return view.substr(0, view.size() - idx);
861  }
862  }
863  return "";
864 }
865 
866 extern "C"
867 {
868  XrdEXPORT void *XrdClGetPlugIn(const void*)
869  {
870  return static_cast<void*>(new Factory());
871  }
872 }
XrdVERSIONINFO(XrdClGetPlugIn, XrdClGetPlugIn) using namespace XrdClS3
XrdEXPORT void * XrdClGetPlugIn(const void *)
ssize_t read(int fildes, void *buf, size_t nbyte)
#define close(a)
Definition: XrdPosix.hh:48
#define open
Definition: XrdPosix.hh:78
XrdOucString File
void getline(uchar *buff, int blen)
virtual XrdCl::FilePlugIn * CreateFile(const std::string &url) override
Create a file plug-in for the given URL.
static std::string_view ExtractHostname(const std::string_view url)
static std::string PathEncode(const std::string_view url)
static std::string CleanObjectName(const std::string &object)
static bool GenerateHttpUrl(const std::string &s3_url, std::string &https_url, std::string *obj_result, std::string &err_msg)
static bool GenerateV4Signature(const std::string &url, const std::string &verb, std::vector< std::pair< std::string, std::string >> &headers, std::string &auth_token, std::string &err_msg)
static std::tuple< std::string, std::string, bool > GetCredentialsForBucket(const std::string &bucket, std::string &err_msg)
static std::string_view TrimView(const std::string_view str)
virtual XrdCl::FileSystemPlugIn * CreateFileSystem(const std::string &url) override
Create a file system plug-in for the given URL.
static std::string GetBucketFromHttpsUrl(const std::string &url)
static Log * GetLog()
Get default log.
static Env * GetEnv()
Get default client environment.
bool PutString(const std::string &key, const std::string &value)
Definition: XrdClEnv.cc:52
bool ImportString(const std::string &key, const std::string &shellKey)
Definition: XrdClEnv.cc:266
bool GetString(const std::string &key, std::string &value)
Definition: XrdClEnv.cc:31
An interface for file plug-ins.
An interface for file plug-ins.
Handle diagnostics.
Definition: XrdClLog.hh:101
void SetTopicName(uint64_t topic, std::string name)
Map a topic number to a string.
Definition: XrdClLog.cc:163
void Warning(uint64_t topic, const char *format,...)
Report a warning.
Definition: XrdClLog.cc:248
void Info(uint64_t topic, const char *format,...)
Print an info.
Definition: XrdClLog.cc:265
std::string_view ltrim_view(const std::string_view &input_view)
const uint64_t kLogXrdClS3