XRootD
XrdClHttp::CurlWorker Class Reference

#include <XrdClHttpWorker.hh>

+ Collaboration diagram for XrdClHttp::CurlWorker:

Public Member Functions

 CurlWorker (const CurlWorker &)=delete
 
 CurlWorker (std::shared_ptr< HandlerQueue > queue, VerbsCache &cache, XrdCl::Log *logger)
 
std::tuple< std::string, std::string > ClientX509CertKeyFile () const
 
void Run ()
 
void Start (std::unique_ptr< XrdClHttp::CurlWorker > self, std::thread tid)
 

Static Public Member Functions

static std::string GetMonitoringJson ()
 
static void RunStatic (CurlWorker *myself)
 
static void SetMaintenancePeriod (unsigned maint)
 

Detailed Description

Definition at line 52 of file XrdClHttpWorker.hh.

Constructor & Destructor Documentation

◆ CurlWorker() [1/2]

CurlWorker::CurlWorker ( std::shared_ptr< HandlerQueue >  queue,
VerbsCache &  cache,
XrdCl::Log *  logger 
)

Definition at line 1040 of file XrdClHttpUtil.cc.

1040  :
1041  m_cache(cache),
1042  m_queue(queue),
1043  m_logger(logger)
1044 {
1045  {
1046  std::unique_lock lk(m_worker_stats_mutex);
1047  m_stats_offset = m_workers_last_completed_cycle.size();
1048  m_workers_last_completed_cycle.push_back(&m_last_completed_cycle);
1049  m_workers_oldest_op.push_back(&m_oldest_op);
1050  }
1051  int pipeInfo[2];
1052  if ((pipe(pipeInfo) == -1) || (fcntl(pipeInfo[0], F_SETFD, FD_CLOEXEC)) || (fcntl(pipeInfo[1], F_SETFD, FD_CLOEXEC))) {
1053  throw std::runtime_error("Failed to create shutdown monitoring pipe for curl worker");
1054  }
1055  m_shutdown_pipe_r = pipeInfo[0];
1056  m_shutdown_pipe_w = pipeInfo[1];
1057 
1058  // Handle setup of the X509 authentication
1059  auto env = XrdCl::DefaultEnv::GetEnv();
1060  env->GetString("HttpClientCertFile", m_x509_client_cert_file);
1061  env->GetString("HttpClientKeyFile", m_x509_client_key_file);
1062 }
static Env * GetEnv()
Get default client environment.

References XrdCl::DefaultEnv::GetEnv().

+ Here is the call graph for this function:

◆ CurlWorker() [2/2]

XrdClHttp::CurlWorker::CurlWorker ( const CurlWorker &  )
delete

Member Function Documentation

◆ ClientX509CertKeyFile()

std::tuple< std::string, std::string > CurlWorker::ClientX509CertKeyFile ( ) const

Definition at line 1064 of file XrdClHttpUtil.cc.

1065 {
1066  return std::make_tuple(m_x509_client_cert_file, m_x509_client_key_file);
1067 }

Referenced by XrdClHttp::CurlOperation::Setup().

+ Here is the caller graph for this function:

◆ GetMonitoringJson()

std::string CurlWorker::GetMonitoringJson ( )
static

Definition at line 1070 of file XrdClHttpUtil.cc.

1071 {
1072  auto now = std::chrono::system_clock::now().time_since_epoch().count();
1073  auto oldest_op = now;
1074  auto oldest_cycle = now;
1075  {
1076  std::unique_lock lk(m_worker_stats_mutex);
1077  for (const auto &entry : m_workers_last_completed_cycle) {
1078  if (!entry) {continue;}
1079  auto cycle = entry->load(std::memory_order_relaxed);
1080  if (cycle < oldest_cycle) oldest_cycle = cycle;
1081  }
1082  for (const auto &entry : m_workers_oldest_op) {
1083  if (!entry) {continue;}
1084  auto op = entry->load(std::memory_order_relaxed);
1085  if (op < oldest_op) oldest_op = op;
1086  }
1087  }
1088  auto oldest_op_dbl = std::chrono::duration<double>(std::chrono::system_clock::time_point(std::chrono::system_clock::duration(oldest_op)).time_since_epoch()).count();
1089  auto oldest_cycle_dbl = std::chrono::duration<double>(std::chrono::system_clock::time_point(std::chrono::system_clock::duration(oldest_cycle)).time_since_epoch()).count();
1090  std::string retval = "{"
1091  "\"oldest_op\":" + std::to_string(oldest_op_dbl) + ","
1092  "\"oldest_cycle\":" + std::to_string(oldest_cycle_dbl) + ","
1093  ;
1094 
1095  for (size_t verb_idx = 0; verb_idx < static_cast<int>(XrdClHttp::CurlOperation::HttpVerb::Count); verb_idx++) {
1096  const auto &verb_str = XrdClHttp::CurlOperation::GetVerbString(static_cast<XrdClHttp::CurlOperation::HttpVerb>(verb_idx));
1097  for (size_t op_idx = 0; op_idx < 402; op_idx++) {
1098  if (op_idx == 401) continue;
1099 
1100  auto &op_stats = m_ops[verb_idx][op_idx];
1101  auto duration = op_stats.m_duration.load(std::memory_order_relaxed);
1102  if (duration == 0) continue;
1103 
1104  std::string prefix = "http_" + verb_str + "_" + ((op_idx == 402) ? "invalid" : std::to_string(200 + op_idx)) + "_";
1105 
1106  auto duration_dbl = std::chrono::duration<double>(std::chrono::steady_clock::duration(duration)).count();
1107  retval += "\"" + prefix + "duration\":" + std::to_string(duration_dbl) + ",";
1108 
1109  duration = op_stats.m_pause_duration.load(std::memory_order_relaxed);
1110  if (duration > 0) {
1111  duration_dbl = std::chrono::duration<double>(std::chrono::steady_clock::duration(duration)).count();
1112  retval += "\"" + prefix + "pause_duration\":" + std::to_string(duration_dbl) + ",";
1113  }
1114 
1115  auto count = op_stats.m_bytes.load(std::memory_order_relaxed);
1116  if (count) retval += "\"" + prefix + "bytes\":" + std::to_string(count) + ",";
1117  count = op_stats.m_error.load(std::memory_order_relaxed);
1118  if (count) retval += "\"" + prefix + "error\":" + std::to_string(count) + ",";
1119  count = op_stats.m_finished.load(std::memory_order_relaxed);
1120  if (count) retval += "\"" + prefix + "finished\":" + std::to_string(count) + ",";
1121  count = op_stats.m_client_timeout.load(std::memory_order_relaxed);
1122  if (count) retval += "\"" + prefix + "client_timeout\":" + std::to_string(count) + ",";
1123  count = op_stats.m_server_timeout.load(std::memory_order_relaxed);
1124  if (count) retval += "\"" + prefix + "server_timeout\":" + std::to_string(count) + ",";
1125  }
1126  {
1127  auto &op_stats = m_ops[verb_idx][401];
1128  auto duration = op_stats.m_duration.load(std::memory_order_relaxed);
1129  if (duration == 0) continue;
1130 
1131  std::string prefix = "http_" + verb_str + "_";
1132 
1133  auto duration_dbl = std::chrono::duration<double>(std::chrono::steady_clock::duration(duration)).count();
1134  retval += "\"" + prefix + "preheader_duration\":" + std::to_string(duration_dbl) + ",";
1135 
1136  auto count = op_stats.m_started.load(std::memory_order_relaxed);
1137  if (count) retval += "\"" + prefix + "started\":" + std::to_string(count) + ",";
1138  count = op_stats.m_error.load(std::memory_order_relaxed);
1139  if (count) retval += "\"" + prefix + "preheader_error\":" + std::to_string(count) + ",";
1140  count = op_stats.m_finished.load(std::memory_order_relaxed);
1141  if (count) retval += "\"" + prefix + "preheader_finished\":" + std::to_string(count) + ",";
1142  count = op_stats.m_server_timeout.load(std::memory_order_relaxed);
1143  if (count) retval += "\"" + prefix + "preheader_timeout\":" + std::to_string(count) + ",";
1144  count = op_stats.m_conncall_timeout.load(std::memory_order_relaxed);
1145  if (count) retval += "\"" + prefix + "conncall_timeout\":" + std::to_string(count) + ",";
1146  }
1147  }
1148 
1149  retval +=
1150  "\"conncall_error\":" + std::to_string(m_conncall_errors.load(std::memory_order_relaxed)) + ","
1151  "\"conncall_started\":" + std::to_string(m_conncall_req.load(std::memory_order_relaxed)) + ","
1152  "\"conncall_success\":" + std::to_string(m_conncall_success.load(std::memory_order_relaxed)) + ","
1153  "\"conncall_timeout\":" + std::to_string(m_conncall_timeout.load(std::memory_order_relaxed)) +
1154  "}";
1155 
1156  return retval;
1157 }
static const std::string GetVerbString(HttpVerb)
XrdClHttp::HttpVerb HttpVerb
Definition: XrdClHttpOps.hh:63

References XrdClHttp::CurlOperation::GetVerbString().

+ Here is the call graph for this function:

◆ Run()

void CurlWorker::Run ( )

Definition at line 1239 of file XrdClHttpUtil.cc.

1239  {
1240  int max_pending = 50;
1241  XrdCl::DefaultEnv::GetEnv()->GetInt("HttpMaxPendingOps", max_pending);
1242  m_continue_queue.reset(new HandlerQueue(max_pending));
1243  auto &queue = *m_queue.get();
1244  m_logger->Debug(kLogXrdClHttp, "Started a curl worker");
1245 
1246  CURLM *multi_handle = curl_multi_init();
1247  if (multi_handle == nullptr) {
1248  throw std::runtime_error("Failed to create curl multi-handle");
1249  }
1250 
1251  int running_handles = 0;
1252  time_t last_maintenance = time(NULL);
1253  CURLMcode mres = CURLM_OK;
1254 
1255  // Map from a file descriptor that has an outstanding broker request
1256  // to the corresponding CURL handle.
1257  std::unordered_map<int, WaitingForBroker> broker_reqs;
1258  std::vector<struct curl_waitfd> waitfds;
1259 
1260  // Detach the handle before clearing the operation's curl callbacks. The
1261  // caller may still hold the operation after this worker stops using it.
1262  auto release_handle = [&](const std::shared_ptr<CurlOperation> &op, CURL *handle) {
1263  if (!handle) return;
1264  curl_multi_remove_handle(multi_handle, handle);
1265  if (op->GetCurlHandle()) op->ReleaseHandle();
1266  curl_easy_cleanup(handle);
1267  };
1268  auto remove_op = [&](CURL *handle) {
1269  auto iter = m_op_map.find(handle);
1270  if (iter == m_op_map.end()) return;
1271  const auto &[op, started_at] = iter->second;
1272  release_handle(op, handle);
1273  m_op_map.erase(iter);
1274  running_handles -= 1;
1275  for (auto req = broker_reqs.begin(); req != broker_reqs.end();) {
1276  if (req->second.curl == handle) {
1277  req = broker_reqs.erase(req);
1278  } else {
1279  ++req;
1280  }
1281  }
1282  };
1283  auto start_options = [&](CURL *parent_handle, const std::shared_ptr<CurlOperation> &parent_op,
1284  const std::string &target) {
1285  std::shared_ptr<CurlOptionsOp> options_op;
1286  CURL *options_handle = nullptr;
1287  try {
1288  std::string modified_url;
1289  options_op = std::make_shared<CurlOptionsOp>(parent_handle, parent_op,
1290  std::string(VerbsCache::GetUrlKey(target, modified_url)),
1291  m_logger, parent_op->GetConnCalloutFunc());
1292  options_handle = queue.GetHandle();
1293  if (!options_handle || !options_op->Setup(options_handle, *this)) {
1294  throw std::runtime_error("Failed to setup the curl handle for OPTIONS");
1295  }
1296  } catch (...) {
1297  m_logger->Debug(kLogXrdClHttp, "Failed to setup the curl handle for OPTIONS");
1298  if (options_op) {
1299  options_op->Fail(XrdCl::errInternal, ENOMEM,
1300  "Failed to setup the curl handle for the OPTIONS operation");
1301  release_handle(options_op, options_handle);
1302  }
1303  parent_op->Fail(XrdCl::errInternal, ENOMEM,
1304  "Failed to setup the curl handle for the OPTIONS operation");
1305  OpRecord(*parent_op, OpKind::Error);
1306  return false;
1307  }
1308  m_op_map[options_handle] = {options_op, std::chrono::system_clock::now()};
1309  OpRecord(*options_op, OpKind::Start);
1310  running_handles += 1;
1311  auto options_res = curl_multi_add_handle(multi_handle, options_handle);
1312  if (options_res == CURLM_OK) return true;
1313  options_op->Fail(XrdCl::errInternal, options_res,
1314  "Unable to add OPTIONS operation to the curl multi-handle");
1315  OpRecord(*options_op, OpKind::Error);
1316  parent_op->Fail(XrdCl::errInternal, options_res,
1317  "Unable to add OPTIONS operation to the curl multi-handle");
1318  OpRecord(*parent_op, OpKind::Error);
1319  remove_op(options_handle);
1320  return false;
1321  };
1322  auto reactivate_parent = [&](const std::shared_ptr<CurlOptionsOp> &options_op) {
1323  auto parent_op = options_op->GetOperation();
1324  auto parent_handle = options_op->GetParentCurlHandle();
1325  if (parent_op->IsDone()) {
1326  remove_op(parent_handle);
1327  return false;
1328  }
1329  // OPTIONS is advisory, including for redirect targets. Resolve a
1330  // pending redirect before adding the parent's handle back to libcurl.
1331  if (parent_op->IsRedirect()) {
1332  std::string target;
1333  auto action = parent_op->Redirect(target);
1334  if (action == CurlOperation::RedirectAction::Fail) {
1335  OpRecord(*parent_op, OpKind::Error);
1336  remove_op(parent_handle);
1337  return false;
1338  }
1339  if (action == CurlOperation::RedirectAction::ReinvokeAfterAllow) {
1340  if (start_options(parent_handle, parent_op, target)) return true;
1341  remove_op(parent_handle);
1342  return false;
1343  }
1344  }
1345  OpRecord(*parent_op, OpKind::Start);
1346  auto callout_socket = parent_op->WaitSocket();
1347  if (callout_socket >= 0) {
1348  broker_reqs[callout_socket] = {parent_handle, time(nullptr) + 20};
1349  m_conncall_req.fetch_add(1, std::memory_order_relaxed);
1350  return true;
1351  }
1352  auto parent_res = curl_multi_add_handle(multi_handle, parent_handle);
1353  if (parent_res == CURLM_OK) return true;
1354  m_logger->Debug(kLogXrdClHttp,
1355  "Unable to re-add the parent operation to the curl multi-handle: %s",
1356  curl_multi_strerror(parent_res));
1357  parent_op->Fail(XrdCl::errInternal, parent_res,
1358  "Unable to re-add the parent operation to the curl multi-handle");
1359  OpRecord(*parent_op, OpKind::Error);
1360  remove_op(parent_handle);
1361  return false;
1362  };
1363 
1364  bool want_shutdown = false;
1365  while (!want_shutdown) {
1366  m_last_completed_cycle.store(std::chrono::system_clock::now().time_since_epoch().count());
1367  auto oldest_op = std::chrono::system_clock::now();
1368  for (const auto &[handle, entry] : m_op_map) {
1369  const auto &[op, started_at] = entry;
1370  OpRecord(*op, OpKind::Update);
1371  if (started_at < oldest_op) {
1372  oldest_op = started_at;
1373  }
1374  }
1375  m_oldest_op.store(oldest_op.time_since_epoch().count());
1376 
1377  // Try continuing any available handles that have more data
1378  while (true) {
1379  auto op = m_continue_queue->TryConsume();
1380  if (!op) {
1381  break;
1382  }
1383  // Avoid race condition where external thread added a continue operation to queue
1384  // while the curl worker thread failed the transfer.
1385  if (op->IsDone()) {
1386  m_logger->Debug(kLogXrdClHttp, "Ignoring continuation of operation that has already completed");
1387  continue;
1388  }
1389  m_logger->Debug(kLogXrdClHttp, "Continuing the curl handle from op %p on thread %d", op.get(), getthreadid());
1390  auto curl = op->GetCurlHandle();
1391  if (!op->ContinueHandle()) {
1392  op->Fail(XrdCl::errInternal, 0, "Failed to continue the curl handle for the operation");
1393  OpRecord(*op, OpKind::Error);
1394  remove_op(curl);
1395  continue;
1396  } else {
1397  auto iter = m_op_map.find(curl);
1398  if (iter != m_op_map.end()) {
1399  auto &[operation, started_at] = iter->second;
1400  started_at = std::chrono::system_clock::now();
1401  }
1402  }
1403  }
1404  // Consume from the shared new operation queue
1405  while (running_handles < static_cast<int>(m_max_ops)) {
1406  auto op = running_handles == 0 ? queue.Consume(std::chrono::seconds(1)) : queue.TryConsume();
1407  if (!op) {
1408  break;
1409  }
1410  auto curl = queue.GetHandle();
1411  if (curl == nullptr) {
1412  m_logger->Debug(kLogXrdClHttp, "Unable to allocate a curl handle");
1413  op->Fail(XrdCl::errInternal, ENOMEM, "Unable to get allocate a curl handle");
1414  continue;
1415  }
1416  try {
1417  auto rv = op->Setup(curl, *this);
1418  if (!rv) {
1419  m_logger->Debug(kLogXrdClHttp, "Failed to setup the curl handle");
1420  op->Fail(XrdCl::errInternal, ENOMEM, "Failed to setup the curl handle for the operation");
1421  release_handle(op, curl);
1422  continue;
1423  }
1424  if (!op->FinishSetup(curl)) {
1425  m_logger->Debug(kLogXrdClHttp, "Failed to finish setup of the curl handle");
1426  op->Fail(XrdCl::errInternal, ENOMEM, "Failed to finish setup of the curl handle for the operation");
1427  release_handle(op, curl);
1428  continue;
1429  }
1430  } catch (...) {
1431  m_logger->Debug(kLogXrdClHttp, "Unable to setup the curl handle");
1432  op->Fail(XrdCl::errInternal, ENOMEM, "Failed to setup the curl handle for the operation");
1433  release_handle(op, curl);
1434  continue;
1435  }
1436  op->SetContinueQueue(m_continue_queue);
1437 
1438  if (op->IsDone()) {
1439  op->ReleaseHandle();
1440  queue.RecycleHandle(curl);
1441  continue;
1442  }
1443  m_op_map[curl] = {op, std::chrono::system_clock::now()};
1444  running_handles += 1;
1445 
1446  // If the operation requires the result of the OPTIONS verb to function, then
1447  // we add that to the multi-handle instead, chaining the two calls together.
1448  if (op->RequiresOptions()) {
1449  if (!start_options(curl, op, op->GetUrl())) {
1450  remove_op(curl);
1451  continue;
1452  }
1453  } else {
1454  OpRecord(*op, OpKind::Start);
1455  auto add_res = curl_multi_add_handle(multi_handle, curl);
1456  if (add_res != CURLM_OK) {
1457  m_logger->Debug(kLogXrdClHttp, "Unable to add operation to the curl multi-handle");
1458  op->Fail(XrdCl::errInternal, add_res, "Unable to add operation to the curl multi-handle");
1459  OpRecord(*op, OpKind::Error);
1460  remove_op(curl);
1461  continue;
1462  }
1463  }
1464  m_logger->Debug(kLogXrdClHttp, "Added request for URL %s to worker thread for processing", op->GetUrl().c_str());
1465  }
1466 
1467  // Maintain the periodic reporting of thread activity and fail any operations
1468  // that have expired / timed out.
1469  time_t now = time(NULL);
1470  time_t next_maintenance = last_maintenance + m_maintenance_period.load(std::memory_order_relaxed);
1471  if (now >= next_maintenance) {
1472  m_queue->Expire();
1473  m_continue_queue->Expire();
1474  m_logger->Debug(kLogXrdClHttp, "Curl worker thread %d is running %d operations",
1475  getthreadid(), running_handles);
1476  last_maintenance = now;
1477 
1478  // Timeout all the pending broker requests.
1479  std::vector<std::pair<int, CURL *>> expired_ops;
1480  for (const auto &[fd, request] : broker_reqs) {
1481  if (request.expiry < now) {
1482  expired_ops.emplace_back(fd, request.curl);
1483  }
1484  }
1485  for (const auto &[fd, handle] : expired_ops) {
1486  auto iter = m_op_map.find(handle);
1487  if (iter == m_op_map.end()) {
1488  m_logger->Warning(kLogXrdClHttp, "Found an expired curl handle with no corresponding operation!");
1489  } else {
1490  auto [op, started_at] = iter->second;
1491  op->Fail(XrdCl::errConnectionError, 1, "Timeout: connection never provided for request");
1492  OpRecord(*op, OpKind::ConncallTimeout);
1493  if (auto options_op = std::dynamic_pointer_cast<CurlOptionsOp>(op))
1494  reactivate_parent(options_op);
1495  remove_op(handle);
1496  }
1497  broker_reqs.erase(fd);
1498  m_conncall_timeout.fetch_add(1, std::memory_order_relaxed);
1499  }
1500 
1501  // Cleanup the fake connection cache entries.
1503  }
1504 
1505  waitfds.clear();
1506  waitfds.resize(3 + broker_reqs.size());
1507 
1508  waitfds[0].fd = queue.PollFD();
1509  waitfds[0].events = CURL_WAIT_POLLIN;
1510  waitfds[0].revents = 0;
1511  waitfds[1].fd = m_continue_queue->PollFD();
1512  waitfds[1].events = CURL_WAIT_POLLIN;
1513  waitfds[1].revents = 0;
1514  waitfds[2].fd = m_shutdown_pipe_r;
1515  waitfds[2].revents = 0;
1516  waitfds[2].events = CURL_WAIT_POLLIN | CURL_WAIT_POLLPRI;
1517 
1518  int idx = 3;
1519  for (const auto &[fd, request] : broker_reqs) {
1520  waitfds[idx].fd = fd;
1521  waitfds[idx].events = CURL_WAIT_POLLIN|CURL_WAIT_POLLPRI;
1522  waitfds[idx].revents = 0;
1523  idx += 1;
1524  }
1525 
1526  long timeo;
1527  curl_multi_timeout(multi_handle, &timeo);
1528  // These commented-out lines are purposely left; will need to revisit after the 0.9.1 release;
1529  // for now, they are too verbose on RHEL7.
1530  //m_logger->Debug(kLogXrdClHttp, "Curl advises a timeout of %ld ms", timeo);
1531  if (running_handles && timeo == -1) {
1532  // Bug workaround: we've seen RHEL7 libcurl have a race condition where it'll not
1533  // set a timeout while doing the DNS lookup; assume that if there are running handles
1534  // but no timeout, we've hit this bug.
1535  //m_logger->Debug(kLogXrdClHttp, "Will sleep for up to 50ms");
1536  mres = curl_multi_wait(multi_handle, &waitfds[0], waitfds.size(), 50, nullptr);
1537  } else {
1538  //m_logger->Debug(kLogXrdClHttp, "Will sleep for up to %d seconds", max_sleep_time);
1539  //mres = curl_multi_wait(multi_handle, &waitfds[0], waitfds.size(), max_sleep_time*1000, nullptr);
1540  // Temporary test: we've been seeing DNS lookups timeout on additional platforms. Switch to always
1541  // poll as curl_multi_wait doesn't seem to get notified when DNS lookups are done.
1542  mres = curl_multi_wait(multi_handle, &waitfds[0], waitfds.size(), 50, nullptr);
1543  }
1544  if (mres != CURLM_OK) {
1545  m_logger->Warning(kLogXrdClHttp, "Failed to wait on multi-handle: %d", mres);
1546  }
1547 
1548  // Iterate through the waiting broker callbacks.
1549  for (const auto &entry : waitfds) {
1550  // Ignore the queue's poll fd.
1551  if (waitfds[0].fd == entry.fd || waitfds[1].fd == entry.fd) {
1552  continue;
1553  }
1554  // Handle shutdown requests
1555  if ((waitfds[2].fd == entry.fd) && entry.revents) {
1556  want_shutdown = true;
1557  break;
1558  }
1559  if ((entry.revents & CURL_WAIT_POLLIN) != CURL_WAIT_POLLIN) {
1560  continue;
1561  }
1562  auto request = broker_reqs.find(entry.fd);
1563  if (request == broker_reqs.end()) continue;
1564  auto handle = request->second.curl;
1565  auto iter = m_op_map.find(handle);
1566  if (iter == m_op_map.end()) {
1567  m_logger->Warning(kLogXrdClHttp, "Internal error: broker responded on FD %d but no corresponding curl operation", entry.fd);
1568  broker_reqs.erase(entry.fd);
1569  m_conncall_errors.fetch_add(1, std::memory_order_relaxed);
1570  continue;
1571  }
1572  std::string err;
1573  auto [op, started_at] = iter->second;
1574  auto result = op->WaitSocketCallback(err);
1575  if (result == -1) {
1576  m_logger->Warning(kLogXrdClHttp, "Error when invoking the broker callback: %s", err.c_str());
1577  op->Fail(XrdCl::errErrorResponse, 1, err);
1578  OpRecord(*op, OpKind::Error);
1579  if (auto options_op = std::dynamic_pointer_cast<CurlOptionsOp>(op))
1580  reactivate_parent(options_op);
1581  remove_op(handle);
1582  broker_reqs.erase(entry.fd);
1583  m_conncall_errors.fetch_add(1, std::memory_order_relaxed);
1584  } else {
1585  broker_reqs.erase(entry.fd);
1586  auto add_res = curl_multi_add_handle(multi_handle, handle);
1587  if (add_res != CURLM_OK) {
1588  op->Fail(XrdCl::errInternal, add_res,
1589  "Unable to add operation to the curl multi-handle after the broker callback");
1590  OpRecord(*op, OpKind::Error);
1591  if (auto options_op = std::dynamic_pointer_cast<CurlOptionsOp>(op))
1592  reactivate_parent(options_op);
1593  remove_op(handle);
1594  m_conncall_errors.fetch_add(1, std::memory_order_relaxed);
1595  continue;
1596  }
1597  m_conncall_success.fetch_add(1, std::memory_order_relaxed);
1598  }
1599  }
1600 
1601  // Do maintenance on the multi-handle
1602  int still_running;
1603  mres = curl_multi_perform(multi_handle, &still_running);
1604  if (mres == CURLM_CALL_MULTI_PERFORM) {
1605  mres = CURLM_OK;
1606  continue;
1607  } else if (mres != CURLM_OK) {
1608  m_logger->Warning(kLogXrdClHttp, "Failed to perform multi-handle operation: %d", mres);
1609  break;
1610  }
1611 
1612  CURLMsg *msg;
1613  do {
1614  int msgq = 0;
1615  msg = curl_multi_info_read(multi_handle, &msgq);
1616  if (msg && (msg->msg == CURLMSG_DONE)) {
1617  if (!msg->easy_handle) {
1618  m_logger->Warning(kLogXrdClHttp, "Logic error: got a callback for a null handle");
1619  mres = CURLM_BAD_EASY_HANDLE;
1620  break;
1621  }
1622  CURL *const easy_handle = msg->easy_handle;
1623  auto iter = m_op_map.find(easy_handle);
1624  if (iter == m_op_map.end()) {
1625  m_logger->Error(kLogXrdClHttp, "Logic error: got a callback for an entry that doesn't exist");
1626  mres = CURLM_BAD_EASY_HANDLE;
1627  break;
1628  }
1629  auto [op, started_at] = iter->second;
1630  auto res = msg->data.result;
1631  // Completion callbacks may reconfigure or release their handle.
1632  // Stop libcurl from owning it before invoking operation methods.
1633  curl_multi_remove_handle(multi_handle, easy_handle);
1634  bool keep_handle = false;
1635  bool waiting_on_callout = false;
1636  bool options_pending = false;
1637  if (res == CURLE_OK) {
1638  auto sc = op->GetStatusCode();
1639  OpRecord(*op, OpKind::Finish);
1640  if (HTTPStatusIsError(sc)) {
1641  auto [code, number] = HTTPStatusConvert(sc);
1642  op->Fail(code, number, op->GetStatusMessage());
1643  op->ReleaseHandle();
1644  // If this was a failed CurlOptionsOp, then we re-activate the parent handle.
1645  // If the parent handle was stopped at a redirect that now returns failure, then
1646  // we'll clean it up.
1647  if (auto options_op = std::dynamic_pointer_cast<CurlOptionsOp>(op))
1648  reactivate_parent(options_op);
1649  // The curl operation was successful, it's just the HTTP request failed; recycle the handle.
1650  queue.RecycleHandle(easy_handle);
1651  } else {
1652  // The child and suspended parent own distinct handles.
1653  // Complete the child, then resume or fail the parent without
1654  // treating the parent's redirect as a child request.
1655  if (auto options_op = std::dynamic_pointer_cast<CurlOptionsOp>(op)) {
1656  options_op->Success();
1657  options_op->ReleaseHandle();
1658  if (!options_op->GetOperation()->IsDone()) {
1659  options_op->GetOperation()->OptionsDone();
1660  }
1661  reactivate_parent(options_op);
1662  queue.RecycleHandle(easy_handle);
1663  } else {
1664  if (op->IsRedirect()) {
1665  std::string target;
1666  switch (op->Redirect(target)) {
1667  case CurlOperation::RedirectAction::Fail:
1668  keep_handle = false;
1669  break;
1670  case CurlOperation::RedirectAction::Reinvoke:
1671  keep_handle = true;
1672  OpRecord(*op, OpKind::Start);
1673  break;
1674  case CurlOperation::RedirectAction::ReinvokeAfterAllow:
1675  {
1676  // The redirect resulted in a new endpoint where the cache lookup failed;
1677  // we need to know what HTTP verbs are in the server's Allow list before this
1678  // operation can continue. Inject a new CurlOptionsOp and chain it to the one
1679  // being processed. Once the OPTIONS request is done, then we'll restart this
1680  // operation.
1681  if (!start_options(easy_handle, op, target)) {
1682  keep_handle = false;
1683  break;
1684  }
1685  m_logger->Debug(kLogXrdClHttp, "Invoking the OPTIONS operation before redirect to %s", target.c_str());
1686  // The original curl operation needs to be kept around without re-adding its handle
1687  // to the multi-handle until the OPTIONS operation completes.
1688  options_pending = true;
1689  keep_handle = true;
1690  }
1691  }
1692  int callout_socket = op->WaitSocket();
1693  if ((waiting_on_callout = keep_handle && !options_pending && callout_socket >= 0)) {
1694  auto expiry = time(nullptr) + 20;
1695  m_logger->Debug(kLogXrdClHttp, "Creating a callout wait request on socket %d", callout_socket);
1696  broker_reqs[callout_socket] = {easy_handle, expiry};
1697  m_conncall_req.fetch_add(1, std::memory_order_relaxed);
1698  }
1699  }
1700  if (keep_handle) {
1701  if (!waiting_on_callout && !options_pending) {
1702  auto redirect_res = curl_multi_add_handle(multi_handle, easy_handle);
1703  if (redirect_res != CURLM_OK) {
1704  op->Fail(XrdCl::errInternal, redirect_res,
1705  "Unable to add redirected operation to the curl multi-handle");
1706  OpRecord(*op, OpKind::Error);
1707  op->ReleaseHandle();
1708  queue.RecycleHandle(easy_handle);
1709  keep_handle = false;
1710  }
1711  }
1712  } else {
1713  if (!op->IsDone()) {
1714  op->Success();
1715  }
1716  if (op->IsDone()) {
1717  op->ReleaseHandle();
1718  // If the handle was successful, then we can recycle it.
1719  queue.RecycleHandle(easy_handle);
1720  } else {
1721  // Multi-step operations may configure their easy handle
1722  // for another request from Success(). Keep ownership of
1723  // the handle and run the next request through this worker's
1724  // multi-handle like any other operation.
1725  auto next_res = curl_multi_add_handle(multi_handle, easy_handle);
1726  if (next_res == CURLM_OK) {
1727  keep_handle = true;
1728  OpRecord(*op, OpKind::Start);
1729  } else {
1730  op->Fail(XrdCl::errInternal, next_res,
1731  "Unable to add the next operation request to the curl multi-handle");
1732  OpRecord(*op, OpKind::Error);
1733  op->ReleaseHandle();
1734  queue.RecycleHandle(easy_handle);
1735  }
1736  }
1737  }
1738  }
1739  }
1740  } else if (res == CURLE_COULDNT_CONNECT && op->UseConnectionCallout() && !op->GetTriedBoker()) {
1741  // In this case, we need to use the broker and the curl handle couldn't reuse
1742  // an existing socket.
1743  keep_handle = true;
1744  op->SetTriedBoker(); // Flag to ensure we try a connection only once per operation.
1745  std::string err;
1746  int wait_socket = -1;
1747  if (!op->StartConnectionCallout(err) || (wait_socket=op->WaitSocket()) == -1) {
1748  m_logger->Error(kLogXrdClHttp, "Failed to start broker-based connection: %s", err.c_str());
1749  if (!op->IsDone())
1750  op->Fail(XrdCl::errConnectionError, 1, err);
1751  OpRecord(*op, OpKind::Error);
1752  if (auto options_op = std::dynamic_pointer_cast<CurlOptionsOp>(op))
1753  reactivate_parent(options_op);
1754  op->ReleaseHandle();
1755  keep_handle = false;
1756  } else {
1757  curl_multi_remove_handle(multi_handle, easy_handle);
1758  auto expiry = time(nullptr) + 20;
1759  m_logger->Debug(kLogXrdClHttp, "Curl operation requires a new TCP socket; waiting for callout to respond on socket %d", wait_socket);
1760  broker_reqs[wait_socket] = {easy_handle, expiry};
1761  m_conncall_req.fetch_add(1, std::memory_order_relaxed);
1762  }
1763  } else {
1764  if (res == CURLE_ABORTED_BY_CALLBACK || res == CURLE_WRITE_ERROR) {
1765  // We cannot invoke the failure from within a callback as the curl thread and
1766  // original thread of execution may fight over the ownership of the handle memory.
1767  switch (op->GetError()) {
1768  case CurlOperation::OpError::ErrHeaderTimeout:
1769 #ifdef HAVE_XPROTOCOL_TIMEREXPIRED
1770  op->Fail(XrdCl::errOperationExpired, 0, "Origin did not respond with headers within timeout");
1771 #else
1772  op->Fail(XrdCl::errOperationExpired, 0, "Origin did not respond within timeout");
1773 #endif
1774  OpRecord(*op, OpKind::Error);
1775  break;
1776  case CurlOperation::OpError::ErrCallback: {
1777  auto [ecode, emsg] = op->GetCallbackError();
1778  op->Fail(XrdCl::errErrorResponse, ecode, emsg);
1779  OpRecord(*op, OpKind::Error);
1780  break;
1781  }
1782  case CurlOperation::OpError::ErrOperationTimeout:
1783  op->Fail(XrdCl::errOperationExpired, 0, "Operation timed out");
1784  OpRecord(*op, op->IsPaused() ? OpKind::ClientTimeout : OpKind::ServerTimeout);
1785  break;
1786  case CurlOperation::OpError::ErrTransferSlow:
1787  op->Fail(XrdCl::errOperationExpired, 0, "Transfer speed below minimum threshold");
1788  OpRecord(*op, OpKind::ServerTimeout);
1789  break;
1790  case CurlOperation::OpError::ErrTransferClientStall:
1791  op->Fail(XrdCl::errOperationExpired, 0, "Transfer stalled for too long");
1792  OpRecord(*op, OpKind::ClientTimeout);
1793  break;
1794  case CurlOperation::OpError::ErrTransferStall:
1795  op->Fail(XrdCl::errOperationExpired, 0, "Transfer stalled for too long");
1796  OpRecord(*op, OpKind::ServerTimeout);
1797  break;
1798  case CurlOperation::OpError::ErrNone:
1799  op->Fail(XrdCl::errInternal, 0, "Operation was aborted without recording an abort reason");
1800  OpRecord(*op, OpKind::Error);
1801  break;
1802  };
1803  if (auto options_op = std::dynamic_pointer_cast<CurlOptionsOp>(op))
1804  reactivate_parent(options_op);
1805  } else {
1806  auto [code, number] = CurlCodeConvert(res);
1807  const auto curl_err = op->GetCurlErrorMessage();
1808  const char *curl_easy_err = curl_easy_strerror(res);
1809  const std::string fail_err = !curl_err.empty() ? curl_err : curl_easy_err;
1810  m_logger->Debug(kLogXrdClHttp, "Curl generated an error: %s (%d)", fail_err.c_str(), res);
1811  op->Fail(code, number, fail_err);
1812  OpRecord(*op, OpKind::Error);
1813  if (auto options_op = std::dynamic_pointer_cast<CurlOptionsOp>(op))
1814  reactivate_parent(options_op);
1815  }
1816  op->ReleaseHandle();
1817  }
1818  if (!keep_handle) {
1819  curl_multi_remove_handle(multi_handle, easy_handle);
1820  if (res != CURLE_OK) {
1821  curl_easy_cleanup(easy_handle);
1822  }
1823  for (auto req = broker_reqs.begin(); req != broker_reqs.end();) {
1824  if (req->second.curl == easy_handle) {
1825  m_logger->Warning(kLogXrdClHttp, "Curl handle finished while a broker operation was outstanding");
1826  m_conncall_errors.fetch_add(1, std::memory_order_relaxed);
1827  req = broker_reqs.erase(req);
1828  } else {
1829  ++req;
1830  }
1831  }
1832  m_op_map.erase(easy_handle);
1833  running_handles -= 1;
1834  }
1835  }
1836  } while (msg);
1837  if (mres != CURLM_OK) break;
1838  }
1839 
1840  for (const auto &[handle, entry] : m_op_map) {
1841  const auto &[op, started_at] = entry;
1842  if (!op->IsDone()) {
1843  const auto err_num = mres == CURLM_OK ? ECANCELED : static_cast<int>(mres);
1844  const std::string err_msg = mres == CURLM_OK
1845  ? "Curl worker shut down before the operation completed"
1846  : curl_multi_strerror(mres);
1847  op->Fail(XrdCl::errInternal, err_num, err_msg);
1848  OpRecord(*op, OpKind::Error);
1849  }
1850  release_handle(op, handle);
1851  }
1852  m_op_map.clear();
1853 
1854  m_queue->ReleaseHandles();
1855  curl_multi_cleanup(multi_handle);
1856 }
std::pair< uint16_t, uint32_t > CurlCodeConvert(CURLcode res)
void CURL
int emsg(int rc, char *msg)
@ Error
static void CleanupDnsCache()
static std::string_view GetUrlKey(const std::string &url, std::string &modified_url)
bool GetInt(const std::string &key, int &value)
Definition: XrdClEnv.cc:115
void Error(uint64_t topic, const char *format,...)
Report an error.
Definition: XrdClLog.cc:231
void Warning(uint64_t topic, const char *format,...)
Report a warning.
Definition: XrdClLog.cc:248
void Debug(uint64_t topic, const char *format,...)
Print a debug message.
Definition: XrdClLog.cc:282
std::pair< uint16_t, uint32_t > HTTPStatusConvert(unsigned status)
bool HTTPStatusIsError(unsigned status)
const uint16_t errErrorResponse
Definition: XrdClStatus.hh:105
const uint16_t errOperationExpired
Definition: XrdClStatus.hh:90
const uint16_t errInternal
Internal error.
Definition: XrdClStatus.hh:56
const uint16_t errConnectionError
Definition: XrdClStatus.hh:78
const uint64_t kLogXrdClHttp

References XrdClHttp::CurlOperation::CleanupDnsCache(), CurlCodeConvert(), XrdCl::Log::Debug(), emsg(), XrdCl::errConnectionError, XrdCl::errErrorResponse, XrdCl::errInternal, XrdCl::errOperationExpired, XrdCl::Log::Error(), Error, XrdCl::DefaultEnv::GetEnv(), XrdCl::Env::GetInt(), XrdClHttp::VerbsCache::GetUrlKey(), XrdClHttp::HTTPStatusConvert(), XrdClHttp::HTTPStatusIsError(), XrdClHttp::kLogXrdClHttp, and XrdCl::Log::Warning().

Referenced by RunStatic().

+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ RunStatic()

void CurlWorker::RunStatic ( CurlWorker *  myself)
static

Definition at line 1220 of file XrdClHttpUtil.cc.

1221 {
1222  {
1223  std::unique_lock lock(myself->m_start_lock);
1224  myself->m_start_complete_cv.wait(lock, [&]{return myself->m_start_complete;});
1225  }
1226  try {
1227  myself->Run();
1228  } catch (...) {
1229  myself->m_logger->Warning(kLogXrdClHttp, "Curl worker got an exception");
1230  {
1231  std::unique_lock lock(m_workers_mutex);
1232  auto iter = std::remove_if(m_workers.begin(), m_workers.end(), [&](std::unique_ptr<XrdClHttp::CurlWorker> &worker){return worker.get() == myself;});
1233  m_workers.erase(iter);
1234  }
1235  }
1236 }

References XrdClHttp::kLogXrdClHttp, Run(), and XrdCl::Log::Warning().

+ Here is the call graph for this function:

◆ SetMaintenancePeriod()

static void XrdClHttp::CurlWorker::SetMaintenancePeriod ( unsigned  maint)
inlinestatic

Definition at line 70 of file XrdClHttpWorker.hh.

70  {
71  m_maintenance_period.store(maint, std::memory_order_relaxed);
72  }

◆ Start()

void CurlWorker::Start ( std::unique_ptr< XrdClHttp::CurlWorker >  self,
std::thread  tid 
)

Definition at line 1207 of file XrdClHttpUtil.cc.

1208 {
1209  {
1210  std::unique_lock lock(m_workers_mutex);
1211  m_workers.emplace_back(std::move(self));
1212  m_self_tid = std::move(tid);
1213  }
1214  std::unique_lock lock(m_start_lock);
1215  m_start_complete = true;
1216  m_start_complete_cv.notify_one();
1217 }

The documentation for this class was generated from the following files: