XRootD
XrdClHttpUtil.hh
Go to the documentation of this file.
1 /******************************************************************************/
2 /* Copyright (C) 2025, Pelican Project, Morgridge Institute for Research */
3 /* */
4 /* This file is part of the XrdClHttp client plugin for XRootD. */
5 /* */
6 /* XRootD is free software: you can redistribute it and/or modify it under */
7 /* the terms of the GNU Lesser General Public License as published by the */
8 /* Free Software Foundation, either version 3 of the License, or (at your */
9 /* option) any later version. */
10 /* */
11 /* XRootD is distributed in the hope that it will be useful, but WITHOUT */
12 /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */
13 /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */
14 /* License for more details. */
15 /* */
16 /* The copyright holder's institutional names and contributor's names may not */
17 /* be used to endorse or promote products derived from this software without */
18 /* specific prior written permission of the institution or contributor. */
19 /******************************************************************************/
20 
21 #ifndef XRDCLHTTPUTIL_HH
22 #define XRDCLHTTPUTIL_HH
23 
24 #include "XrdClHttpChecksum.hh"
25 #include "XrdClHttpOptionsCache.hh"
26 #include "XrdClHttpResponseInfo.hh"
27 
28 #include <chrono>
29 #include <condition_variable>
30 #include <ctime>
31 #include <deque>
32 #include <memory>
33 #include <mutex>
34 #include <string>
35 #include <unordered_map>
36 #include <vector>
37 
38 // Forward dec'ls
39 typedef void CURL;
40 struct curl_slist;
41 
42 namespace XrdCl {
43 
44 class ResponseHandler;
45 class Log;
46 class URL;
47 
48 }
49 
50 namespace XrdClHttp {
51 
52 class CurlOperation;
53 
54 const uint64_t kLogXrdClHttp = 73173;
55 
56 bool HTTPStatusIsError(unsigned status);
57 
58 std::pair<uint16_t, uint32_t> HTTPStatusConvert(unsigned status);
59 
60 // Read a bearer token from the standard XrdCl environment options, falling
61 // back to the corresponding process environment variables.
62 std::string GetBearerToken(XrdCl::Log *logger = nullptr);
63 
64 // Return whether bearer-token authentication is the first available method
65 // requested through XrdSecPROTOCOL. With no explicit preference, use a token
66 // whenever one is available.
67 bool ShouldUseBearerToken(const std::string &protocols,
68  bool hasX509Credential,
69  bool hasBearerToken);
70 
71 // Add a bearer-token Authorization header through the common XrdClHttp
72 // authentication path when it is the preferred available credential.
74  const XrdCl::URL &url,
75  std::vector<std::pair<std::string, std::string>> &headers,
76  XrdCl::Log *logger = nullptr);
77 
78 // Trim the left side of a string_view for space
79 std::string_view ltrim_view(const std::string_view &input_view);
80 
81 // Trim the left and right side of a string_view of whitespace
82 std::string_view trim_view(const std::string_view &input_view);
83 
84 // Apply the common XrdClHttp configuration to a curl handle.
85 void ConfigureHandle(CURL *curl, bool verbose);
86 
87 // Returns a newly-created curl handle (no internal caching) with the
88 // various configurations needed to be used by XrdClHttp
89 CURL *GetHandle(bool verbose);
90 
91 // Parser for headers as emitted by libcurl.
92 //
93 // Records specific headers known to be used by the project but ignores others.
94 class HeaderParser {
95 public:
97 
98  bool Parse(const std::string &headers);
99 
100  int64_t GetContentLength() const {return m_content_length;}
101 
102  // Modification time from Last-Modified, or -1 when the header is absent.
103  time_t GetModTime() const {return m_modtime;}
104 
105  uint64_t GetOffset() const {return m_response_offset;}
106 
107  static bool Canonicalize(std::string &headerName);
108 
109  bool HeadersDone() const {return m_recv_all_headers;}
110 
111  // Move the received headers to the caller.
112  //
113  // Only invoke once HeadersDone() returns true.
114  ResponseInfo::HeaderMap && MoveHeaders() {return std::move(m_headers);}
115 
116  int GetStatusCode() const {return m_status_code;}
117 
118  // Setter for the status code
119  // Intended for use in unit tests.
120  void SetStatusCode(int sc) {m_status_code = sc;}
121 
122  // Return whether the server response specified this is a multipart range.
123  bool IsMultipartByterange() const {return m_multipart_byteranges;}
124 
125  // Return the separator specified in the server response with the
126  // `--` prefix included..
127  const std::string &MultipartSeparator() const {return m_multipart_sep;}
128 
129  // Set the separator used for multipart messages; a `--` prefix
130  // will be added to the Getter.
131  void SetMultipartSeparator(const std::string_view &sep) {
132  m_multipart_sep = "--" + std::string(sep);
133  m_multipart_byteranges = true;
134  }
135 
137  {
138  return VerbsCache::HttpVerbs(m_allow_verbs);
139  }
140 
141  std::string GetStatusMessage() const {return m_resp_message;}
142 
143  const std::string &GetLocation() const {return m_location;}
144  const std::string &GetETag() const {return m_etag;}
145  const std::string &GetCacheControl() const {return m_cache_control;}
146 
147  // Returns a reference to the checksums parsed from the headers.
148  const XrdClHttp::ChecksumInfo &GetChecksums() const {return m_checksums;}
149 
150  // Parse a RFC 3230 header, updating the checksum info structure.
151  static void ParseDigest(const std::string &digest, XrdClHttp::ChecksumInfo &info);
152 
153  // Decode a base64-encoded string into a binary buffer.
154  static bool Base64Decode(
155  std::string_view input,
156  std::array<unsigned char, g_max_checksum_length> &output);
157 
158  // Convert a checksum type to a RFC 3230 digest name.
159  static std::string ChecksumTypeToDigestName(XrdClHttp::ChecksumType type);
160 
161 private:
162 
163  static bool validHeaderByte(unsigned char c);
164 
165  int64_t m_content_length{-1};
166  time_t m_modtime{-1};
167  uint64_t m_response_offset{0};
168 
169  XrdClHttp::ChecksumInfo m_checksums;
170 
171  bool m_recv_all_headers{false};
172  bool m_recv_status_line{false};
173  bool m_multipart_byteranges{false};
174 
175  int m_status_code{-1};
176  std::string m_resp_protocol;
177  std::string m_resp_message;
178  std::string m_location;
179  std::string m_multipart_sep;
180  std::string m_etag;
181  std::string m_cache_control;
182 
183  ResponseInfo::HeaderMap m_headers;
184 
186 };
187 
197 public:
198  HandlerQueue(unsigned max_pending_ops);
199 
200  void Produce(std::shared_ptr<CurlOperation> handler);
201 
202  std::shared_ptr<CurlOperation> Consume(std::chrono::steady_clock::duration);
203  std::shared_ptr<CurlOperation> TryConsume();
204 
205  int PollFD() const {return m_read_fd;}
206 
207  CURL *GetHandle();
208  void RecycleHandle(CURL *);
209 
210  // Check all the operations in queue to see if any have expired.
211  //
212  // Each curl operation has a header timeout; if no headers have been received
213  // by the time the timeout expires, the operation is considered to have
214  // expired. This function checks all operations in the queue and
215  // removes any that have expired.
216  void Expire();
217 
218  void Shutdown();
219  // Cleanup all idle handles in current thread.
220  void ReleaseHandles();
221 
222  // Returns the class default number of pending operations.
223  static unsigned GetDefaultMaxPendingOps() {return m_default_max_pending_ops;}
224 
225  // Returns a summary of the queue's performance statistics.
226  static std::string GetMonitoringJson();
227 
228 private:
229  bool m_shutdown{false};
230  std::deque<std::shared_ptr<CurlOperation>> m_ops;
231  static std::atomic<uint64_t> m_ops_consumed; // Count of operations consumed from the queue.
232  static std::atomic<uint64_t> m_ops_produced; // Count of operations added to the queue.
233  static std::atomic<uint64_t> m_ops_rejected; // Count of operations rejected by the queue.
234  thread_local static std::vector<CURL*> m_handles;
235  std::condition_variable m_consumer_cv;
236  std::condition_variable m_producer_cv;
237  std::mutex m_mutex;
238  const static unsigned m_default_max_pending_ops{50};
239  const unsigned m_max_pending_ops{50};
240  int m_read_fd{-1};
241  int m_write_fd{-1};
242 };
243 
244 }
245 
246 #endif // XRDCLHTTPUTIL_HH
void CURL
std::shared_ptr< CurlOperation > Consume(std::chrono::steady_clock::duration)
HandlerQueue(unsigned max_pending_ops)
void Produce(std::shared_ptr< CurlOperation > handler)
static std::string GetMonitoringJson()
static unsigned GetDefaultMaxPendingOps()
std::shared_ptr< CurlOperation > TryConsume()
const std::string & GetETag() const
void SetMultipartSeparator(const std::string_view &sep)
ResponseInfo::HeaderMap && MoveHeaders()
static void ParseDigest(const std::string &digest, XrdClHttp::ChecksumInfo &info)
const XrdClHttp::ChecksumInfo & GetChecksums() const
std::string GetStatusMessage() const
uint64_t GetOffset() const
bool IsMultipartByterange() const
int64_t GetContentLength() const
const std::string & MultipartSeparator() const
static bool Canonicalize(std::string &headerName)
time_t GetModTime() const
const std::string & GetLocation() const
bool Parse(const std::string &headers)
VerbsCache::HttpVerbs GetAllowedVerbs() const
static std::string ChecksumTypeToDigestName(XrdClHttp::ChecksumType type)
static bool Base64Decode(std::string_view input, std::array< unsigned char, g_max_checksum_length > &output)
const std::string & GetCacheControl() const
std::unordered_map< std::string, HeaderValues > HeaderMap
Handle diagnostics.
Definition: XrdClLog.hh:101
URL representation.
Definition: XrdClURL.hh:31
std::pair< uint16_t, uint32_t > HTTPStatusConvert(unsigned status)
void InjectBearerToken(const XrdCl::URL &url, std::vector< std::pair< std::string, std::string >> &headers, XrdCl::Log *logger=nullptr)
CURL * GetHandle(bool verbose)
bool HTTPStatusIsError(unsigned status)
bool ShouldUseBearerToken(const std::string &protocols, bool hasX509Credential, bool hasBearerToken)
std::string_view ltrim_view(const std::string_view &input_view)
std::string GetBearerToken(XrdCl::Log *logger=nullptr)
void ConfigureHandle(CURL *curl, bool verbose)
std::string_view trim_view(const std::string_view &input_view)
XrdSysError Log
Definition: XrdConfig.cc:113
const uint64_t kLogXrdClHttp