Line data Source code
1 : #include "http_client_base.hpp"
2 : #include "client_error.hpp"
3 : #include "websocket_util.hpp"
4 : #include "../../util/logger.hpp"
5 : #include "../../asio/sockets/ssl_socket.hpp"
6 : #include "../../asio/sockets/unix_socket.hpp"
7 : #include "../../asio/sockets/websocket.hpp"
8 : #include "../../asio/ssl/certificate_manager.hpp"
9 :
10 : #include <algorithm>
11 : #include <filesystem>
12 : #include <fstream>
13 : #include <sstream>
14 : #include <string_view>
15 : #include <boost/algorithm/string.hpp>
16 :
17 : namespace thinger::http {
18 :
19 : namespace {
20 :
21 : // The CAs a TLS context trusts besides the system ones: those of the client, then those
22 : // of the request
23 : struct trusted_cas {
24 : std::vector<const std::string*> files;
25 : std::vector<const std::string*> pems;
26 :
27 : trusted_cas() = default;
28 762 : trusted_cas(const std::vector<std::string>& client_files, const std::vector<std::string>& client_pems,
29 762 : const http_request& request) {
30 774 : for (const auto& file : client_files) files.push_back(&file);
31 765 : for (const auto& file : request.get_ca_files()) files.push_back(&file);
32 1479 : for (const auto& pem : client_pems) pems.push_back(&pem);
33 774 : for (const auto& pem : request.get_ca_pems()) pems.push_back(&pem);
34 762 : }
35 : };
36 :
37 : // What a context trusts identifies it: files by path, modification time and size, so a
38 : // file changed in place is loaded again
39 762 : std::string tls_context_key(bool system_ca, const trusted_cas& cas) {
40 762 : std::string key = system_ca ? "system" : "custom";
41 777 : for (const auto* file : cas.files) {
42 15 : std::error_code time_ec, size_ec;
43 15 : auto time = std::filesystem::last_write_time(*file, time_ec);
44 15 : auto size = std::filesystem::file_size(*file, size_ec);
45 : // In nanoseconds: the count of a file time is a 128-bit integer with libc++
46 15 : auto nanoseconds = std::chrono::duration_cast<std::chrono::nanoseconds>(time.time_since_epoch()).count();
47 33 : key += "\nfile:" + *file + ":" + (time_ec ? "-" : std::to_string(nanoseconds)) +
48 51 : ":" + (size_ec ? "-" : std::to_string(size));
49 : }
50 1491 : for (const auto* pem : cas.pems) key += "\npem:" + *pem;
51 762 : return key;
52 0 : }
53 :
54 : // False if a CA file or PEM cannot be loaded; the system CAs are optional
55 327 : bool load_cas(boost::asio::ssl::context& context, bool system_ca, const trusted_cas& cas) {
56 327 : boost::system::error_code ec;
57 327 : if (system_ca) {
58 324 : context.set_default_verify_paths(ec);
59 324 : if (ec) LOG_WARNING("cannot load the system CAs: {}", ec.message());
60 : }
61 339 : for (const auto* file : cas.files) {
62 15 : context.load_verify_file(*file, ec);
63 15 : if (ec) {
64 3 : LOG_ERROR("cannot load the CA file {}: {}", *file, ec.message());
65 3 : return false;
66 : }
67 : }
68 612 : for (const auto* pem : cas.pems) {
69 291 : context.add_certificate_authority(boost::asio::buffer(*pem), ec);
70 291 : if (ec) {
71 3 : LOG_ERROR("cannot load a CA certificate (PEM): {}", ec.message());
72 3 : return false;
73 : }
74 : }
75 321 : return true;
76 : }
77 :
78 : // The content of a PEM file of the client certificate: false (logged) if it cannot be read
79 18 : bool read_pem_file(const std::string& path, std::string& content) {
80 18 : std::ifstream file(path, std::ios::binary);
81 18 : std::ostringstream stream;
82 18 : if (file) stream << file.rdbuf();
83 18 : if (!file || file.bad()) {
84 6 : LOG_ERROR("cannot read the file {} of the client certificate", path);
85 6 : return false;
86 : }
87 12 : content = stream.str();
88 12 : return true;
89 18 : }
90 :
91 72 : boost::system::error_code certificate_error_code(asio::certificate_error error) {
92 72 : switch (error) {
93 45 : case asio::certificate_error::none: return {};
94 9 : case asio::certificate_error::invalid_certificate: return client_error::invalid_client_certificate;
95 6 : case asio::certificate_error::invalid_private_key: return client_error::invalid_client_key;
96 12 : case asio::certificate_error::key_mismatch: return client_error::client_key_mismatch;
97 : }
98 0 : return client_error::invalid_client_certificate;
99 : }
100 :
101 : // 303 always changes the method to GET, and 301 and 302 change POST, PUT and DELETE to GET,
102 : // as browsers do; 307 and 308 keep it
103 223 : method redirect_method(int status, method original) {
104 223 : if (status == 303) return method::GET;
105 220 : if ((status == 301 || status == 302) &&
106 202 : (original == method::POST || original == method::PUT || original == method::DELETE)) {
107 6 : return method::GET;
108 : }
109 214 : return original;
110 : }
111 :
112 : // Whether a reference has a scheme: text before a ':' that comes before any '/', '?' or '#'
113 461 : bool has_scheme(std::string_view reference) {
114 461 : auto end = reference.find_first_of(":/?#");
115 461 : return end != std::string_view::npos && reference[end] == ':';
116 : }
117 :
118 : // RFC 3986 section 5.2.4: drop each "." segment, and each ".." with the segment before it,
119 : // stopping at the root
120 87 : std::string remove_dot_segments(std::string_view path) {
121 87 : std::vector<std::string_view> segments;
122 87 : bool directory = false;
123 276 : for (size_t start = path.starts_with('/') ? 1 : 0; start <= path.size();) {
124 189 : size_t end = std::min(path.find('/', start), path.size());
125 189 : auto segment = path.substr(start, end - start);
126 189 : directory = segment == "." || segment == "..";
127 189 : if (segment == ".." && !segments.empty()) segments.pop_back();
128 174 : else if (!directory) segments.push_back(segment);
129 189 : start = end + 1;
130 : }
131 87 : std::string result;
132 234 : for (auto segment : segments) (result += '/') += segment;
133 87 : if (result.empty() || directory) result += '/';
134 174 : return result;
135 87 : }
136 :
137 : // The URL a Location points to, resolved against the request when it is relative (RFC 3986
138 : // section 5.2.2); one with a scheme is taken as it is
139 238 : std::string resolve_location(const http_request& request, const std::string& location) {
140 238 : if (has_scheme(location)) return location;
141 87 : std::string_view reference = location;
142 87 : std::string origin = request.get_base_path();
143 87 : std::string_view current = request.get_uri();
144 87 : current = current.substr(0, current.find_first_of("?#"));
145 87 : std::string path;
146 87 : if (reference.starts_with("//")) {
147 18 : auto authority = std::min(reference.find_first_of("/?#", 2), reference.size());
148 18 : origin = (request.is_ssl() ? "https:" : "http:") + std::string(reference.substr(0, authority));
149 18 : reference.remove_prefix(authority);
150 18 : current = "/"; // an empty path is the root of that host
151 : }
152 87 : auto path_end = std::min(reference.find_first_of("?#"), reference.size());
153 87 : auto relative = reference.substr(0, path_end);
154 87 : if (relative.starts_with('/')) path = relative;
155 36 : else if (relative.empty()) path = current;
156 30 : else path = std::string(current.substr(0, current.rfind('/') + 1)).append(relative);
157 87 : return origin + remove_dot_segments(path) + std::string(reference.substr(path_end));
158 87 : }
159 :
160 : // ws:// and wss:// URLs, or http:// and https:// ones
161 313 : std::optional<websocket_util::url_components> parse_websocket_url(const std::string& url) {
162 313 : if (auto components = websocket_util::parse_websocket_url(url)) return components;
163 313 : if (url.starts_with("http://")) return websocket_util::parse_websocket_url("ws://" + url.substr(7));
164 48 : if (url.starts_with("https://")) return websocket_util::parse_websocket_url("wss://" + url.substr(8));
165 0 : return std::nullopt;
166 : }
167 :
168 : // Turn a request into the opening handshake (RFC 6455 section 4.1); returns its key
169 310 : std::string set_upgrade_headers(http_request& request, const websocket_util::url_components& url,
170 : const std::string& subprotocol) {
171 310 : request.set_url((url.secure ? "https://" : "http://") + url.host + ":" + url.port + url.path);
172 310 : request.set_method(method::GET);
173 : // set_header: the request may already have them
174 1240 : request.set_header("Upgrade", "websocket");
175 1240 : request.set_header("Connection", "Upgrade");
176 310 : std::string key = websocket_util::generate_websocket_key();
177 930 : request.add_header("Sec-WebSocket-Key", key);
178 1240 : request.add_header("Sec-WebSocket-Version", "13");
179 392 : if (!subprotocol.empty()) request.add_header("Sec-WebSocket-Protocol", subprotocol);
180 310 : return key;
181 0 : }
182 :
183 : // The accept key must match ours, and the subprotocol accepted be one offered (RFC 6455
184 : // section 4.1)
185 265 : bool valid_upgrade(const http_response& response, const std::string& key, const std::string& subprotocol) {
186 265 : if (!websocket_util::validate_accept_key(response.get_header("Sec-WebSocket-Accept"), key)) {
187 0 : LOG_ERROR("Invalid Sec-WebSocket-Accept key");
188 0 : return false;
189 : }
190 265 : const auto& accepted = response.get_header("Sec-WebSocket-Protocol");
191 265 : if (accepted.empty()) return true;
192 29 : std::vector<std::string> offered;
193 29 : boost::split(offered, subprotocol, boost::is_any_of(","));
194 61 : for (auto& protocol : offered) boost::trim(protocol);
195 29 : if (std::find(offered.begin(), offered.end(), accepted) == offered.end()) {
196 3 : LOG_ERROR("WebSocket server accepted a subprotocol not offered: {}", accepted);
197 3 : return false;
198 : }
199 26 : return true;
200 29 : }
201 :
202 : }
203 :
204 2728 : http_client_base::http_client_base()
205 8184 : : env_proxy_(std::make_shared<const proxy_resolver>(proxy_config::from_environment())) {
206 2728 : }
207 :
208 2728 : http_client_base::~http_client_base() {
209 2728 : LOG_DEBUG("Destroying HTTP client base");
210 2728 : pool_.clear();
211 2728 : }
212 :
213 15 : bool http_client_base::is_same_origin(const std::string& url1, const std::string& url2) {
214 15 : http_request req1, req2;
215 15 : if (!req1.set_url(url1) || !req2.set_url(url2)) {
216 0 : return false;
217 : }
218 15 : return req1.get_base_path() == req2.get_base_path();
219 15 : }
220 :
221 7725 : std::shared_ptr<http_request> http_client_base::create_request(method m, const std::string& url) {
222 7725 : auto request = std::make_shared<http_request>();
223 7725 : request->set_method(m);
224 7725 : request->set_url(url);
225 7725 : if (!unix_socket_.empty()) {
226 570 : request->set_unix_socket(unix_socket_);
227 : }
228 7725 : apply_default_headers(request);
229 7725 : return request;
230 0 : }
231 :
232 8365 : void http_client_base::apply_default_headers(const std::shared_ptr<http_request>& request) {
233 8365 : if (!request->has_header("User-Agent")) {
234 25086 : request->add_header("User-Agent", user_agent_);
235 : }
236 8365 : if (auto_decompress_ && !request->has_header("Accept-Encoding")) {
237 41245 : request->add_header("Accept-Encoding", "gzip, deflate");
238 : }
239 8365 : }
240 :
241 135 : http_client_base& http_client_base::proxy(const std::string& url, const std::string& no_proxy) {
242 135 : return proxy(proxy_config{url, url, no_proxy});
243 : }
244 :
245 138 : http_client_base& http_client_base::proxy(const proxy_config& config) {
246 138 : proxy_ = std::make_shared<const proxy_resolver>(config);
247 138 : return *this;
248 : }
249 :
250 18 : http_client_base& http_client_base::proxy_from_environment(bool enabled) {
251 18 : env_proxy_ = enabled ? std::make_shared<const proxy_resolver>(proxy_config::from_environment()) : nullptr;
252 18 : return *this;
253 : }
254 :
255 7934 : proxy_resolver::choice http_client_base::select_proxy(const http_request& request) const {
256 7934 : if (!request.get_unix_socket().empty()) return {};
257 : // The settings of the request, of the client, or of the environment: the first set
258 7931 : const auto& host = request.get_host();
259 7931 : const auto& port = request.get_port();
260 7931 : if (const auto& config = request.get_proxy_config()) {
261 24 : return proxy_resolver(*config).select(request.is_ssl(), host, port);
262 : }
263 7907 : if (proxy_) return proxy_->select(request.is_ssl(), host, port);
264 7706 : if (env_proxy_) return env_proxy_->select(request.is_ssl(), host, port);
265 15 : return {};
266 18 : }
267 :
268 87 : std::string http_client_base::proxy_for(const http_request& request) const {
269 87 : auto choice = select_proxy(request);
270 174 : return choice.proxy ? choice.proxy->url() : std::string{};
271 87 : }
272 :
273 78 : std::string http_client_base::proxy_for(const std::string& url) const {
274 78 : http_request request;
275 78 : if (!request.set_url(url)) return {};
276 75 : return proxy_for(request);
277 78 : }
278 :
279 51 : boost::system::error_code http_client_base::client_certificate(const std::string& certificate,
280 : const std::string& private_key,
281 : const std::string& chain) {
282 51 : std::string full_chain = certificate;
283 51 : if (!chain.empty() && !full_chain.empty() && full_chain.back() != '\n') full_chain += '\n';
284 51 : full_chain += chain;
285 : // Checked as the contexts of the connections will take it
286 51 : boost::asio::ssl::context test(boost::asio::ssl::context::tls_client);
287 51 : if (auto ec = certificate_error_code(asio::use_certificate(test, full_chain, private_key))) return ec;
288 24 : std::lock_guard lock(tls_mutex_);
289 24 : identity_ = client_identity{std::move(full_chain), private_key, ++next_identity_id_};
290 24 : return {};
291 51 : }
292 :
293 9 : boost::system::error_code http_client_base::client_certificate_file(const std::string& certificate_path,
294 : const std::string& private_key_path,
295 : const std::string& chain_path) {
296 9 : std::string certificate, key, chain;
297 9 : if (!read_pem_file(certificate_path, certificate)) return client_error::invalid_client_certificate;
298 6 : if (!read_pem_file(private_key_path, key)) return client_error::invalid_client_key;
299 3 : if (!chain_path.empty() && !read_pem_file(chain_path, chain)) return client_error::invalid_client_certificate;
300 3 : auto ec = client_certificate(certificate, key, chain);
301 3 : if (ec) LOG_ERROR("cannot use the client certificate {} with the key {}: {}", certificate_path, private_key_path, ec.message());
302 3 : return ec;
303 9 : }
304 :
305 3 : void http_client_base::clear_client_certificate() {
306 3 : std::lock_guard lock(tls_mutex_);
307 3 : identity_ = client_identity{};
308 3 : }
309 :
310 15 : bool http_client_base::has_client_certificate() const {
311 15 : std::lock_guard lock(tls_mutex_);
312 15 : return identity_.id != 0;
313 15 : }
314 :
315 869 : std::shared_ptr<boost::asio::ssl::context> http_client_base::tls_context(
316 : const http_request& request, uint64_t& id, boost::system::error_code& ec) {
317 869 : trusted_cas cas;
318 869 : std::string key = "none";
319 869 : if (verify_ssl_) {
320 762 : cas = trusted_cas(ca_files_, ca_pems_, request);
321 762 : key = tls_context_key(system_ca_, cas);
322 : }
323 :
324 869 : std::lock_guard lock(tls_mutex_);
325 : // The certificate presented tells contexts apart too (by its id, never by the key)
326 869 : if (identity_.id != 0) key += "\nclient:" + std::to_string(identity_.id);
327 869 : if (auto it = tls_contexts_.find(key); it != tls_contexts_.end()) {
328 450 : id = it->second.id;
329 450 : return it->second.context;
330 : }
331 :
332 419 : auto context = std::make_shared<boost::asio::ssl::context>(boost::asio::ssl::context::sslv23_client);
333 419 : if (!verify_ssl_) {
334 92 : context->set_verify_mode(boost::asio::ssl::verify_none);
335 : } else {
336 327 : context->set_verify_mode(boost::asio::ssl::verify_peer);
337 327 : if (!load_cas(*context, system_ca_, cas)) {
338 6 : ec = client_error::invalid_ca_certificate;
339 6 : return nullptr;
340 : }
341 : }
342 : // Checked when set, so this fails only if OpenSSL changed its mind
343 413 : if (identity_.id != 0) {
344 21 : ec = certificate_error_code(asio::use_certificate(*context, identity_.chain, identity_.key));
345 21 : if (ec) return nullptr;
346 : }
347 :
348 : // Bounded: CAs that change, or many per-request sets, would grow it forever. The
349 : // connections keep the contexts they use
350 413 : if (tls_contexts_.size() >= MAX_TLS_CONTEXTS) tls_contexts_.clear();
351 413 : id = ++next_tls_id_;
352 413 : tls_contexts_.emplace(std::move(key), tls_entry{id, context});
353 413 : return context;
354 869 : }
355 :
356 7853 : http_client_base::connection_plan http_client_base::plan_connection(const http_request& request,
357 : boost::system::error_code& ec) {
358 7853 : connection_plan plan;
359 7853 : uint64_t tls_id = 0;
360 7853 : if (request.is_ssl()) {
361 869 : plan.tls = tls_context(request, tls_id, ec);
362 869 : if (ec) return plan;
363 : }
364 :
365 7847 : auto choice = select_proxy(request);
366 7847 : if (!choice.error.empty()) {
367 12 : LOG_ERROR("cannot use the proxy for {}: {}", util::url::loggable_uri(request.get_url(), body_logging_.redact),
368 : choice.error);
369 12 : ec = client_error::invalid_proxy;
370 12 : return plan;
371 : }
372 :
373 : // Connections are shared only by requests reaching their origin the same way, and
374 : // trusting the same CAs: never a direct one for a proxied request, nor the other way
375 7835 : plan.key = "tls:" + std::to_string(tls_id);
376 7835 : if (choice.proxy) {
377 117 : plan.route.proxy = choice.proxy;
378 : // Plain requests are forwarded by the proxy. TLS goes end to end through a tunnel, as
379 : // do protocol upgrades (WebSocket), which a forwarding proxy may not pass on, as
380 : // browsers do
381 117 : plan.route.tunnel = request.is_ssl() || request.has_header(header::upgrade);
382 117 : plan.route.tls = plan.tls;
383 117 : plan.key += (plan.route.tunnel ? "|connect:" : "|forward:") + choice.proxy->key();
384 : }
385 7835 : return plan;
386 7847 : }
387 :
388 4601 : std::shared_ptr<client_connection> http_client_base::make_connection(const connection_plan& plan) {
389 4601 : auto& io_context = get_io_context();
390 4601 : std::shared_ptr<client_connection> connection;
391 4601 : if (plan.route.proxy) {
392 105 : connection = std::make_shared<client_connection>(io_context, plan.route, timeout_);
393 : } else {
394 4496 : std::shared_ptr<thinger::asio::socket> sock;
395 4496 : if (plan.tls) {
396 593 : sock = std::make_shared<thinger::asio::ssl_socket>("http_client", io_context, plan.tls);
397 : } else {
398 3903 : sock = std::make_shared<thinger::asio::tcp_socket>("http_client", io_context);
399 : }
400 4496 : connection = std::make_shared<client_connection>(sock, timeout_);
401 4496 : }
402 4601 : connection->set_max_content_size(max_content_size_);
403 4601 : connection->set_decompress(auto_decompress_);
404 4601 : connection->set_body_logging(body_logging_);
405 4601 : return connection;
406 0 : }
407 :
408 8284 : connection_pool::lease http_client_base::get_connection(const std::shared_ptr<http_request>& request,
409 : boost::system::error_code& ec, bool reuse) {
410 : // A request on which a setter refused a header is not sent, as Go's client does: it fails
411 : // as an unparseable URL does, before any connection is opened or reused
412 8284 : if (request->has_refused_header()) {
413 60 : ec = boost::asio::error::invalid_argument;
414 60 : return {};
415 : }
416 8224 : ec = {};
417 8224 : const std::string& socket_path = request->get_unix_socket();
418 :
419 : // Connections are interchangeable if they reach the same origin the same way. Through a
420 : // forwarding proxy, they reach the proxy: any origin is requested on them, as curl does
421 8224 : std::string key;
422 8224 : std::string origin;
423 8224 : connection_plan plan;
424 8224 : if (!socket_path.empty()) {
425 : // Unix sockets are local: never through a proxy
426 630 : key = origin = "unix:" + socket_path;
427 : } else {
428 7594 : plan = plan_connection(*request, ec);
429 7594 : if (ec) return {};
430 7576 : if (plan.route.proxy && !plan.route.tunnel) {
431 66 : key = plan.key;
432 66 : origin = "proxy " + plan.route.proxy->url();
433 : } else {
434 7510 : origin = (request->is_ssl() ? "https://" : "http://") + request->get_host() + ":" + request->get_port();
435 7510 : key = origin + "|" + plan.key;
436 : }
437 : }
438 :
439 8206 : if (reuse) {
440 8143 : if (auto lease = pool_.acquire(key)) {
441 3420 : LOG_DEBUG("Reusing connection from pool for {}", origin);
442 3420 : lease->set_max_content_size(max_content_size_);
443 3420 : lease->set_decompress(auto_decompress_);
444 3420 : lease->set_body_logging(body_logging_);
445 3420 : lease->set_timeout(timeout_);
446 3420 : return lease;
447 8143 : }
448 : }
449 :
450 4786 : std::shared_ptr<client_connection> connection;
451 4786 : if (!socket_path.empty()) {
452 444 : auto sock = std::make_shared<thinger::asio::unix_socket>("http_client", get_io_context());
453 444 : connection = std::make_shared<client_connection>(sock, socket_path, timeout_);
454 444 : connection->set_max_content_size(max_content_size_);
455 444 : connection->set_decompress(auto_decompress_);
456 444 : connection->set_body_logging(body_logging_);
457 444 : } else {
458 4342 : LOG_DEBUG("Creating new connection for {}{}", request->get_host(),
459 : plan.route.proxy ? " through the proxy " + plan.route.proxy->url() : std::string{});
460 4342 : connection = make_connection(plan);
461 : }
462 4786 : return pool_.add(key, origin, std::move(connection));
463 8224 : }
464 :
465 874 : http_client_base& http_client_base::max_idle_connections_per_host(size_t max) {
466 874 : auto config = pool_.get_config();
467 874 : config.max_idle_per_host = max;
468 874 : pool_.configure(config);
469 874 : return *this;
470 : }
471 :
472 0 : http_client_base& http_client_base::max_idle_connections(size_t max) {
473 0 : auto config = pool_.get_config();
474 0 : config.max_idle = max;
475 0 : pool_.configure(config);
476 0 : return *this;
477 : }
478 :
479 30 : http_client_base& http_client_base::idle_timeout(std::chrono::milliseconds timeout) {
480 30 : auto config = pool_.get_config();
481 30 : config.idle_timeout = timeout;
482 30 : pool_.configure(config);
483 30 : return *this;
484 : }
485 :
486 1551 : bool http_client_base::can_retry(const client_connection& connection, const http_request& request) {
487 : // A server may close an idle connection at any time (RFC 9112 section 9.3.1): a request
488 : // failing on a reused one before any byte of its response arrived most likely met that.
489 : // It is sent again on a new connection if that is safe, as Go does: nothing of it was
490 : // written, or it is idempotent (RFC 9110 section 9.2.2) or has an idempotency key.
491 : // Otherwise, e.g. for a POST, the server may have processed it, and the error is returned.
492 : // Never after a timeout: the server may be just slow
493 1551 : const auto& ec = connection.last_error();
494 1551 : if (!ec || !connection.reused() || connection.response_started()) return false;
495 693 : if (ec == boost::asio::error::timed_out || ec == boost::asio::error::operation_aborted) return false;
496 81 : if (connection.request_unsent()) return true;
497 81 : switch (request.get_method()) {
498 54 : case method::GET:
499 : case method::HEAD:
500 : case method::OPTIONS:
501 : case method::TRACE:
502 : case method::PUT:
503 : case method::DELETE:
504 54 : return true;
505 27 : default:
506 27 : return request.has_header("Idempotency-Key") || request.has_header("X-Idempotency-Key");
507 : }
508 : }
509 :
510 8042 : awaitable<std::shared_ptr<http_response>> http_client_base::send_request(const std::shared_ptr<http_request>& request,
511 : connection_pool::lease& lease) {
512 : auto response = co_await lease->send_request(request);
513 : if (!response && can_retry(*lease, *request)) {
514 : LOG_DEBUG("reused connection to {} failed before the response ({}): sending again on a new one",
515 : request->get_host(), lease->last_error().message());
516 : boost::system::error_code ec;
517 : if (auto fresh = get_connection(request, ec, false)) {
518 : lease = std::move(fresh);
519 : response = co_await lease->send_request(request);
520 : }
521 : }
522 : co_return response;
523 16084 : }
524 :
525 7444 : awaitable<client_response> http_client_base::get(const std::string& url, headers_map headers) {
526 : auto request = create_request(method::GET, url);
527 : for (const auto& [key, value] : headers) {
528 : request->add_header(key, value);
529 : }
530 : co_return co_await send(request);
531 14888 : }
532 :
533 376 : awaitable<client_response> http_client_base::post(const std::string& url, std::string body,
534 : std::string content_type, headers_map headers) {
535 : auto request = create_request(method::POST, url);
536 : if (!body.empty()) {
537 : request->set_content(std::move(body), std::move(content_type));
538 : }
539 : for (const auto& [key, value] : headers) {
540 : request->add_header(key, value);
541 : }
542 : co_return co_await send(request);
543 752 : }
544 :
545 3 : awaitable<client_response> http_client_base::post(const std::string& url, const form& form, headers_map headers) {
546 3 : return post(url, form.body(), form.content_type(), std::move(headers));
547 : }
548 :
549 54 : awaitable<client_response> http_client_base::put(const std::string& url, std::string body,
550 : std::string content_type, headers_map headers) {
551 : auto request = create_request(method::PUT, url);
552 : if (!body.empty()) {
553 : request->set_content(std::move(body), std::move(content_type));
554 : }
555 : for (const auto& [key, value] : headers) {
556 : request->add_header(key, value);
557 : }
558 : co_return co_await send(request);
559 108 : }
560 :
561 45 : awaitable<client_response> http_client_base::patch(const std::string& url, std::string body,
562 : std::string content_type, headers_map headers) {
563 : auto request = create_request(method::PATCH, url);
564 : if (!body.empty()) {
565 : request->set_content(std::move(body), std::move(content_type));
566 : }
567 : for (const auto& [key, value] : headers) {
568 : request->add_header(key, value);
569 : }
570 : co_return co_await send(request);
571 90 : }
572 :
573 44 : awaitable<client_response> http_client_base::del(const std::string& url, headers_map headers) {
574 : auto request = create_request(method::DELETE, url);
575 : for (const auto& [key, value] : headers) {
576 : request->add_header(key, value);
577 : }
578 : co_return co_await send(request);
579 88 : }
580 :
581 19 : awaitable<client_response> http_client_base::head(const std::string& url, headers_map headers) {
582 : auto request = create_request(method::HEAD, url);
583 : for (const auto& [key, value] : headers) {
584 : request->add_header(key, value);
585 : }
586 : co_return co_await send(request);
587 38 : }
588 :
589 35 : awaitable<client_response> http_client_base::options(const std::string& url, headers_map headers) {
590 : auto request = create_request(method::OPTIONS, url);
591 : for (const auto& [key, value] : headers) {
592 : request->add_header(key, value);
593 : }
594 : co_return co_await send(request);
595 70 : }
596 :
597 7882 : awaitable<client_response> http_client_base::send(std::shared_ptr<http_request> request) {
598 : // A request not made by create_request() (a request_builder's) reaches the socket of the
599 : // client too, unless it names its own
600 : if (request->get_unix_socket().empty() && !unix_socket_.empty()) request->set_unix_socket(unix_socket_);
601 : co_return co_await send_with_redirects(std::move(request));
602 15764 : }
603 :
604 107 : awaitable<stream_result> http_client_base::send_streaming(std::shared_ptr<http_request> request,
605 : stream_callback callback) {
606 : if (request->get_unix_socket().empty() && !unix_socket_.empty()) request->set_unix_socket(unix_socket_);
607 : // Streamed bodies are handed as received, so they are not asked compressed. Before
608 : // apply_default_headers, which would ask for gzip and deflate
609 : if (!request->has_header("Accept-Encoding")) {
610 : request->add_header("Accept-Encoding", "identity");
611 : }
612 : apply_default_headers(request);
613 : boost::system::error_code ec;
614 : auto lease = get_connection(request, ec);
615 : if (!lease) {
616 : stream_result result;
617 : result.error = ec.message();
618 : co_return result;
619 : }
620 : // The callback is not called before the response starts: never twice for a request
621 : // sent again
622 20822 : auto forward = [&callback](const stream_info& info) { return callback(info); };
623 : auto result = co_await lease->send_request_streaming(request, forward);
624 : if (lease->last_error() && can_retry(*lease, *request)) {
625 : LOG_DEBUG("reused connection to {} failed before the response ({}): sending again on a new one",
626 : request->get_host(), lease->last_error().message());
627 : if (auto fresh = get_connection(request, ec, false)) {
628 : lease = std::move(fresh);
629 : result = co_await lease->send_request_streaming(request, forward);
630 : }
631 : }
632 : // Back to the pool only if the body was read whole, not aborted by the callback
633 : co_return result;
634 214 : }
635 :
636 : awaitable<std::pair<client_response, std::shared_ptr<client_connection>>>
637 9 : http_client_base::send_with_connection(std::shared_ptr<http_request> request) {
638 : std::shared_ptr<client_connection> connection;
639 : auto response = co_await send_with_redirects(std::move(request), &connection);
640 : co_return std::make_pair(std::move(response), std::move(connection));
641 18 : }
642 :
643 7891 : awaitable<client_response> http_client_base::send_with_redirects(std::shared_ptr<http_request> request,
644 : std::shared_ptr<client_connection>* used) {
645 : for (unsigned int redirect_count = 0;; ++redirect_count) {
646 : boost::system::error_code ec;
647 : auto lease = get_connection(request, ec);
648 : if (!lease) co_return client_response(ec, nullptr);
649 :
650 : auto response = co_await send_request(request, lease);
651 : if (used) *used = lease.connection();
652 :
653 : if (!response) {
654 : ec = lease->last_error();
655 : // With the answer of a proxy refusing the tunnel, if that was the error
656 : co_return client_response(ec ? ec : boost::asio::error::invalid_argument, lease->proxy_response());
657 : }
658 :
659 : // Back to the pool, if it can carry another request: the redirect may reuse it
660 : lease.release();
661 :
662 : if (!should_follow(*response, redirect_count)) {
663 : co_return client_response(boost::system::error_code{}, response);
664 : }
665 : std::string location = response->get_header("Location");
666 : if (LOG_DEBUG_ENABLED()) {
667 : LOG_DEBUG("Following redirect #{} to: {}", redirect_count + 1,
668 : util::url::loggable_uri(location, body_logging_.redact));
669 : }
670 : request = redirect_request(*request, *response, location);
671 : // A Location that is not a URL the client can reach ends the chain, with the redirect
672 : if (!request) co_return client_response(boost::asio::error::invalid_argument, response);
673 : }
674 15782 : }
675 :
676 6590 : bool http_client_base::should_follow(const http_response& response, unsigned int redirect_count) const {
677 6834 : return follow_redirects_ && response.is_redirect_response() && redirect_count < max_redirects_ &&
678 6834 : !response.get_header("Location").empty();
679 : }
680 :
681 238 : std::shared_ptr<http_request> http_client_base::redirect_request(http_request& request, const http_response& response,
682 : const std::string& location) {
683 238 : int status = response.get_status_code();
684 238 : auto url = resolve_location(request, location);
685 238 : auto next = std::make_shared<http_request>();
686 238 : if (!next->set_url(url)) return nullptr;
687 223 : next->set_method(redirect_method(status, request.get_method()));
688 : // A Location on the same host reaches it over the same Unix socket
689 223 : bool same_authority = !has_scheme(location) && !location.starts_with("//");
690 223 : next->set_unix_socket(same_authority ? request.get_unix_socket() : unix_socket_);
691 223 : apply_default_headers(next);
692 :
693 : // 307 and 308 send the body again (set_content sets its Content-Length)
694 223 : if ((status == 307 || status == 308) && !request.get_body().empty()) {
695 12 : next->set_content(request.get_body());
696 12 : if (request.has_header("Content-Type")) {
697 36 : next->add_header("Content-Type", request.get_header("Content-Type"));
698 : }
699 : }
700 :
701 : // Credentials only to the same origin
702 223 : if (request.has_header("Authorization") && is_same_origin(request.get_url(), url)) {
703 27 : next->add_header("Authorization", request.get_header("Authorization"));
704 9 : LOG_DEBUG("Preserving Authorization header for same-origin redirect");
705 : }
706 :
707 223 : if (request.get_cookie_store().update_from_headers(response)) {
708 0 : next->set_header(header::cookie, request.get_cookie_store().get_cookie_string());
709 : }
710 :
711 : // The proxy and CAs of the request apply to every hop; whether the proxy is used is
712 : // decided again for the host of each one (NO_PROXY)
713 223 : if (const auto& config = request.get_proxy_config()) next->set_proxy_config(*config);
714 223 : for (const auto& file : request.get_ca_files()) next->add_ca_file(file);
715 223 : for (const auto& pem : request.get_ca_pems()) next->add_ca_pem(pem);
716 223 : return next;
717 238 : }
718 :
719 282 : awaitable<std::optional<websocket_client>> http_client_base::upgrade_websocket(
720 : const std::string& url, const std::string& subprotocol) {
721 : auto request = std::make_shared<http_request>();
722 : request->set_url(url);
723 : co_return co_await upgrade_websocket(std::move(request), subprotocol);
724 564 : }
725 :
726 313 : awaitable<std::optional<websocket_client>> http_client_base::upgrade_websocket(
727 : std::shared_ptr<http_request> request, const std::string& subprotocol) {
728 : auto url = parse_websocket_url(request->get_url());
729 : if (!url) {
730 : LOG_ERROR("Invalid WebSocket URL: {}", util::url::loggable_uri(request->get_url(), body_logging_.redact));
731 : co_return std::nullopt;
732 : }
733 : auto key = set_upgrade_headers(*request, *url, subprotocol);
734 : apply_default_headers(request);
735 : if (request->has_refused_header()) {
736 : LOG_ERROR("WebSocket upgrade not sent: the request has an invalid header");
737 : co_return std::nullopt;
738 : }
739 :
740 : // A connection of its own, never pooled: it becomes the WebSocket. Over the Unix socket of
741 : // the request or of the client, as the other requests; otherwise through a proxy it is a
742 : // tunnel (CONNECT) for ws:// too, as the request has Upgrade
743 : if (request->get_unix_socket().empty() && !unix_socket_.empty()) request->set_unix_socket(unix_socket_);
744 : std::shared_ptr<client_connection> connection;
745 : if (!request->get_unix_socket().empty()) {
746 : auto sock = std::make_shared<thinger::asio::unix_socket>("http_client", get_io_context());
747 : connection = std::make_shared<client_connection>(sock, request->get_unix_socket(), timeout_);
748 : connection->set_body_logging(body_logging_);
749 : } else {
750 : boost::system::error_code ec;
751 : auto plan = plan_connection(*request, ec);
752 : if (ec) {
753 : LOG_ERROR("WebSocket connect error: {}", ec.message());
754 : co_return std::nullopt;
755 : }
756 : connection = make_connection(plan);
757 : }
758 :
759 : auto response = co_await connection->send_request(request);
760 : if (!response || response->get_status_code() != 101) {
761 : if (!response) {
762 : LOG_ERROR("WebSocket connect error: {}", connection->last_error().message());
763 : }
764 : LOG_ERROR("WebSocket upgrade failed: {}",
765 : response ? std::to_string(response->get_status_code()) : "no response");
766 : co_return std::nullopt;
767 : }
768 : if (!valid_upgrade(*response, key, subprotocol)) co_return std::nullopt;
769 :
770 : // Used, and running its keepalive and closing handshake, where the caller runs: on its
771 : // strand, never at the same time as the caller on an io_context run by several threads
772 : auto ws = std::make_shared<asio::websocket>(connection->release_socket(), false, false,
773 : co_await boost::asio::this_coro::executor);
774 : ws->set_protocol(response->get_header("Sec-WebSocket-Protocol"));
775 : // Frames the server sent right after its response, read along with it
776 : ws->set_read_ahead(connection->take_buffered());
777 : // Every network wait has a deadline: the keepalive of the server, 60 s and 120 s
778 : ws->start_timeout();
779 :
780 : LOG_INFO("WebSocket connected to {}",
781 : util::url::loggable_uri(url->scheme + "://" + url->host + ":" + url->port + url->path, body_logging_.redact));
782 : co_return websocket_client(std::move(ws), owns_io_context());
783 626 : }
784 :
785 : } // namespace thinger::http
|
