Line data Source code
1 : #ifndef THINGER_HTTP_CLIENT_HTTP_CLIENT_BASE_HPP
2 : #define THINGER_HTTP_CLIENT_HTTP_CLIENT_BASE_HPP
3 :
4 : #include "../common/http_request.hpp"
5 : #include "../common/http_response.hpp"
6 : #include "../common/body_logging.hpp"
7 : #include "client_connection.hpp"
8 : #include "client_response.hpp"
9 : #include "connection_pool.hpp"
10 : #include "proxy.hpp"
11 : #include "websocket_client.hpp"
12 : #include "stream_types.hpp"
13 : #include "../../util/types.hpp"
14 : #include "form.hpp"
15 : #include <memory>
16 : #include <string>
17 : #include <chrono>
18 : #include <map>
19 : #include <mutex>
20 : #include <optional>
21 : #include <utility>
22 : #include <vector>
23 : #include <boost/asio/ssl/context.hpp>
24 :
25 : namespace thinger::http {
26 :
27 : using headers_map = std::map<std::string, std::string>;
28 :
29 : class http_client_base {
30 : protected:
31 2728 : std::chrono::seconds timeout_{30};
32 : unsigned int max_redirects_{5};
33 : bool follow_redirects_{true};
34 : std::string user_agent_{"ThingerHTTP/2.0"};
35 : bool auto_decompress_{true};
36 : bool verify_ssl_{true};
37 : std::string unix_socket_;
38 : body_logging body_logging_;
39 : size_t max_content_size_{8 * 1048576}; // 8 MB default; buffered-mode responses above this are rejected
40 :
41 : // Proxy set with proxy() (null if none), and the one of the environment (null if not
42 : // used): the first one set applies
43 : std::shared_ptr<const proxy_resolver> proxy_;
44 : std::shared_ptr<const proxy_resolver> env_proxy_;
45 :
46 : // CAs trusted besides (or instead of) the system ones
47 : std::vector<std::string> ca_files_;
48 : std::vector<std::string> ca_pems_;
49 : bool system_ca_{true};
50 :
51 : // Connections kept alive between requests
52 : connection_pool pool_;
53 :
54 : // Each derived class provides its own io_context
55 : virtual boost::asio::io_context& get_io_context() = 0;
56 :
57 : // Whether the io_context is the client's own, run only within its synchronous calls: the
58 : // synchronous calls of its WebSockets then wait running it (see websocket_client)
59 119 : virtual bool owns_io_context() const { return false; }
60 :
61 : void apply_default_headers(const std::shared_ptr<http_request>& request);
62 :
63 : // A connection for a request: an idle one of the pool if `reuse` and there is one, or a
64 : // new one (empty, and why in `ec`, if its proxy or CAs cannot be used)
65 : connection_pool::lease get_connection(const std::shared_ptr<http_request>& request,
66 : boost::system::error_code& ec, bool reuse = true);
67 :
68 : public:
69 : http_client_base();
70 : virtual ~http_client_base();
71 :
72 1455 : http_client_base& timeout(std::chrono::seconds t) { timeout_ = t; return *this; }
73 47 : http_client_base& max_redirects(unsigned int max) { max_redirects_ = max; return *this; }
74 84 : http_client_base& follow_redirects(bool follow) { follow_redirects_ = follow; return *this; }
75 22 : http_client_base& user_agent(const std::string& agent) { user_agent_ = agent; return *this; }
76 : // Ask for gzip and deflate bodies, and decode them (the default; also a list of codings,
77 : // see util::decode_content); otherwise bodies are kept as received, with their
78 : // Content-Encoding
79 27 : http_client_base& auto_decompress(bool decompress) { auto_decompress_ = decompress; return *this; }
80 110 : http_client_base& verify_ssl(bool verify) { verify_ssl_ = verify; return *this; }
81 :
82 : // Log the header values and the start of the bodies (up to `max_length` bytes) of the
83 : // requests and responses: an opt-in for debugging, off by default (see body_logging)
84 15 : http_client_base& log_bodies(bool enabled = true, size_t max_length = body_logging::default_max_length) {
85 15 : body_logging_.enabled = enabled;
86 15 : body_logging_.max_length = max_length;
87 15 : return *this;
88 : }
89 : // The query parameters hidden from the URIs this client logs, instead of the defaults:
90 : // start from util::url::redacted_parameters::defaults() to add or remove a name. Applies
91 : // from the next request, on a pooled connection too
92 42 : http_client_base& redact_parameters(util::url::redacted_parameters names) {
93 42 : body_logging_.redact = std::move(names);
94 42 : return *this;
95 : }
96 273 : http_client_base& unix_socket(const std::string& path) { unix_socket_ = path; return *this; }
97 90 : http_client_base& max_content_size(size_t size) { max_content_size_ = size; return *this; }
98 :
99 : // Proxy for every request (see proxy_config), instead of the one of the environment: one
100 : // URL for every scheme ("" to connect directly), and the hosts reached directly
101 : http_client_base& proxy(const std::string& url, const std::string& no_proxy = "");
102 : http_client_base& proxy(const proxy_config& config);
103 : // Use the proxy of the environment variables (read now) when none is set with proxy();
104 : // on by default (read when the client is created)
105 : http_client_base& proxy_from_environment(bool enabled);
106 :
107 : // CAs trusted to verify servers, besides the system ones: a PEM file (one or more
108 : // certificates), or PEM text. Loaded when first needed, again if the file changes
109 9 : http_client_base& add_ca_file(const std::string& path) { ca_files_.push_back(path); return *this; }
110 294 : http_client_base& add_ca_pem(const std::string& pem) { ca_pems_.push_back(pem); return *this; }
111 : // Trust the system CAs (the default); false trusts only those added
112 3 : http_client_base& use_system_ca(bool enabled) { system_ca_ = enabled; return *this; }
113 :
114 : // Certificate the client presents to the servers that ask for one (TLS client
115 : // authentication), on every TLS connection: requests, WebSockets and proxy tunnels. The
116 : // certificate (PEM), its private key (PEM, not encrypted) and, optionally, the
117 : // intermediate certificates sent along with it (PEM, one or more). Checked now: an error
118 : // if one cannot be read (client_error::invalid_client_certificate or invalid_client_key,
119 : // an encrypted key included) or the key is not the certificate's (client_key_mismatch),
120 : // and the current one is kept; logged, never the key. Files are read now: set it again to
121 : // read them again. It may be replaced while requests run: connections already open (and
122 : // pooled) keep the one they were opened with, new ones take the new one
123 : boost::system::error_code client_certificate(const std::string& certificate, const std::string& private_key,
124 : const std::string& chain = "");
125 : boost::system::error_code client_certificate_file(const std::string& certificate_path,
126 : const std::string& private_key_path,
127 : const std::string& chain_path = "");
128 : // Present no certificate (the default)
129 : void clear_client_certificate();
130 : bool has_client_certificate() const;
131 :
132 : // Keep-alive pool (see connection_pool): idle connections kept per origin (0 disables
133 : // it: a new connection for each request), in total, and for how long
134 : http_client_base& max_idle_connections_per_host(size_t max);
135 : http_client_base& max_idle_connections(size_t max);
136 : http_client_base& idle_timeout(std::chrono::milliseconds timeout);
137 :
138 18 : std::chrono::seconds get_timeout() const { return timeout_; }
139 30 : unsigned int get_max_redirects() const { return max_redirects_; }
140 36 : bool get_follow_redirects() const { return follow_redirects_; }
141 18 : const std::string& get_user_agent() const { return user_agent_; }
142 18 : bool get_auto_decompress() const { return auto_decompress_; }
143 22 : bool get_verify_ssl() const { return verify_ssl_; }
144 12 : const body_logging& get_body_logging() const { return body_logging_; }
145 : size_t get_max_content_size() const { return max_content_size_; }
146 : size_t get_max_idle_connections_per_host() const { return pool_.get_config().max_idle_per_host; }
147 : size_t get_max_idle_connections() const { return pool_.get_config().max_idle; }
148 : std::chrono::milliseconds get_idle_timeout() const { return pool_.get_config().idle_timeout; }
149 :
150 : // The proxy a request to `url` goes through ("http://host:port"), empty if it connects
151 : // directly (or if the proxy cannot be used)
152 : std::string proxy_for(const std::string& url) const;
153 : std::string proxy_for(const http_request& request) const;
154 :
155 : std::shared_ptr<http_request> create_request(method m, const std::string& url);
156 :
157 : awaitable<client_response> get(const std::string& url, headers_map headers = {});
158 : awaitable<client_response> post(const std::string& url, std::string body = {},
159 : std::string content_type = "application/json", headers_map headers = {});
160 : awaitable<client_response> post(const std::string& url, const form& form, headers_map headers = {});
161 : awaitable<client_response> put(const std::string& url, std::string body = {},
162 : std::string content_type = "application/json", headers_map headers = {});
163 : awaitable<client_response> patch(const std::string& url, std::string body = {},
164 : std::string content_type = "application/json", headers_map headers = {});
165 : awaitable<client_response> del(const std::string& url, headers_map headers = {});
166 : awaitable<client_response> head(const std::string& url, headers_map headers = {});
167 : awaitable<client_response> options(const std::string& url, headers_map headers = {});
168 :
169 : awaitable<client_response> send(std::shared_ptr<http_request> request);
170 :
171 : // Hand the response body to the callback as it arrives, without holding it in memory
172 : awaitable<stream_result> send_streaming(std::shared_ptr<http_request> request,
173 : stream_callback callback);
174 :
175 : // Send and get the connection of the (last) response back: for upgrades like WebSocket,
176 : // as one that switched protocols is never reused. Any other may be reused by the client
177 : awaitable<std::pair<client_response, std::shared_ptr<client_connection>>>
178 : send_with_connection(std::shared_ptr<http_request> request);
179 :
180 : // The WebSocket runs on the executor of the coroutine opening it (e.g. its strand)
181 : awaitable<std::optional<websocket_client>> upgrade_websocket(
182 : const std::string& url,
183 : const std::string& subprotocol = "");
184 :
185 : // With a request of its own: custom headers, proxy, CAs
186 : awaitable<std::optional<websocket_client>> upgrade_websocket(
187 : std::shared_ptr<http_request> request,
188 : const std::string& subprotocol = "");
189 :
190 : // Connection pool management: close the idle connections, count them, and the
191 : // connections by origin, idle and in use
192 2252 : void clear_connections() { pool_.clear(); }
193 89 : size_t pool_size() const { return pool_.size(); }
194 87 : connection_pool::stats pool_stats() const { return pool_.get_stats(); }
195 :
196 : private:
197 : // How a connection reaches the origin of a request: through a proxy or not, with which
198 : // TLS context, and the key of connections that can be shared
199 : struct connection_plan {
200 : proxy_route route;
201 : std::shared_ptr<boost::asio::ssl::context> tls;
202 : std::string key;
203 : };
204 :
205 : // TLS contexts by what they trust, shared by the connections (and their id)
206 : struct tls_entry {
207 : uint64_t id;
208 : std::shared_ptr<boost::asio::ssl::context> context;
209 : };
210 : static constexpr size_t MAX_TLS_CONTEXTS = 32;
211 : mutable std::mutex tls_mutex_;
212 : std::map<std::string, tls_entry> tls_contexts_;
213 : uint64_t next_tls_id_{0};
214 :
215 : // The client certificate (see client_certificate()): its chain (PEM, the certificate
216 : // first) and key, and what tells it apart in the TLS contexts (0: none). Guarded by
217 : // tls_mutex_
218 : struct client_identity {
219 : std::string chain;
220 : std::string key;
221 : uint64_t id = 0;
222 : };
223 : client_identity identity_;
224 : uint64_t next_identity_id_{0};
225 :
226 : // The proxy of a request: none, with an error if it is set but cannot be used
227 : proxy_resolver::choice select_proxy(const http_request& request) const;
228 :
229 : // The TLS context of a request, from the CAs of the client and the request
230 : std::shared_ptr<boost::asio::ssl::context> tls_context(const http_request& request, uint64_t& id,
231 : boost::system::error_code& ec);
232 :
233 : connection_plan plan_connection(const http_request& request, boost::system::error_code& ec);
234 : std::shared_ptr<client_connection> make_connection(const connection_plan& plan);
235 :
236 : // Send following redirects; the connection of the last response in `used`, if given
237 : awaitable<client_response> send_with_redirects(std::shared_ptr<http_request> request,
238 : std::shared_ptr<client_connection>* used = nullptr);
239 : bool should_follow(const http_response& response, unsigned int redirect_count) const;
240 : // The request following a redirect to `location`, taking the cookies `response` sets;
241 : // null if `location` does not give a URL the client can parse
242 : std::shared_ptr<http_request> redirect_request(http_request& request, const http_response& response,
243 : const std::string& location);
244 :
245 : // Send on the connection of `lease`, and again on a new one (then in `lease`) if a reused
246 : // one failed and the request can be sent twice (see can_retry)
247 : awaitable<std::shared_ptr<http_response>> send_request(const std::shared_ptr<http_request>& request,
248 : connection_pool::lease& lease);
249 : static bool can_retry(const client_connection& connection, const http_request& request);
250 :
251 : // Credentials are forwarded on redirects only to the same origin
252 : static bool is_same_origin(const std::string& url1, const std::string& url2);
253 : };
254 :
255 : } // namespace thinger::http
256 :
257 : #endif
|
