Line data Source code
1 : #ifndef THINGER_UTIL_COMPRESSION_HPP
2 : #define THINGER_UTIL_COMPRESSION_HPP
3 :
4 : #include <algorithm>
5 : #include <cstddef>
6 : #include <limits>
7 : #include <optional>
8 : #include <string>
9 : #include <string_view>
10 : #include <zlib.h>
11 : #include "ascii.hpp"
12 :
13 : namespace thinger::util {
14 :
15 : namespace detail {
16 :
17 : enum class inflate_format { zlib, gzip, raw };
18 :
19 : // Inflate `data` up to `max_size` bytes, checked as it is inflated (a decompression bomb
20 : // takes no more memory than the limit). gzip decodes every member of the data, one after
21 : // another, and accepts zero bytes after the last one (padding, as Node and gzip(1) do);
22 : // anything else after the end of the data fails: a zlib or raw deflate stream must take
23 : // the whole data, and a gzip member be followed by another one (as Go does).
24 580 : inline std::optional<std::string> inflate_data(std::string_view data, inflate_format format,
25 : size_t max_size, bool* too_large) {
26 580 : if (too_large) *too_large = false;
27 :
28 580 : int window_bits = format == inflate_format::gzip ? 15 + 16 : format == inflate_format::raw ? -15 : 15;
29 580 : z_stream strm{};
30 580 : if (inflateInit2(&strm, window_bits) != Z_OK) {
31 0 : return std::nullopt;
32 : }
33 : struct end_guard {
34 : z_stream& strm;
35 580 : ~end_guard() { inflateEnd(&strm); }
36 580 : } guard{strm};
37 :
38 : // The input is given in pieces fitting in avail_in
39 580 : size_t consumed = 0;
40 49933 : auto feed = [&] {
41 49933 : if (strm.avail_in != 0 || consumed == data.size()) return;
42 571 : auto size = std::min<size_t>(data.size() - consumed, std::numeric_limits<uInt>::max());
43 571 : strm.next_in = reinterpret_cast<Bytef*>(const_cast<char*>(data.data() + consumed));
44 571 : strm.avail_in = static_cast<uInt>(size);
45 571 : consumed += size;
46 580 : };
47 :
48 580 : std::string result;
49 : char buffer[16384];
50 : while (true) {
51 49933 : feed();
52 49933 : strm.next_out = reinterpret_cast<Bytef*>(buffer);
53 49933 : strm.avail_out = sizeof(buffer);
54 :
55 : // Anything but progress is an error; Z_BUF_ERROR means the input ended before
56 : // the compressed stream did (it would never reach Z_STREAM_END)
57 49933 : int ret = inflate(&strm, Z_NO_FLUSH);
58 49933 : if (ret != Z_OK && ret != Z_STREAM_END) {
59 138 : return std::nullopt;
60 : }
61 :
62 49795 : size_t produced = sizeof(buffer) - strm.avail_out;
63 49795 : if (produced > max_size - result.size()) {
64 81 : if (too_large) *too_large = true;
65 81 : return std::nullopt;
66 : }
67 49714 : result.append(buffer, produced);
68 49714 : if (ret != Z_STREAM_END) continue;
69 :
70 : // End of the stream (or of a gzip member): what follows it, if anything
71 460 : auto rest = data.substr(consumed - strm.avail_in);
72 460 : if (rest.empty()) return result;
73 123 : if (format != inflate_format::gzip) return std::nullopt;
74 102 : if (rest.find_first_not_of('\0') == std::string_view::npos) return result;
75 99 : if (inflateReset(&strm) != Z_OK) return std::nullopt;
76 49353 : }
77 580 : }
78 :
79 : // Whether `data` starts with a zlib header (RFC 1950, 2.2): deflate method, a window of
80 : // at most 32 KB, and a check of both bytes
81 189 : inline bool zlib_header(std::string_view data) {
82 189 : if (data.size() < 2) return false;
83 189 : auto cmf = static_cast<unsigned char>(data[0]);
84 189 : auto flg = static_cast<unsigned char>(data[1]);
85 189 : return (cmf & 0x0f) == 8 && (cmf >> 4) <= 7 && ((cmf << 8) | flg) % 31 == 0;
86 : }
87 :
88 : }
89 :
90 : class gzip {
91 : public:
92 : // Compress string to gzip format
93 436 : static std::optional<std::string> compress(const std::string& data) {
94 436 : z_stream strm{};
95 : // windowBits = 15 + 16 enables gzip encoding
96 436 : if (deflateInit2(&strm, Z_DEFAULT_COMPRESSION, Z_DEFLATED, 15 + 16, 8, Z_DEFAULT_STRATEGY) != Z_OK) {
97 0 : return std::nullopt;
98 : }
99 :
100 436 : strm.next_in = reinterpret_cast<Bytef*>(const_cast<char*>(data.data()));
101 436 : strm.avail_in = static_cast<uInt>(data.size());
102 :
103 436 : std::string result;
104 436 : result.resize(deflateBound(&strm, data.size()));
105 :
106 436 : strm.next_out = reinterpret_cast<Bytef*>(result.data());
107 436 : strm.avail_out = static_cast<uInt>(result.size());
108 :
109 436 : int ret = deflate(&strm, Z_FINISH);
110 436 : deflateEnd(&strm);
111 :
112 436 : if (ret != Z_STREAM_END) {
113 0 : return std::nullopt;
114 : }
115 :
116 436 : result.resize(strm.total_out);
117 436 : return result;
118 436 : }
119 :
120 : // Decompress gzip data: all its members, concatenated (RFC 1952, 2.2), and zero bytes
121 : // after them (padding). Fails on invalid or truncated data, other bytes after a member,
122 : // or if the result would exceed `max_size` bytes (then `too_large` is set, if given).
123 385 : static std::optional<std::string> decompress(std::string_view data,
124 : size_t max_size = std::numeric_limits<size_t>::max(),
125 : bool* too_large = nullptr) {
126 385 : return detail::inflate_data(data, detail::inflate_format::gzip, max_size, too_large);
127 : }
128 :
129 : // Check if data might be gzip compressed (by checking magic bytes)
130 : static bool is_gzip(const std::string& data) {
131 : return data.size() >= 2 &&
132 : static_cast<unsigned char>(data[0]) == 0x1f &&
133 : static_cast<unsigned char>(data[1]) == 0x8b;
134 : }
135 : };
136 :
137 : class deflate {
138 : public:
139 : // Compress string using deflate (zlib format)
140 173 : static std::optional<std::string> compress(const std::string& data) {
141 173 : z_stream strm{};
142 : // windowBits = 15 for zlib format
143 173 : if (deflateInit(&strm, Z_DEFAULT_COMPRESSION) != Z_OK) {
144 0 : return std::nullopt;
145 : }
146 :
147 173 : strm.next_in = reinterpret_cast<Bytef*>(const_cast<char*>(data.data()));
148 173 : strm.avail_in = static_cast<uInt>(data.size());
149 :
150 173 : std::string result;
151 173 : result.resize(deflateBound(&strm, data.size()));
152 :
153 173 : strm.next_out = reinterpret_cast<Bytef*>(result.data());
154 173 : strm.avail_out = static_cast<uInt>(result.size());
155 :
156 173 : int ret = ::deflate(&strm, Z_FINISH);
157 173 : deflateEnd(&strm);
158 :
159 173 : if (ret != Z_STREAM_END) {
160 0 : return std::nullopt;
161 : }
162 :
163 173 : result.resize(strm.total_out);
164 173 : return result;
165 173 : }
166 :
167 : // Decompress deflate data: the zlib format, or raw deflate if it does not start with a
168 : // zlib header (as some servers send it; curl and browsers accept it too). Fails on
169 : // invalid or truncated data, bytes after its end, or if the result would exceed
170 : // `max_size` bytes (then `too_large` is set, if given).
171 195 : static std::optional<std::string> decompress(std::string_view data,
172 : size_t max_size = std::numeric_limits<size_t>::max(),
173 : bool* too_large = nullptr) {
174 195 : auto format = data.size() < 2 || detail::zlib_header(data) ? detail::inflate_format::zlib
175 195 : : detail::inflate_format::raw;
176 195 : return detail::inflate_data(data, format, max_size, too_large);
177 : }
178 : };
179 :
180 : // Outcome of decode_content()
181 : enum class content_decoding {
182 : decoded, // the body is decoded (or only listed identity)
183 : unsupported, // a coding other than gzip, x-gzip, deflate or identity: the body is kept
184 : invalid, // not valid data of its codings (corrupt, truncated, trailing bytes), or too many codings
185 : too_large // larger than the maximum size once decoded
186 : };
187 :
188 : // Most codings a body may be encoded with (as curl and Node's fetch, against chains of
189 : // decompressions taking a decoded size each)
190 : inline constexpr size_t max_content_codings = 5;
191 :
192 : // Decode a body, in place, as its Content-Encoding (`codings`: its comma-separated values, of
193 : // every header) says (RFC 9110, 8.4): the codings were applied in the order listed, so they
194 : // are undone in reverse. Names are case-insensitive, and x-gzip is gzip (8.4.1.3). Each step
195 : // stops at `max_size` bytes decoded. A body listing a coding that is not supported is kept as
196 : // it is (as Node's fetch does), with nothing decoded.
197 415 : inline content_decoding decode_content(std::string_view codings, std::string& body, size_t max_size) {
198 : using ::thinger::util::ascii::iequals;
199 : enum class coding { gzip, deflate };
200 : coding listed[max_content_codings];
201 415 : size_t count = 0;
202 415 : bool unsupported = false;
203 415 : size_t elements = 0;
204 920 : while (!codings.empty()) {
205 514 : auto comma = codings.find(',');
206 514 : auto element = codings.substr(0, comma);
207 514 : codings = comma == std::string_view::npos ? std::string_view{} : codings.substr(comma + 1);
208 514 : auto first = element.find_first_not_of(" \t");
209 514 : if (first == std::string_view::npos) continue;
210 508 : element = element.substr(first, element.find_last_not_of(" \t") - first + 1);
211 :
212 508 : if (++elements > max_content_codings) return content_decoding::invalid;
213 499 : if (iequals(element, "gzip") || iequals(element, "x-gzip")) {
214 352 : listed[count++] = coding::gzip;
215 147 : } else if (iequals(element, "deflate")) {
216 114 : listed[count++] = coding::deflate;
217 33 : } else if (!iequals(element, "identity")) {
218 27 : unsupported = true;
219 : }
220 : }
221 406 : if (unsupported) return content_decoding::unsupported;
222 :
223 668 : while (count > 0) {
224 406 : bool too_large = false;
225 406 : auto decoded = listed[--count] == coding::gzip ? gzip::decompress(body, max_size, &too_large)
226 406 : : deflate::decompress(body, max_size, &too_large);
227 406 : if (!decoded) return too_large ? content_decoding::too_large : content_decoding::invalid;
228 289 : body = std::move(*decoded);
229 406 : }
230 262 : return content_decoding::decoded;
231 : }
232 :
233 : } // namespace thinger::util
234 :
235 : #endif // THINGER_UTIL_COMPRESSION_HPP
|
