Add compression buffer size customization (#644)

* add compression buffer size customization and small brotli refactor

* allocat brotli buffer once

* add init to brotli decoder buffer

httplib.h

@@ -80,6 +80,10 @@
 #define CPPHTTPLIB_RECV_BUFSIZ size_t(4096u)
 #endif
 
+#ifndef CPPHTTPLIB_COMPRESSION_BUFSIZ
+#define CPPHTTPLIB_COMPRESSION_BUFSIZ size_t(16384u)
+#endif
+
 #ifndef CPPHTTPLIB_THREAD_POOL_COUNT
 #define CPPHTTPLIB_THREAD_POOL_COUNT                                           \
   ((std::max)(8u, std::thread::hardware_concurrency() > 0                      \
@@ -1248,6 +1252,14 @@ inline std::string from_i_to_hex(size_t n) {
   return ret;
 }
 
+inline bool start_with(const std::string &a, const std::string &b) {
+  if (a.size() < b.size()) { return false; }
+  for (size_t i = 0; i < b.size(); i++) {
+    if (std::tolower(a[i]) != std::tolower(b[i])) { return false; }
+  }
+  return true;
+}
+
 inline size_t to_utf8(int code, char *buff) {
   if (code < 0x0080) {
     buff[0] = (code & 0x7F);
@@ -1441,34 +1453,32 @@ inline std::string file_extension(const std::string &path) {
   return std::string();
 }
 
-inline std::pair<int, int> trim(const char *b, const char *e, int left,
-                                int right) {
-  while (b + left < e && b[left] == ' ') {
+inline bool is_space_or_tab(char c) { return c == ' ' || c == '\t'; }
+
+inline std::pair<size_t, size_t> trim(const char *b, const char *e, size_t left,
+                                      size_t right) {
+  while (b + left < e && is_space_or_tab(b[left])) {
     left++;
   }
-  while (right - 1 >= 0 && b[right - 1] == ' ') {
+  while (right > 0 && is_space_or_tab(b[right - 1])) {
     right--;
   }
   return std::make_pair(left, right);
 }
 
-inline void trim(std::string &s) {
-  auto is_not_space = [](int ch) { return !std::isspace(ch); };
-  s.erase(s.begin(), std::find_if(s.begin(), s.end(), is_not_space));
-  s.erase(std::find_if(s.rbegin(), s.rend(), is_not_space).base(), s.end());
+inline std::string trim_copy(const std::string &s) {
+  auto r = trim(s.data(), s.data() + s.size(), 0, s.size());
+  return s.substr(r.first, r.second - r.first);
 }
 
-
 template <class Fn> void split(const char *b, const char *e, char d, Fn fn) {
-  int i = 0;
-  int beg = 0;
+  size_t i = 0;
+  size_t beg = 0;
 
   while (e ? (b + i < e) : (b[i] != '\0')) {
     if (b[i] == d) {
       auto r = trim(b, e, beg, i);
-      if (r.first < r.second) {
-        fn(&b[r.first], &b[r.second]);
-      }
+      if (r.first < r.second) { fn(&b[r.first], &b[r.second]); }
       beg = i + 1;
     }
     i++;
@@ -1476,9 +1486,7 @@ template <class Fn> void split(const char *b, const char *e, char d, Fn fn) {
 
   if (i) {
     auto r = trim(b, e, beg, i);
-    if (r.first < r.second) {
-      fn(&b[r.first], &b[r.second]);
-    }
+    if (r.first < r.second) { fn(&b[r.first], &b[r.second]); }
   }
 }
 
@@ -2222,7 +2230,7 @@ public:
 
     int ret = Z_OK;
 
-    std::array<char, 16384> buff{};
+    std::array<char, CPPHTTPLIB_COMPRESSION_BUFSIZ> buff{};
     do {
       strm_.avail_out = buff.size();
       strm_.next_out = reinterpret_cast<Bytef *>(buff.data());
@@ -2273,7 +2281,7 @@ public:
     strm_.avail_in = static_cast<decltype(strm_.avail_in)>(data_length);
     strm_.next_in = const_cast<Bytef *>(reinterpret_cast<const Bytef *>(data));
 
-    std::array<char, 16384> buff{};
+    std::array<char, CPPHTTPLIB_COMPRESSION_BUFSIZ> buff{};
     while (strm_.avail_in > 0) {
       strm_.avail_out = buff.size();
       strm_.next_out = reinterpret_cast<Bytef *>(buff.data());
@@ -2311,7 +2319,7 @@ public:
 
   bool compress(const char *data, size_t data_length, bool last,
                 Callback callback) override {
-    std::array<uint8_t, 16384> buff{};
+    std::array<uint8_t, CPPHTTPLIB_COMPRESSION_BUFSIZ> buff{};
 
     auto operation = last ? BROTLI_OPERATION_FINISH : BROTLI_OPERATION_PROCESS;
     auto available_in = data_length;
@@ -2373,18 +2381,18 @@ public:
 
     decoder_r = BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT;
 
+    std::array<char, CPPHTTPLIB_COMPRESSION_BUFSIZ> buff{};
     while (decoder_r == BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT) {
-      char output[1024];
-      char *next_out = output;
-      size_t avail_out = sizeof(output);
+      char *next_out = buff.data();
+      size_t avail_out = buff.size();
 
       decoder_r = BrotliDecoderDecompressStream(
           decoder_s, &avail_in, &next_in, &avail_out,
-          reinterpret_cast<unsigned char **>(&next_out), &total_out);
+          reinterpret_cast<uint8_t **>(&next_out), &total_out);
 
       if (decoder_r == BROTLI_DECODER_RESULT_ERROR) { return false; }
 
-      if (!callback((const char *)output, sizeof(output) - avail_out)) {
+      if (!callback(buff.data(), buff.size() - avail_out)) {
         return false;
       }
     }
@@ -2429,26 +2437,34 @@ inline uint64_t get_header_value<uint64_t>(const Headers &headers,
   return def;
 }
 
-inline void parse_header(const char *beg, const char *end, Headers &headers) {
+template <typename T>
+inline bool parse_header(const char *beg, const char *end, T fn) {
+  // Skip trailing spaces and tabs.
+  while (beg < end && is_space_or_tab(end[-1])) {
+    end--;
+  }
+
   auto p = beg;
   while (p < end && *p != ':') {
     p++;
   }
-  if (p < end) {
-    auto key_end = p;
-    p++; // skip ':'
-    while (p < end && (*p == ' ' || *p == '\t')) {
-      p++;
-    }
-    if (p < end) {
-      auto val_begin = p;
-      while (p < end) {
-        p++;
-      }
-      headers.emplace(std::string(beg, key_end),
-                      decode_url(std::string(val_begin, end), true));
-    }
+
+  if (p == end) { return false; }
+
+  auto key_end = p;
+
+  if (*p++ != ':') { return false; }
+
+  while (p < end && is_space_or_tab(*p)) {
+    p++;
   }
+
+  if (p < end) {
+    fn(std::string(beg, key_end), decode_url(std::string(p, end), true));
+    return true;
+  }
+
+  return false;
 }
 
 inline bool read_headers(Stream &strm, Headers &headers) {
@@ -2467,13 +2483,13 @@ inline bool read_headers(Stream &strm, Headers &headers) {
       continue; // Skip invalid line.
     }
 
-    // Skip trailing spaces and tabs.
+    // Exclude CRLF
     auto end = line_reader.ptr() + line_reader.size() - 2;
-    while (line_reader.ptr() < end && (end[-1] == ' ' || end[-1] == '\t')) {
-      end--;
-    }
 
-    parse_header(line_reader.ptr(), end, headers);
+    parse_header(line_reader.ptr(), end,
+                 [&](std::string &&key, std::string &&val) {
+                   headers.emplace(std::move(key), std::move(val));
+                 });
   }
 
   return true;
@@ -2835,18 +2851,6 @@ inline bool redirect(T &cli, const Request &req, Response &res,
   return ret;
 }
 
-inline bool contains_header(const std::string &header, const std::string &name) {
-  if (header.length() >= name.length()) {
-    for (int i = 0; i < name.length(); ++i) {
-      if (std::tolower(header[i]) != std::tolower(name[i])) {
-        return false;
-      }
-    }
-    return true;
-  }
-  return false;
-}
-
 inline std::string params_to_query_str(const Params &params) {
   std::string query;
 
@@ -2871,7 +2875,7 @@ inline void parse_query_text(const std::string &s, Params &params) {
       }
     });
 
-    if(!key.empty()) {
+    if (!key.empty()) {
       params.emplace(decode_url(key, true), decode_url(val, true));
     }
   });
@@ -2975,15 +2979,13 @@ public:
             break;
           }
 
-          auto header = buf_.substr(0, pos);
-          {
+          static const std::string header_name = "content-type:";
+          const auto header = buf_.substr(0, pos);
+          if (start_with(header, header_name)) {
+            file_.content_type = trim_copy(header.substr(header_name.size()));
+          } else {
             std::smatch m;
-            const std::string header_name = "content-type:";
-            if (contains_header(header, header_name)) {
-              header.erase(header.begin(), header.begin() + header_name.size());
-              trim(header);
-              file_.content_type = header;
-            } else if (std::regex_match(header, m, re_content_disposition)) {
+            if (std::regex_match(header, m, re_content_disposition)) {
               file_.name = m[1];
               file_.filename = m[2];
             }

test/test.cc

@@ -43,22 +43,11 @@ TEST(StartupTest, WSAStartup) {
   ASSERT_EQ(0, ret);
 }
 #endif
+
 TEST(TrimTests, TrimStringTests) {
-  {
-    std::string s = "abc";
-    detail::trim(s);
-    EXPECT_EQ("abc", s);
-  }
-  {
-    std::string s = "  abc  ";
-    detail::trim(s);
-    EXPECT_EQ("abc", s);
-  }
-  {
-    std::string s = "";
-    detail::trim(s);
-    EXPECT_TRUE( s.empty() );
-  }
+  EXPECT_EQ("abc", detail::trim_copy("abc"));
+  EXPECT_EQ("abc", detail::trim_copy("  abc  "));
+  EXPECT_TRUE(detail::trim_copy("").empty());
 }
 
 TEST(SplitTest, ParseQueryString) {
@@ -100,7 +89,6 @@ TEST(SplitTest, ParseInvalidQueryTests) {
   }
 }
 
-
 TEST(ParseQueryTest, ParseQueryString) {
   string s = "key1=val1&key2=val2&key3=val3";
   Params dic;
@@ -1827,8 +1815,7 @@ TEST_F(ServerTest, MultipartFormData) {
       {"file1", "h\ne\n\nl\nl\no\n", "hello.txt", "text/plain"},
       {"file2", "{\n  \"world\", true\n}\n", "world.json", "application/json"},
       {"file3", "", "", "application/octet-stream"},
-      {"file4", "", "", "   application/json  tmp-string    "}
-  };
+      {"file4", "", "", "   application/json  tmp-string    "}};
 
   auto res = cli_.Post("/multipart", items);
 
@@ -2201,7 +2188,6 @@ TEST_F(ServerTest, GetStreamedChunkedWithGzip2) {
   EXPECT_EQ(std::string("123456789"), res->body);
 }
 
-
 TEST(GzipDecompressor, ChunkedDecompression) {
   std::string data;
   for (size_t i = 0; i < 32 * 1024; ++i) {
@@ -2212,10 +2198,8 @@ TEST(GzipDecompressor, ChunkedDecompression) {
   {
     httplib::detail::gzip_compressor compressor;
     bool result = compressor.compress(
-        data.data(),
-        data.size(),
-        /*last=*/true,
-        [&] (const char* data, size_t size) {
+        data.data(), data.size(),
+        /*last=*/true, [&](const char *data, size_t size) {
           compressed_data.insert(compressed_data.size(), data, size);
           return true;
         });
@@ -2226,15 +2210,16 @@ TEST(GzipDecompressor, ChunkedDecompression) {
   {
     httplib::detail::gzip_decompressor decompressor;
 
-    // Chunk size is chosen specificaly to have a decompressed chunk size equal to 16384 bytes
-    // 16384 bytes is the size of decompressor output buffer
+    // Chunk size is chosen specificaly to have a decompressed chunk size equal
+    // to 16384 bytes 16384 bytes is the size of decompressor output buffer
     size_t chunk_size = 130;
-    for (size_t chunk_begin = 0; chunk_begin < compressed_data.size(); chunk_begin += chunk_size) {
-      size_t current_chunk_size = std::min(compressed_data.size() - chunk_begin, chunk_size);
+    for (size_t chunk_begin = 0; chunk_begin < compressed_data.size();
+         chunk_begin += chunk_size) {
+      size_t current_chunk_size =
+          std::min(compressed_data.size() - chunk_begin, chunk_size);
       bool result = decompressor.decompress(
-          compressed_data.data() + chunk_begin,
-          current_chunk_size,
-          [&] (const char* data, size_t size) {
+          compressed_data.data() + chunk_begin, current_chunk_size,
+          [&](const char *data, size_t size) {
             decompressed_data.insert(decompressed_data.size(), data, size);
             return true;
           });
@@ -2819,15 +2804,14 @@ TEST(StreamingTest, NoContentLengthStreaming) {
   Server svr;
 
   svr.Get("/stream", [](const Request & /*req*/, Response &res) {
-    res.set_content_provider(
-        "text/plain", [](size_t offset, DataSink &sink) {
-          if (offset < 6) {
-            sink.os << (offset < 3 ? "a" : "b");
-          } else {
-            sink.done();
-          }
-          return true;
-        });
+    res.set_content_provider("text/plain", [](size_t offset, DataSink &sink) {
+      if (offset < 6) {
+        sink.os << (offset < 3 ? "a" : "b");
+      } else {
+        sink.done();
+      }
+      return true;
+    });
   });
 
   auto listen_thread = std::thread([&svr]() { svr.listen("localhost", PORT); });