Compare commits

...

14 Commits

Author SHA1 Message Date
yhirose f6a2365ca5 Fix #282 2019-12-06 12:21:15 -05:00
yhirose df1ff7510b Made code more readable 2019-12-06 12:02:08 -05:00
yhirose 379905bd34 Merge branch 'whitespace-and-libcxx-compat' of https://github.com/matvore/cpp-httplib 2019-12-06 09:51:21 -05:00
yhirose 66719ae3d4 Merge pull request #283 from barryam3/noexcept
Remove use of exceptions.
2019-12-05 21:32:06 -05:00
Matthew DeVore bc9251ea49 Work around incompatibility in <regex> in libc++
libc++ (the implementation of the C++ standard library usually used by
Clang) throws an exception for the regex used by parse_headers before
this patch for certain strings. Work around this by simplifying the
regex and parsing the header lines "by hand" partially. I have repro'd
this problem with Xcode 11.1 which I believe uses libc++ version 8.

This may be a bug in libc++ as I can't see why the regex would result in
asymptotic run-time complexity for any strings. However, it may take a
while for libc++ to be fixed and for everyone to migrate to it, so it
makes sense to work around it in this codebase for now.
2019-12-05 17:14:16 -08:00
Matthew DeVore a9e942d755 Properly trim whitespace from headers
HTTP Whitespace and regex whitespace are not the same, so we can't use
\s in regexes when parsing HTTP headers. Instead, explicitly specify
what is considered whitespace in the regex.
2019-12-05 17:14:16 -08:00
Barry McNamara e1785d6723 Remove use of exceptions. 2019-12-05 15:56:55 -08:00
yhirose b9539b8921 Fixed build errors 2019-12-03 10:30:07 -05:00
yhirose 4c93b973ff Fixed typo in README 2019-12-02 09:50:52 -05:00
yhirose 033bc35723 Improve multipart content reader interface 2019-12-02 07:11:12 -05:00
yhirose d910bfc303 Merge pull request #279 from yhirose/multipart
Content receiver support for multipart content (Fix #241)
2019-12-01 22:12:29 -05:00
yhirose b69c0a1dcb Content receiver support for multipart content (Fix #241) 2019-12-01 22:04:26 -05:00
yhirose 5e37e38398 Updated README 2019-11-29 23:33:19 -05:00
yhirose 295e4d58aa Fix #276 2019-11-29 17:07:51 -05:00
9 changed files with 718 additions and 257 deletions
+41 -4
View File
@@ -50,7 +50,16 @@ svr.listen_after_bind();
### Static File Server
```cpp
svr.set_base_dir("./www");
svr.set_base_dir("./www"); // This is same as `svr.set_base_dir("./www", "/")`;
```
```cpp
svr.set_base_dir("./www", "/public");
```
```cpp
svr.set_base_dir("./www1", "/public"); // 1st order
svr.set_base_dir("./www2", "/public"); // 2nd order
```
### Logging
@@ -81,12 +90,12 @@ svr.Post("/multipart", [&](const auto& req, auto& res) {
const auto& file = req.get_file_value("name1");
// file.filename;
// file.content_type;
auto body = req.body.substr(file.offset, file.length);
// file.content;
});
```
### Stream content with Content provider
### Send content with Content provider
```cpp
const uint64_t DATA_CHUNK_SIZE = 4;
@@ -104,6 +113,34 @@ svr.Get("/stream", [&](const Request &req, Response &res) {
});
```
### Receive content with Content receiver
```cpp
svr.Post("/content_receiver",
[&](const Request &req, Response &res, const ContentReader &content_reader) {
if (req.is_multipart_form_data()) {
MultipartFiles files;
content_reader(
[&](const std::string &name, const MultipartFile &file) {
files.emplace(name, file);
return true;
},
[&](const std::string &name, const char *data, size_t data_length) {
auto &file = files.find(name)->second;
file.content.append(data, data_length);
return true;
});
} else {
std::string body;
content_reader([&](const char *data, size_t data_length) {
body.append(data, data_length);
return true;
});
res.set_content(body, "text/plain");
}
});
```
### Chunked transfer encoding
```cpp
@@ -119,7 +156,7 @@ svr.Get("/chunked", [&](const Request& req, Response& res) {
});
```
### Default thread pool supporet
### Default thread pool support
Set thread count to 8:
+1 -4
View File
@@ -46,10 +46,7 @@ string dump_multipart_files(const MultipartFiles &files) {
snprintf(buf, sizeof(buf), "content type: %s\n", file.content_type.c_str());
s += buf;
snprintf(buf, sizeof(buf), "text offset: %lu\n", file.offset);
s += buf;
snprintf(buf, sizeof(buf), "text length: %lu\n", file.length);
snprintf(buf, sizeof(buf), "text length: %lu\n", file.content.size());
s += buf;
s += "----------------\n";
+2 -2
View File
@@ -37,10 +37,10 @@ int main(void) {
svr.Post("/post", [](const Request & req, Response &res) {
auto file = req.get_file_value("file");
cout << "file: " << file.offset << ":" << file.length << ":" << file.filename << endl;
cout << "file length: " << file.content.length() << ":" << file.filename << endl;
ofstream ofs(file.filename, ios::binary);
ofs << req.body.substr(file.offset, file.length);
ofs << file.content;
res.set_content("done", "text/plain");
});
+458 -225
View File
@@ -119,8 +119,8 @@ using socket_t = SOCKET;
#ifdef CPPHTTPLIB_USE_POLL
#include <poll.h>
#endif
#include <pthread.h>
#include <csignal>
#include <pthread.h>
#include <sys/select.h>
#include <sys/socket.h>
#include <unistd.h>
@@ -129,8 +129,9 @@ using socket_t = int;
#define INVALID_SOCKET (-1)
#endif //_WIN32
#include <cassert>
#include <array>
#include <atomic>
#include <cassert>
#include <condition_variable>
#include <errno.h>
#include <fcntl.h>
@@ -145,7 +146,6 @@ using socket_t = int;
#include <string>
#include <sys/stat.h>
#include <thread>
#include <array>
#ifdef CPPHTTPLIB_OPENSSL_SUPPORT
#include <openssl/err.h>
@@ -196,13 +196,11 @@ using DataSink = std::function<void(const char *data, size_t data_len)>;
using Done = std::function<void()>;
using ContentProvider = std::function<void(size_t offset, size_t length, DataSink sink)>;
using ContentProvider =
std::function<void(size_t offset, size_t length, DataSink sink)>;
using ContentProviderWithCloser = std::function<void(size_t offset, size_t length, DataSink sink, Done done)>;
using ContentReceiver = std::function<bool(const char *data, size_t data_length)>;
using ContentReader = std::function<bool(ContentReceiver receiver)>;
using ContentProviderWithCloser =
std::function<void(size_t offset, size_t length, DataSink sink, Done done)>;
using Progress = std::function<bool(uint64_t current, uint64_t total)>;
@@ -212,8 +210,7 @@ using ResponseHandler = std::function<bool(const Response &response)>;
struct MultipartFile {
std::string filename;
std::string content_type;
size_t offset = 0;
size_t length = 0;
std::string content;
};
using MultipartFiles = std::multimap<std::string, MultipartFile>;
@@ -225,6 +222,35 @@ struct MultipartFormData {
};
using MultipartFormDataItems = std::vector<MultipartFormData>;
using ContentReceiver =
std::function<bool(const char *data, size_t data_length)>;
using MultipartContentHeader =
std::function<bool(const std::string &name, const MultipartFile &file)>;
using MultipartContentReceiver =
std::function<bool(const std::string& name, const char *data, size_t data_length)>;
class ContentReader {
public:
using Reader = std::function<bool(ContentReceiver receiver)>;
using MultipartReader = std::function<bool(MultipartContentHeader header, MultipartContentReceiver receiver)>;
ContentReader(Reader reader, MultipartReader muitlpart_reader)
: reader_(reader), muitlpart_reader_(muitlpart_reader) {}
bool operator()(MultipartContentHeader header, MultipartContentReceiver receiver) const {
return muitlpart_reader_(header, receiver);
}
bool operator()(ContentReceiver receiver) const {
return reader_(receiver);
}
Reader reader_;
MultipartReader muitlpart_reader_;
};
using Range = std::pair<ssize_t, ssize_t>;
using Ranges = std::vector<Range>;
@@ -262,6 +288,8 @@ struct Request {
std::string get_param_value(const char *key, size_t id = 0) const;
size_t get_param_value_count(const char *key) const;
bool is_multipart_form_data() const;
bool has_file(const char *key) const;
MultipartFile get_file_value(const char *key) const;
@@ -394,7 +422,7 @@ public:
cond_.notify_all();
// Join...
for (auto& t : threads_) {
for (auto &t : threads_) {
t.join();
}
}
@@ -475,20 +503,17 @@ public:
NoThread() {}
virtual ~NoThread() {}
virtual void enqueue(std::function<void()> fn) override {
fn();
}
virtual void enqueue(std::function<void()> fn) override { fn(); }
virtual void shutdown() override {
}
virtual void shutdown() override {}
};
#endif
class Server {
public:
using Handler = std::function<void(const Request &, Response &)>;
using HandlerWithContentReader = std::function<void(const Request &, Response &,
const ContentReader &content_reader)>;
using HandlerWithContentReader = std::function<void(
const Request &, Response &, const ContentReader &content_reader)>;
using Logger = std::function<void(const Request &, const Response &)>;
Server();
@@ -507,7 +532,7 @@ public:
Server &Delete(const char *pattern, Handler handler);
Server &Options(const char *pattern, Handler handler);
bool set_base_dir(const char *path);
bool set_base_dir(const char *dir, const char *mount_point = nullptr);
void set_file_request_handler(Handler handler);
void set_error_handler(Handler handler);
@@ -531,7 +556,7 @@ public:
protected:
bool process_request(Stream &strm, bool last_connection,
bool &connection_close,
const std::function<void(Request &)>& setup_request);
const std::function<void(Request &)> &setup_request);
size_t keep_alive_max_count_;
time_t read_timeout_sec_;
@@ -540,7 +565,8 @@ protected:
private:
using Handlers = std::vector<std::pair<std::regex, Handler>>;
using HandersForContentReader = std::vector<std::pair<std::regex, HandlerWithContentReader>>;
using HandersForContentReader =
std::vector<std::pair<std::regex, HandlerWithContentReader>>;
socket_t create_server_socket(const char *host, int port,
int socket_flags) const;
@@ -564,21 +590,28 @@ private:
Response &res);
bool read_content_with_content_receiver(Stream &strm, bool last_connection,
Request &req, Response &res,
ContentReceiver reveiver);
ContentReceiver receiver,
MultipartContentHeader multipart_header,
MultipartContentReceiver multipart_receiver);
bool read_content_core(Stream &strm, bool last_connection,
Request &req, Response &res,
ContentReceiver receiver,
MultipartContentHeader mulitpart_header,
MultipartContentReceiver multipart_receiver);
virtual bool process_and_close_socket(socket_t sock);
std::atomic<bool> is_running_;
std::atomic<socket_t> svr_sock_;
std::string base_dir_;
std::vector<std::pair<std::string, std::string>> base_dirs_;
Handler file_request_handler_;
Handlers get_handlers_;
Handlers post_handlers_;
HandersForContentReader post_handlers_for_content_reader;
HandersForContentReader post_handlers_for_content_reader_;
Handlers put_handlers_;
HandersForContentReader put_handlers_for_content_reader;
HandersForContentReader put_handlers_for_content_reader_;
Handlers patch_handlers_;
HandersForContentReader patch_handlers_for_content_reader;
HandersForContentReader patch_handlers_for_content_reader_;
Handlers delete_handlers_;
Handlers options_handlers_;
Handler error_handler_;
@@ -1081,6 +1114,11 @@ public:
}
}
bool end_with_crlf() const {
auto end = ptr() + size();
return size() >= 2 && end[-2] == '\r' && end[-1] == '\n';
}
bool getline() {
fixed_buffer_used_size_ = 0;
glowable_buffer_.clear();
@@ -1191,7 +1229,8 @@ inline bool wait_until_socket_is_ready(socket_t sock, time_t sec, time_t usec) {
(FD_ISSET(sock, &fdsr) || FD_ISSET(sock, &fdsw))) {
int error = 0;
socklen_t len = sizeof(error);
return getsockopt(sock, SOL_SOCKET, SO_ERROR, reinterpret_cast<char*>(&error), &len) >= 0 &&
return getsockopt(sock, SOL_SOCKET, SO_ERROR,
reinterpret_cast<char *>(&error), &len) >= 0 &&
!error;
}
return false;
@@ -1323,6 +1362,26 @@ inline bool is_connection_error() {
#endif
}
inline socket_t create_client_socket(
const char *host, int port, time_t timeout_sec) {
return create_socket(
host, port, [=](socket_t sock, struct addrinfo &ai) -> bool {
set_nonblocking(sock, true);
auto ret = ::connect(sock, ai.ai_addr, static_cast<int>(ai.ai_addrlen));
if (ret < 0) {
if (is_connection_error() ||
!wait_until_socket_is_ready(sock, timeout_sec, 0)) {
close_socket(sock);
return false;
}
}
set_nonblocking(sock, false);
return true;
});
}
inline std::string get_remote_addr(socket_t sock) {
struct sockaddr_storage addr;
socklen_t len = sizeof(addr);
@@ -1330,8 +1389,8 @@ inline std::string get_remote_addr(socket_t sock) {
if (!getpeername(sock, reinterpret_cast<struct sockaddr *>(&addr), &len)) {
std::array<char, NI_MAXHOST> ipstr{};
if (!getnameinfo(reinterpret_cast<struct sockaddr *>(&addr), len, ipstr.data(), ipstr.size(),
nullptr, 0, NI_NUMERICHOST)) {
if (!getnameinfo(reinterpret_cast<struct sockaddr *>(&addr), len,
ipstr.data(), ipstr.size(), nullptr, 0, NI_NUMERICHOST)) {
return ipstr.data();
}
}
@@ -1420,7 +1479,7 @@ inline bool compress(std::string &content) {
std::array<char, 16384> buff{};
do {
strm.avail_out = buff.size();
strm.next_out = reinterpret_cast<Bytef*>(buff.data());
strm.next_out = reinterpret_cast<Bytef *>(buff.data());
ret = deflate(&strm, Z_FINISH);
assert(ret != Z_STREAM_ERROR);
compressed.append(buff.data(), buff.size() - strm.avail_out);
@@ -1462,7 +1521,7 @@ public:
std::array<char, 16384> buff{};
do {
strm.avail_out = buff.size();
strm.next_out = reinterpret_cast<Bytef*>(buff.data());
strm.next_out = reinterpret_cast<Bytef *>(buff.data());
ret = inflate(&strm, Z_NO_FLUSH);
assert(ret != Z_STREAM_ERROR);
@@ -1472,7 +1531,9 @@ public:
case Z_MEM_ERROR: inflateEnd(&strm); return false;
}
if (!callback(buff.data(), buff.size() - strm.avail_out)) { return false; }
if (!callback(buff.data(), buff.size() - strm.avail_out)) {
return false;
}
} while (strm.avail_out == 0);
return ret == Z_OK || ret == Z_STREAM_END;
@@ -1506,18 +1567,35 @@ inline uint64_t get_header_value_uint64(const Headers &headers, const char *key,
}
inline bool read_headers(Stream &strm, Headers &headers) {
static std::regex re(R"((.+?):\s*(.+?)\s*\r\n)");
const auto bufsiz = 2048;
char buf[bufsiz];
stream_line_reader line_reader(strm, buf, bufsiz);
for (;;) {
if (!line_reader.getline()) { return false; }
if (!strcmp(line_reader.ptr(), "\r\n")) { break; }
// Check if the line ends with CRLF.
if (line_reader.end_with_crlf()) {
// Blank line indicates end of headers.
if (line_reader.size() == 2) { break; }
} else {
continue; // Skip invalid line.
}
// Skip trailing spaces and tabs.
auto end = line_reader.ptr() + line_reader.size() - 2;
while (line_reader.ptr() < end && (end[-1] == ' ' || end[-1] == '\t')) {
end--;
}
// Horizontal tab and ' ' are considered whitespace and are ignored when on
// the left or right side of the header value:
// - https://stackoverflow.com/questions/50179659/
// - https://www.w3.org/Protocols/rfc2616/rfc2616-sec4.html
static const std::regex re(R"((.+?):[\t ]*(.+))");
std::cmatch m;
if (std::regex_match(line_reader.ptr(), m, re)) {
if (std::regex_match(line_reader.ptr(), end, m, re)) {
auto key = std::string(m[1]);
auto val = std::string(m[2]);
headers.emplace(key, val);
@@ -1844,113 +1922,211 @@ inline bool parse_multipart_boundary(const std::string &content_type,
return true;
}
inline bool parse_multipart_formdata(const std::string &boundary,
const std::string &body,
MultipartFiles &files) {
static std::string dash = "--";
static std::string crlf = "\r\n";
inline bool parse_range_header(const std::string &s, Ranges &ranges) {
static auto re_first_range =
std::regex(R"(bytes=(\d*-\d*(?:,\s*\d*-\d*)*))");
std::smatch m;
if (std::regex_match(s, m, re_first_range)) {
auto pos = m.position(1);
auto len = m.length(1);
bool all_valid_ranges = true;
detail::split(
&s[pos], &s[pos + len], ',', [&](const char *b, const char *e) {
if (!all_valid_ranges) return;
static auto re_another_range = std::regex(R"(\s*(\d*)-(\d*))");
std::cmatch m;
if (std::regex_match(b, e, m, re_another_range)) {
ssize_t first = -1;
if (!m.str(1).empty()) {
first = static_cast<ssize_t>(std::stoll(m.str(1)));
}
static std::regex re_content_type("Content-Type: (.*?)$",
std::regex_constants::icase);
ssize_t last = -1;
if (!m.str(2).empty()) {
last = static_cast<ssize_t>(std::stoll(m.str(2)));
}
static std::regex re_content_disposition(
"Content-Disposition: form-data; name=\"(.*?)\"(?:; filename=\"(.*?)\")?",
std::regex_constants::icase);
if (first != -1 && last != -1 && first > last) {
all_valid_ranges = false;
return;
}
ranges.emplace_back(std::make_pair(first, last));
}
});
return all_valid_ranges;
}
return false;
}
auto dash_boundary = dash + boundary;
class MultipartFormDataParser {
public:
MultipartFormDataParser() {}
auto pos = body.find(dash_boundary);
if (pos != 0) { return false; }
pos += dash_boundary.size();
auto next_pos = body.find(crlf, pos);
if (next_pos == std::string::npos) { return false; }
pos = next_pos + crlf.size();
while (pos < body.size()) {
next_pos = body.find(crlf, pos);
if (next_pos == std::string::npos) { return false; }
std::string name;
MultipartFile file;
auto header = body.substr(pos, (next_pos - pos));
while (pos != next_pos) {
std::smatch m;
if (std::regex_match(header, m, re_content_type)) {
file.content_type = m[1];
} else if (std::regex_match(header, m, re_content_disposition)) {
name = m[1];
file.filename = m[2];
}
pos = next_pos + crlf.size();
next_pos = body.find(crlf, pos);
if (next_pos == std::string::npos) { return false; }
header = body.substr(pos, (next_pos - pos));
}
pos = next_pos + crlf.size();
next_pos = body.find(crlf + dash_boundary, pos);
if (next_pos == std::string::npos) { return false; }
file.offset = pos;
file.length = next_pos - pos;
pos = next_pos + crlf.size() + dash_boundary.size();
next_pos = body.find(crlf, pos);
if (next_pos == std::string::npos) { return false; }
files.emplace(name, file);
pos = next_pos + crlf.size();
void set_boundary(const std::string &boundary) {
boundary_ = boundary;
}
return true;
}
bool is_valid() const { return is_valid_; }
inline bool parse_range_header(const std::string &s, Ranges &ranges) {
try {
static auto re_first_range =
std::regex(R"(bytes=(\d*-\d*(?:,\s*\d*-\d*)*))");
std::smatch m;
if (std::regex_match(s, m, re_first_range)) {
auto pos = m.position(1);
auto len = m.length(1);
detail::split(
&s[pos], &s[pos + len], ',', [&](const char *b, const char *e) {
static auto re_another_range = std::regex(R"(\s*(\d*)-(\d*))");
std::cmatch m;
if (std::regex_match(b, e, m, re_another_range)) {
ssize_t first = -1;
if (!m.str(1).empty()) {
first = static_cast<ssize_t>(std::stoll(m.str(1)));
}
template <typename T, typename U>
bool parse(const char *buf, size_t n, T content_callback, U header_callback) {
static const std::regex re_content_type(R"(^Content-Type:\s*(.*?)\s*$)",
std::regex_constants::icase);
ssize_t last = -1;
if (!m.str(2).empty()) {
last = static_cast<ssize_t>(std::stoll(m.str(2)));
}
static const std::regex re_content_disposition(
"^Content-Disposition:\\s*form-data;\\s*name=\"(.*?)\"(?:;\\s*filename="
"\"(.*?)\")?\\s*$",
std::regex_constants::icase);
if (first != -1 && last != -1 && first > last) {
throw std::runtime_error("invalid range error");
}
ranges.emplace_back(std::make_pair(first, last));
buf_.append(buf, n); // TODO: performance improvement
while (!buf_.empty()) {
switch (state_) {
case 0: { // Initial boundary
auto pattern = dash_ + boundary_ + crlf_;
if (pattern.size() > buf_.size()) { return true; }
auto pos = buf_.find(pattern);
if (pos != 0) {
is_done_ = true;
return false;
}
buf_.erase(0, pattern.size());
off_ += pattern.size();
state_ = 1;
break;
}
case 1: { // New entry
clear_file_info();
state_ = 2;
break;
}
case 2: { // Headers
auto pos = buf_.find(crlf_);
while (pos != std::string::npos) {
if (pos == 0) {
if (!header_callback(name_, file_)) {
is_valid_ = false;
is_done_ = false;
return false;
}
});
return true;
buf_.erase(0, crlf_.size());
off_ += crlf_.size();
state_ = 3;
break;
}
auto header = buf_.substr(0, pos);
{
std::smatch m;
if (std::regex_match(header, m, re_content_type)) {
file_.content_type = m[1];
} else if (std::regex_match(header, m, re_content_disposition)) {
name_ = m[1];
file_.filename = m[2];
}
}
buf_.erase(0, pos + crlf_.size());
off_ += pos + crlf_.size();
pos = buf_.find(crlf_);
}
break;
}
case 3: { // Body
{
auto pattern = crlf_ + dash_;
auto pos = buf_.find(pattern);
if (pos == std::string::npos) {
pos = buf_.size();
}
if (!content_callback(name_, buf_.data(), pos)) {
is_valid_ = false;
is_done_ = false;
return false;
}
off_ += pos;
buf_.erase(0, pos);
}
{
auto pattern = crlf_ + dash_ + boundary_;
if (pattern.size() > buf_.size()) { return true; }
auto pos = buf_.find(pattern);
if (pos != std::string::npos) {
if (!content_callback(name_, buf_.data(), pos)) {
is_valid_ = false;
is_done_ = false;
return false;
}
off_ += pos + pattern.size();
buf_.erase(0, pos + pattern.size());
state_ = 4;
} else {
if (!content_callback(name_, buf_.data(), pattern.size())) {
is_valid_ = false;
is_done_ = false;
return false;
}
off_ += pattern.size();
buf_.erase(0, pattern.size());
}
}
break;
}
case 4: { // Boundary
if (crlf_.size() > buf_.size()) { return true; }
if (buf_.find(crlf_) == 0) {
buf_.erase(0, crlf_.size());
off_ += crlf_.size();
state_ = 1;
} else {
auto pattern = dash_ + crlf_;
if (pattern.size() > buf_.size()) { return true; }
if (buf_.find(pattern) == 0) {
buf_.erase(0, pattern.size());
off_ += pattern.size();
is_valid_ = true;
state_ = 5;
} else {
is_done_ = true;
return true;
}
}
break;
}
case 5: { // Done
is_valid_ = false;
return false;
}
}
}
return false;
} catch (...) { return false; }
}
return true;
}
private:
void clear_file_info() {
name_.clear();
file_.filename.clear();
file_.content_type.clear();
}
const std::string dash_ = "--";
const std::string crlf_ = "\r\n";
std::string boundary_;
std::string buf_;
size_t state_ = 0;
size_t is_valid_ = false;
size_t is_done_ = false;
size_t off_ = 0;
std::string name_;
MultipartFile file_;
};
inline std::string to_lower(const char *beg, const char *end) {
std::string out;
@@ -2102,6 +2278,15 @@ get_range_offset_and_length(const Request &req, const Response &res,
return std::make_pair(r.first, r.second - r.first + 1);
}
inline bool expect_content(const Request &req) {
if (req.method == "POST" || req.method == "PUT" || req.method == "PATCH" ||
req.method == "PRI") {
return true;
}
// TODO: check if Content-Length is set
return false;
}
#ifdef _WIN32
class WSInit {
public:
@@ -2177,6 +2362,11 @@ inline size_t Request::get_param_value_count(const char *key) const {
return std::distance(r.first, r.second);
}
inline bool Request::is_multipart_form_data() const {
const auto &content_type = get_header_value("Content-Type");
return !content_type.find("multipart/form-data");
}
inline bool Request::has_file(const char *key) const {
return files.find(key) != files.end();
}
@@ -2369,7 +2559,7 @@ inline Server &Server::Post(const char *pattern, Handler handler) {
inline Server &Server::Post(const char *pattern,
HandlerWithContentReader handler) {
post_handlers_for_content_reader.push_back(
post_handlers_for_content_reader_.push_back(
std::make_pair(std::regex(pattern), handler));
return *this;
}
@@ -2381,7 +2571,7 @@ inline Server &Server::Put(const char *pattern, Handler handler) {
inline Server &Server::Put(const char *pattern,
HandlerWithContentReader handler) {
put_handlers_for_content_reader.push_back(
put_handlers_for_content_reader_.push_back(
std::make_pair(std::regex(pattern), handler));
return *this;
}
@@ -2393,7 +2583,7 @@ inline Server &Server::Patch(const char *pattern, Handler handler) {
inline Server &Server::Patch(const char *pattern,
HandlerWithContentReader handler) {
patch_handlers_for_content_reader.push_back(
patch_handlers_for_content_reader_.push_back(
std::make_pair(std::regex(pattern), handler));
return *this;
}
@@ -2408,10 +2598,13 @@ inline Server &Server::Options(const char *pattern, Handler handler) {
return *this;
}
inline bool Server::set_base_dir(const char *path) {
if (detail::is_dir(path)) {
base_dir_ = path;
return true;
inline bool Server::set_base_dir(const char *dir, const char *mount_point) {
if (detail::is_dir(dir)) {
std::string mnt = mount_point ? mount_point : "/";
if (!mnt.empty() && mnt[0] == '/') {
base_dirs_.emplace_back(mnt, dir);
return true;
}
}
return false;
}
@@ -2643,25 +2836,78 @@ Server::write_content_with_provider(Stream &strm, const Request &req,
inline bool Server::read_content(Stream &strm, bool last_connection,
Request &req, Response &res) {
auto ret = read_content_core(strm, last_connection, req, res,
// Regular
[&](const char *buf, size_t n) {
if (req.body.size() + n > req.body.max_size()) { return false; }
req.body.append(buf, n);
return true;
},
// Multipart
[&](const std::string &name, const MultipartFile &file) {
req.files.emplace(name, file);
return true;
},
[&](const std::string &name, const char *buf, size_t n) {
// TODO: handle elements with a same key
auto it = req.files.find(name);
auto &content = it->second.content;
if (content.size() + n > content.max_size()) { return false; }
content.append(buf, n);
return true;
}
);
const auto &content_type = req.get_header_value("Content-Type");
if (!content_type.find("application/x-www-form-urlencoded")) {
detail::parse_query_text(req.body, req.params);
}
return ret;
}
inline bool
Server::read_content_with_content_receiver(Stream &strm, bool last_connection,
Request &req, Response &res,
ContentReceiver receiver,
MultipartContentHeader multipart_header,
MultipartContentReceiver multipart_receiver) {
return read_content_core(strm, last_connection, req, res,
receiver, multipart_header, multipart_receiver);
}
inline bool
Server::read_content_core(Stream &strm, bool last_connection,
Request &req, Response &res,
ContentReceiver receiver,
MultipartContentHeader mulitpart_header,
MultipartContentReceiver multipart_receiver) {
detail::MultipartFormDataParser multipart_form_data_parser;
ContentReceiver out;
if (req.is_multipart_form_data()) {
const auto &content_type = req.get_header_value("Content-Type");
std::string boundary;
if (!detail::parse_multipart_boundary(content_type, boundary)) {
res.status = 400;
return write_response(strm, last_connection, req, res);
}
multipart_form_data_parser.set_boundary(boundary);
out = [&](const char *buf, size_t n) {
return multipart_form_data_parser.parse(buf, n, multipart_receiver, mulitpart_header);
};
} else {
out = receiver;
}
if (!detail::read_content(strm, req, payload_max_length_, res.status,
Progress(), [&](const char *buf, size_t n) {
if (req.body.size() + n > req.body.max_size()) {
return false;
}
req.body.append(buf, n);
return true;
})) {
Progress(), out)) {
return write_response(strm, last_connection, req, res);
}
const auto &content_type = req.get_header_value("Content-Type");
if (!content_type.find("application/x-www-form-urlencoded")) {
detail::parse_query_text(req.body, req.params);
} else if (!content_type.find("multipart/form-data")) {
std::string boundary;
if (!detail::parse_multipart_boundary(content_type, boundary) ||
!detail::parse_multipart_formdata(boundary, req.body, req.files)) {
if (req.is_multipart_form_data()) {
if (!multipart_form_data_parser.is_valid()) {
res.status = 400;
return write_response(strm, last_connection, req, res);
}
@@ -2670,35 +2916,29 @@ inline bool Server::read_content(Stream &strm, bool last_connection,
return true;
}
inline bool
Server::read_content_with_content_receiver(Stream &strm, bool last_connection,
Request &req, Response &res,
ContentReceiver receiver) {
if (!detail::read_content(
strm, req, payload_max_length_, res.status, Progress(),
[&](const char *buf, size_t n) { return receiver(buf, n); })) {
return write_response(strm, last_connection, req, res);
}
return true;
}
inline bool Server::handle_file_request(Request &req, Response &res) {
if (!base_dir_.empty() && detail::is_valid_path(req.path)) {
std::string path = base_dir_ + req.path;
for (const auto &kv : base_dirs_) {
const auto &mount_point = kv.first;
const auto &base_dir = kv.second;
if (!path.empty() && path.back() == '/') { path += "index.html"; }
// Prefix match
if (!req.path.find(mount_point)) {
std::string sub_path = "/" + req.path.substr(mount_point.size());
if (detail::is_valid_path(sub_path)) {
auto path = base_dir + sub_path;
if (path.back() == '/') { path += "index.html"; }
if (detail::is_file(path)) {
detail::read_file(path, res.body);
auto type = detail::find_content_type(path);
if (type) { res.set_header("Content-Type", type); }
res.status = 200;
if (file_request_handler_) { file_request_handler_(req, res); }
return true;
if (detail::is_file(path)) {
detail::read_file(path, res.body);
auto type = detail::find_content_type(path);
if (type) { res.set_header("Content-Type", type); }
res.status = 200;
if (file_request_handler_) { file_request_handler_(req, res); }
return true;
}
}
}
}
return false;
}
@@ -2734,7 +2974,8 @@ inline int Server::bind_internal(const char *host, int port, int socket_flags) {
if (address.ss_family == AF_INET) {
return ntohs(reinterpret_cast<struct sockaddr_in *>(&address)->sin_port);
} else if (address.ss_family == AF_INET6) {
return ntohs(reinterpret_cast<struct sockaddr_in6 *>(&address)->sin6_port);
return ntohs(
reinterpret_cast<struct sockaddr_in6 *>(&address)->sin6_port);
} else {
return -1;
}
@@ -2790,39 +3031,45 @@ inline bool Server::listen_internal() {
return ret;
}
inline bool Server::routing(Request &req, Response &res, Stream &strm, bool last_connection) {
inline bool Server::routing(Request &req, Response &res, Stream &strm,
bool last_connection) {
// File handler
if (req.method == "GET" && handle_file_request(req, res)) { return true; }
// Content reader handler
if (req.method == "POST" || req.method == "PUT" || req.method == "PATCH") {
ContentReader content_reader = [&](ContentReceiver receiver) {
return read_content_with_content_receiver(strm, last_connection, req, res, receiver);
};
if (detail::expect_content(req)) {
// Content reader handler
{
ContentReader reader(
[&](ContentReceiver receiver) {
return read_content_with_content_receiver(strm, last_connection, req, res,
receiver, nullptr, nullptr);
},
[&](MultipartContentHeader header, MultipartContentReceiver receiver) {
return read_content_with_content_receiver(strm, last_connection, req, res,
nullptr, header, receiver);
}
);
if (req.method == "POST") {
if (dispatch_request_for_content_reader(req, res, content_reader,
post_handlers_for_content_reader)) {
return true;
}
} else if (req.method == "PUT") {
if (dispatch_request_for_content_reader(req, res, content_reader,
put_handlers_for_content_reader)) {
return true;
}
} else if (req.method == "PATCH") {
if (dispatch_request_for_content_reader(
req, res, content_reader, patch_handlers_for_content_reader)) {
return true;
if (req.method == "POST") {
if (dispatch_request_for_content_reader(
req, res, reader, post_handlers_for_content_reader_)) {
return true;
}
} else if (req.method == "PUT") {
if (dispatch_request_for_content_reader(
req, res, reader, put_handlers_for_content_reader_)) {
return true;
}
} else if (req.method == "PATCH") {
if (dispatch_request_for_content_reader(
req, res, reader, patch_handlers_for_content_reader_)) {
return true;
}
}
}
}
// Read content into `req.body`
if (req.method == "POST" || req.method == "PUT" || req.method == "PATCH" || req.method == "PRI") {
if (!read_content(strm, last_connection, req, res)) {
return false;
}
// Read content into `req.body`
if (!read_content(strm, last_connection, req, res)) { return false; }
}
// Regular handler
@@ -2877,7 +3124,7 @@ Server::dispatch_request_for_content_reader(Request &req, Response &res,
inline bool
Server::process_request(Stream &strm, bool last_connection,
bool &connection_close,
const std::function<void(Request &)>& setup_request) {
const std::function<void(Request &)> &setup_request) {
std::array<char, 2048> buf{};
detail::stream_line_reader line_reader(strm, buf.data(), buf.size());
@@ -2960,22 +3207,7 @@ inline Client::~Client() {}
inline bool Client::is_valid() const { return true; }
inline socket_t Client::create_client_socket() const {
return detail::create_socket(
host_.c_str(), port_, [=](socket_t sock, struct addrinfo &ai) -> bool {
detail::set_nonblocking(sock, true);
auto ret = connect(sock, ai.ai_addr, static_cast<int>(ai.ai_addrlen));
if (ret < 0) {
if (detail::is_connection_error() ||
!detail::wait_until_socket_is_ready(sock, timeout_sec_, 0)) {
detail::close_socket(sock);
return false;
}
}
detail::set_nonblocking(sock, false);
return true;
});
return detail::create_client_socket(host_.c_str(), port_, timeout_sec_);
}
inline bool Client::read_response_line(Stream &strm, Response &res) {
@@ -3332,7 +3564,8 @@ inline std::shared_ptr<Response> Client::Get(const char *path,
ResponseHandler response_handler,
ContentReceiver content_receiver) {
Progress dummy;
return Get(path, headers, std::move(response_handler), content_receiver, dummy);
return Get(path, headers, std::move(response_handler), content_receiver,
dummy);
}
inline std::shared_ptr<Response> Client::Get(const char *path,
+198 -22
View File
@@ -30,6 +30,12 @@ const std::string JSON_DATA = "{\"hello\":\"world\"}";
const string LARGE_DATA = string(1024 * 1024 * 100, '@'); // 100MB
MultipartFile& get_file_value(MultipartFiles &files, const char *key) {
auto it = files.find(key);
if (it != files.end()) { return it->second; }
throw std::runtime_error("invalid mulitpart form data name error");
}
#ifdef _WIN32
TEST(StartupTest, WSAStartup) {
WSADATA wsaData;
@@ -567,6 +573,7 @@ protected:
virtual void SetUp() {
svr_.set_base_dir("./www");
svr_.set_base_dir("./www2", "/mount");
svr_.Get("/hi",
[&](const Request & /*req*/, Response &res) {
@@ -675,29 +682,27 @@ protected:
{
const auto &file = req.get_file_value("text1");
EXPECT_EQ("", file.filename);
EXPECT_EQ("text default",
req.body.substr(file.offset, file.length));
EXPECT_EQ("text default", file.content);
}
{
const auto &file = req.get_file_value("text2");
EXPECT_EQ("", file.filename);
EXPECT_EQ("aωb", req.body.substr(file.offset, file.length));
EXPECT_EQ("aωb", file.content);
}
{
const auto &file = req.get_file_value("file1");
EXPECT_EQ("hello.txt", file.filename);
EXPECT_EQ("text/plain", file.content_type);
EXPECT_EQ("h\ne\n\nl\nl\no\n",
req.body.substr(file.offset, file.length));
EXPECT_EQ("h\ne\n\nl\nl\no\n", file.content);
}
{
const auto &file = req.get_file_value("file3");
EXPECT_EQ("", file.filename);
EXPECT_EQ("application/octet-stream", file.content_type);
EXPECT_EQ(0u, file.length);
EXPECT_EQ(0u, file.content.size());
}
})
.Post("/empty",
@@ -752,16 +757,57 @@ protected:
EXPECT_EQ("5", req.get_header_value("Content-Length"));
})
.Post("/content_receiver",
[&](const Request & /*req*/, Response &res,
const ContentReader &content_reader) {
std::string body;
content_reader([&](const char *data, size_t data_length) {
EXPECT_EQ(data_length, 7);
body.append(data, data_length);
return true;
});
EXPECT_EQ(body, "content");
res.set_content(body, "text/plain");
[&](const Request & req, Response &res, const ContentReader &content_reader) {
if (req.is_multipart_form_data()) {
MultipartFiles files;
content_reader(
[&](const std::string &name, const MultipartFile &file) {
files.emplace(name, file);
return true;
},
[&](const std::string &name, const char *data, size_t data_length) {
auto &file = files.find(name)->second;
file.content.append(data, data_length);
return true;
});
EXPECT_EQ(5u, files.size());
{
const auto &file = get_file_value(files, "text1");
EXPECT_EQ("", file.filename);
EXPECT_EQ("text default", file.content);
}
{
const auto &file = get_file_value(files, "text2");
EXPECT_EQ("", file.filename);
EXPECT_EQ("aωb", file.content);
}
{
const auto &file = get_file_value(files, "file1");
EXPECT_EQ("hello.txt", file.filename);
EXPECT_EQ("text/plain", file.content_type);
EXPECT_EQ("h\ne\n\nl\nl\no\n", file.content);
}
{
const auto &file = get_file_value(files, "file3");
EXPECT_EQ("", file.filename);
EXPECT_EQ("application/octet-stream", file.content_type);
EXPECT_EQ(0u, file.content.size());
}
} else {
std::string body;
content_reader([&](const char *data, size_t data_length) {
EXPECT_EQ(data_length, 7);
body.append(data, data_length);
return true;
});
EXPECT_EQ(body, "content");
res.set_content(body, "text/plain");
}
})
.Put("/content_receiver",
[&](const Request & /*req*/, Response &res,
@@ -808,14 +854,13 @@ protected:
{
const auto &file = req.get_file_value("key1");
EXPECT_EQ("", file.filename);
EXPECT_EQ("test", req.body.substr(file.offset, file.length));
EXPECT_EQ("test", file.content);
}
{
const auto &file = req.get_file_value("key2");
EXPECT_EQ("", file.filename);
EXPECT_EQ("--abcdefg123",
req.body.substr(file.offset, file.length));
EXPECT_EQ("--abcdefg123", file.content);
}
})
#endif
@@ -1003,9 +1048,42 @@ TEST_F(ServerTest, GetMethodOutOfBaseDir2) {
EXPECT_EQ(404, res->status);
}
TEST_F(ServerTest, InvalidBaseDir) {
EXPECT_EQ(false, svr_.set_base_dir("invalid_dir"));
EXPECT_EQ(true, svr_.set_base_dir("."));
TEST_F(ServerTest, GetMethodDirMountTest) {
auto res = cli_.Get("/mount/dir/test.html");
ASSERT_TRUE(res != nullptr);
EXPECT_EQ(200, res->status);
EXPECT_EQ("text/html", res->get_header_value("Content-Type"));
EXPECT_EQ("test.html", res->body);
}
TEST_F(ServerTest, GetMethodDirMountTestWithDoubleDots) {
auto res = cli_.Get("/mount/dir/../dir/test.html");
ASSERT_TRUE(res != nullptr);
EXPECT_EQ(200, res->status);
EXPECT_EQ("text/html", res->get_header_value("Content-Type"));
EXPECT_EQ("test.html", res->body);
}
TEST_F(ServerTest, GetMethodInvalidMountPath) {
auto res = cli_.Get("/mount/dir/../test.html");
ASSERT_TRUE(res != nullptr);
EXPECT_EQ(404, res->status);
}
TEST_F(ServerTest, GetMethodOutOfBaseDirMount) {
auto res = cli_.Get("/mount/../www2/dir/test.html");
ASSERT_TRUE(res != nullptr);
EXPECT_EQ(404, res->status);
}
TEST_F(ServerTest, GetMethodOutOfBaseDirMount2) {
auto res = cli_.Get("/mount/dir/../../www2/dir/test.html");
ASSERT_TRUE(res != nullptr);
EXPECT_EQ(404, res->status);
}
TEST_F(ServerTest, InvalidBaseDirMount) {
EXPECT_EQ(false, svr_.set_base_dir("./www3", "invalid_mount_point"));
}
TEST_F(ServerTest, EmptyRequest) {
@@ -1484,6 +1562,21 @@ TEST_F(ServerTest, PostContentReceiver) {
ASSERT_EQ("content", res->body);
}
TEST_F(ServerTest, PostMulitpartFilsContentReceiver) {
MultipartFormDataItems items = {
{"text1", "text default", "", ""},
{"text2", "aωb", "", ""},
{"file1", "h\ne\n\nl\nl\no\n", "hello.txt", "text/plain"},
{"file2", "{\n \"world\", true\n}\n", "world.json", "application/json"},
{"file3", "", "", "application/octet-stream"},
};
auto res = cli_.Post("/content_receiver", items);
ASSERT_TRUE(res != nullptr);
EXPECT_EQ(200, res->status);
}
TEST_F(ServerTest, PostContentReceiverGzip) {
auto res = cli_.Post("/content_receiver", "content", "text/plain", true);
ASSERT_TRUE(res != nullptr);
@@ -1673,6 +1766,89 @@ TEST_F(ServerTest, MultipartFormDataGzip) {
}
#endif
// Sends a raw request to a server listening at HOST:PORT.
static bool send_request(time_t read_timeout_sec, const std::string& req) {
auto client_sock =
detail::create_client_socket(HOST, PORT, /*timeout_sec=*/5);
if (client_sock == INVALID_SOCKET) { return false; }
return detail::process_and_close_socket(
true, client_sock, 1, read_timeout_sec, 0,
[&](Stream& strm, bool /*last_connection*/,
bool &/*connection_close*/) -> bool {
if (req.size() !=
static_cast<size_t>(strm.write(req.data(), req.size()))) {
return false;
}
char buf[512];
detail::stream_line_reader line_reader(strm, buf, sizeof(buf));
while (line_reader.getline()) {}
return true;
});
}
TEST(ServerRequestParsingTest, TrimWhitespaceFromHeaderValues) {
Server svr;
std::string header_value;
svr.Get("/validate-ws-in-headers",
[&](const Request &req, Response &res) {
header_value = req.get_header_value("foo");
res.set_content("ok", "text/plain");
});
thread t = thread([&] { svr.listen(HOST, PORT); });
while (!svr.is_running()) {
msleep(1);
}
// Only space and horizontal tab are whitespace. Make sure other whitespace-
// like characters are not treated the same - use vertical tab and escape.
const std::string req =
"GET /validate-ws-in-headers HTTP/1.1\r\n"
"foo: \t \v bar \e\t \r\n"
"Connection: close\r\n"
"\r\n";
ASSERT_TRUE(send_request(5, req));
svr.stop();
t.join();
EXPECT_EQ(header_value, "\v bar \e");
}
TEST(ServerRequestParsingTest, ReadHeadersRegexComplexity) {
Server svr;
svr.Get("/hi",
[&](const Request & /*req*/, Response &res) {
res.set_content("ok", "text/plain");
});
// Server read timeout must be longer than the client read timeout for the
// bug to reproduce, probably to force the server to process a request
// without a trailing blank line.
const time_t client_read_timeout_sec = 1;
svr.set_read_timeout(client_read_timeout_sec + 1, 0);
bool listen_thread_ok = false;
thread t = thread([&] { listen_thread_ok = svr.listen(HOST, PORT); });
while (!svr.is_running()) {
msleep(1);
}
// A certain header line causes an exception if the header property is parsed
// naively with a single regex. This occurs with libc++ but not libstdc++.
const std::string req =
"GET /hi HTTP/1.1\r\n"
" : "
" ";
ASSERT_TRUE(send_request(client_read_timeout_sec, req));
svr.stop();
t.join();
EXPECT_TRUE(listen_thread_ok);
}
class ServerTestWithAI_PASSIVE : public ::testing::Test {
protected:
ServerTestWithAI_PASSIVE()
+8
View File
@@ -0,0 +1,8 @@
<html>
<head>
</head>
<body>
<a href="/dir/test.html">Test</a>
<a href="/hi">hi</a>
</body>
</html>
+1
View File
@@ -0,0 +1 @@
test.html
+8
View File
@@ -0,0 +1,8 @@
<html>
<head>
</head>
<body>
<a href="/dir/test.html">Test</a>
<a href="/hi">hi</a>
</body>
</html>
+1
View File
@@ -0,0 +1 @@
test.html