Compare commits

...

7 Commits

Author SHA1 Message Date
yhirose 8e10d4e8e7 Release v0.11.2 2022-09-12 11:36:14 -04:00
Changbin Park b57f79f438 Detecting client disconnection (#1373)
* SocketStream need to check connectivity for writability.

When the stream is used for SSE server which works via chunked content
provider it's not possible to check connectivity over writability
because it's wrapped by DataSink. It could make the server stalled, when
the provider wants to only keep connection without send data and certain
amount of clients consumes entire threadpool.

* add unittest for SocketStream::is_writable()

SocketStream::is_writable() should return false if peer is disconnected.

* revise broken unittest ServerTest.ClientStop

DataSink could be unwritable if it's backed by connection based. Because
it could be disconnected by client right after enter centent provider.

Co-authored-by: Changbin Park <changbin.park@ahnlab.com>
2022-08-30 21:11:19 -04:00
Andrea Pappacoda a9cf097951 build: set soversion to major.minor (#1357)
Release 0.11 broke backwards compatibility, meaning that different
cpp-httplib versions are compatible with each other only if the major
and minor version numbers are the same.

This patch reflects this in the build systems.

See #1209 for some more context.
2022-08-12 13:48:40 -04:00
yhirose 5c3624e1af Updated example/uploader.sh 2022-08-06 14:43:56 -04:00
yhirose cba9ef8c0b Issue 49740 in oss-fuzz: cpp-httplib:server_fuzzer: Timeout in server_fuzzer 2022-08-06 08:08:08 -04:00
yhirose 4f8407a3a7 Refactoring the previous commit 2022-08-04 20:56:02 -04:00
Gopinath K 656e936f49 add multipart formdata for PUT requests. (#1351)
* httplib.h

  add multipart formdata for PUT in addition to POST as some REST
  APIs use that.

  Factor the boundary checking code into a helper and use it from
  both Post() and Put().

* test/test.cc

  add test cases for the above.
2022-08-04 20:42:13 -04:00
5 changed files with 420 additions and 42 deletions
+3 -3
View File
@@ -176,7 +176,7 @@ if(HTTPLIB_COMPILE)
set_target_properties(${PROJECT_NAME}
PROPERTIES
VERSION ${${PROJECT_NAME}_VERSION}
SOVERSION ${${PROJECT_NAME}_VERSION_MAJOR}
SOVERSION "${${PROJECT_NAME}_VERSION_MAJOR}.${${PROJECT_NAME}_VERSION_MINOR}"
)
else()
# This is for header-only.
@@ -247,13 +247,13 @@ if(HTTPLIB_COMPILE)
write_basic_package_version_file("${PROJECT_NAME}ConfigVersion.cmake"
# Example: if you find_package(httplib 0.5.4)
# then anything >= 0.5 and <= 1.0 is accepted
COMPATIBILITY SameMajorVersion
COMPATIBILITY SameMinorVersion
)
else()
write_basic_package_version_file("${PROJECT_NAME}ConfigVersion.cmake"
# Example: if you find_package(httplib 0.5.4)
# then anything >= 0.5 and <= 1.0 is accepted
COMPATIBILITY SameMajorVersion
COMPATIBILITY SameMinorVersion
# Tells Cmake that it's a header-only lib
# Mildly useful for end-users :)
ARCH_INDEPENDENT
+1 -1
View File
@@ -1,5 +1,5 @@
#/usr/bin/env bash
for i in {1..10000}
for i in {1..1000000}
do
echo "#### $i ####"
curl -X POST -F image_file=@$1 http://localhost:1234/post > /dev/null
+116 -37
View File
@@ -8,7 +8,7 @@
#ifndef CPPHTTPLIB_HTTPLIB_H
#define CPPHTTPLIB_HTTPLIB_H
#define CPPHTTPLIB_VERSION "0.11.1"
#define CPPHTTPLIB_VERSION "0.11.2"
/*
* Configuration
@@ -70,6 +70,10 @@
#define CPPHTTPLIB_REDIRECT_MAX_COUNT 20
#endif
#ifndef CPPHTTPLIB_MULTIPART_FORM_DATA_FILE_MAX_COUNT
#define CPPHTTPLIB_MULTIPART_FORM_DATA_FILE_MAX_COUNT 1024
#endif
#ifndef CPPHTTPLIB_PAYLOAD_MAX_LENGTH
#define CPPHTTPLIB_PAYLOAD_MAX_LENGTH ((std::numeric_limits<size_t>::max)())
#endif
@@ -949,6 +953,11 @@ public:
Result Put(const std::string &path, const Params &params);
Result Put(const std::string &path, const Headers &headers,
const Params &params);
Result Put(const std::string &path, const MultipartFormDataItems &items);
Result Put(const std::string &path, const Headers &headers,
const MultipartFormDataItems &items);
Result Put(const std::string &path, const Headers &headers,
const MultipartFormDataItems &items, const std::string &boundary);
Result Patch(const std::string &path);
Result Patch(const std::string &path, const char *body, size_t content_length,
@@ -1304,6 +1313,11 @@ public:
Result Put(const std::string &path, const Params &params);
Result Put(const std::string &path, const Headers &headers,
const Params &params);
Result Put(const std::string &path, const MultipartFormDataItems &items);
Result Put(const std::string &path, const Headers &headers,
const MultipartFormDataItems &items);
Result Put(const std::string &path, const Headers &headers,
const MultipartFormDataItems &items, const std::string &boundary);
Result Patch(const std::string &path);
Result Patch(const std::string &path, const char *body, size_t content_length,
const std::string &content_type);
@@ -2582,7 +2596,7 @@ socket_t create_socket(const std::string &host, const std::string &ip, int port,
addr.sun_family = AF_UNIX;
std::copy(host.begin(), host.end(), addr.sun_path);
hints.ai_addr = reinterpret_cast<sockaddr*>(&addr);
hints.ai_addr = reinterpret_cast<sockaddr *>(&addr);
hints.ai_addrlen = static_cast<socklen_t>(
sizeof(addr) - sizeof(addr.sun_path) + addrlen);
@@ -4064,6 +4078,44 @@ inline std::string make_multipart_data_boundary() {
return result;
}
inline bool is_multipart_boundary_chars_valid(const std::string &boundary) {
auto valid = true;
for (size_t i = 0; i < boundary.size(); i++) {
auto c = boundary[i];
if (!std::isalnum(c) && c != '-' && c != '_') {
valid = false;
break;
}
}
return valid;
}
inline std::string
serialize_multipart_formdata(const MultipartFormDataItems &items,
const std::string &boundary,
std::string &content_type) {
std::string body;
for (const auto &item : items) {
body += "--" + boundary + "\r\n";
body += "Content-Disposition: form-data; name=\"" + item.name + "\"";
if (!item.filename.empty()) {
body += "; filename=\"" + item.filename + "\"";
}
body += "\r\n";
if (!item.content_type.empty()) {
body += "Content-Type: " + item.content_type + "\r\n";
}
body += "\r\n";
body += item.content + "\r\n";
}
body += "--" + boundary + "--\r\n";
content_type = "multipart/form-data; boundary=" + boundary;
return body;
}
inline std::pair<size_t, size_t>
get_range_offset_and_length(const Request &req, size_t content_length,
size_t index) {
@@ -4670,7 +4722,8 @@ inline bool SocketStream::is_readable() const {
}
inline bool SocketStream::is_writable() const {
return select_write(sock_, write_timeout_sec_, write_timeout_usec_) > 0;
return select_write(sock_, write_timeout_sec_, write_timeout_usec_) > 0 &&
is_socket_alive(sock_);
}
inline ssize_t SocketStream::read(char *ptr, size_t size) {
@@ -5207,6 +5260,7 @@ Server::write_content_with_provider(Stream &strm, const Request &req,
inline bool Server::read_content(Stream &strm, Request &req, Response &res) {
MultipartFormDataMap::iterator cur;
auto file_count = 0;
if (read_content_core(
strm, req, res,
// Regular
@@ -5217,6 +5271,9 @@ inline bool Server::read_content(Stream &strm, Request &req, Response &res) {
},
// Multipart
[&](const MultipartFormData &file) {
if (file_count++ == CPPHTTPLIB_MULTIPART_FORM_DATA_FILE_MAX_COUNT) {
return false;
}
cur = req.files.emplace(file.name, file);
return true;
},
@@ -6745,37 +6802,22 @@ inline Result ClientImpl::Post(const std::string &path,
inline Result ClientImpl::Post(const std::string &path, const Headers &headers,
const MultipartFormDataItems &items) {
return Post(path, headers, items, detail::make_multipart_data_boundary());
std::string content_type;
const auto &body = detail::serialize_multipart_formdata(
items, detail::make_multipart_data_boundary(), content_type);
return Post(path, headers, body, content_type.c_str());
}
inline Result ClientImpl::Post(const std::string &path, const Headers &headers,
const MultipartFormDataItems &items,
const std::string &boundary) {
for (size_t i = 0; i < boundary.size(); i++) {
char c = boundary[i];
if (!std::isalnum(c) && c != '-' && c != '_') {
return Result{nullptr, Error::UnsupportedMultipartBoundaryChars};
}
if (!detail::is_multipart_boundary_chars_valid(boundary)) {
return Result{nullptr, Error::UnsupportedMultipartBoundaryChars};
}
std::string body;
for (const auto &item : items) {
body += "--" + boundary + "\r\n";
body += "Content-Disposition: form-data; name=\"" + item.name + "\"";
if (!item.filename.empty()) {
body += "; filename=\"" + item.filename + "\"";
}
body += "\r\n";
if (!item.content_type.empty()) {
body += "Content-Type: " + item.content_type + "\r\n";
}
body += "\r\n";
body += item.content + "\r\n";
}
body += "--" + boundary + "--\r\n";
std::string content_type = "multipart/form-data; boundary=" + boundary;
std::string content_type;
const auto &body =
detail::serialize_multipart_formdata(items, boundary, content_type);
return Post(path, headers, body, content_type.c_str());
}
@@ -6848,6 +6890,32 @@ inline Result ClientImpl::Put(const std::string &path, const Headers &headers,
return Put(path, headers, query, "application/x-www-form-urlencoded");
}
inline Result ClientImpl::Put(const std::string &path,
const MultipartFormDataItems &items) {
return Put(path, Headers(), items);
}
inline Result ClientImpl::Put(const std::string &path, const Headers &headers,
const MultipartFormDataItems &items) {
std::string content_type;
const auto &body = detail::serialize_multipart_formdata(
items, detail::make_multipart_data_boundary(), content_type);
return Put(path, headers, body, content_type);
}
inline Result ClientImpl::Put(const std::string &path, const Headers &headers,
const MultipartFormDataItems &items,
const std::string &boundary) {
if (!detail::is_multipart_boundary_chars_valid(boundary)) {
return Result{nullptr, Error::UnsupportedMultipartBoundaryChars};
}
std::string content_type;
const auto &body =
detail::serialize_multipart_formdata(items, boundary, content_type);
return Put(path, headers, body, content_type);
}
inline Result ClientImpl::Patch(const std::string &path) {
return Patch(path, std::string(), std::string());
}
@@ -6973,9 +7041,7 @@ inline size_t ClientImpl::is_socket_open() const {
return socket_.is_open();
}
inline socket_t ClientImpl::socket() const {
return socket_.sock;
}
inline socket_t ClientImpl::socket() const { return socket_.sock; }
inline void ClientImpl::stop() {
std::lock_guard<std::mutex> guard(socket_mutex_);
@@ -7280,8 +7346,8 @@ inline bool SSLSocketStream::is_readable() const {
}
inline bool SSLSocketStream::is_writable() const {
return detail::select_write(sock_, write_timeout_sec_, write_timeout_usec_) >
0;
return select_write(sock_, write_timeout_sec_, write_timeout_usec_) > 0 &&
is_socket_alive(sock_);
}
inline ssize_t SSLSocketStream::read(char *ptr, size_t size) {
@@ -7883,13 +7949,13 @@ inline Client::Client(const std::string &scheme_host_port,
if (is_ssl) {
#ifdef CPPHTTPLIB_OPENSSL_SUPPORT
cli_ = detail::make_unique<SSLClient>(host, port,
client_cert_path, client_key_path);
cli_ = detail::make_unique<SSLClient>(host, port, client_cert_path,
client_key_path);
is_ssl_ = is_ssl;
#endif
} else {
cli_ = detail::make_unique<ClientImpl>(host, port,
client_cert_path, client_key_path);
cli_ = detail::make_unique<ClientImpl>(host, port, client_cert_path,
client_key_path);
}
} else {
cli_ = detail::make_unique<ClientImpl>(scheme_host_port, 80,
@@ -8099,6 +8165,19 @@ inline Result Client::Put(const std::string &path, const Headers &headers,
const Params &params) {
return cli_->Put(path, headers, params);
}
inline Result Client::Put(const std::string &path,
const MultipartFormDataItems &items) {
return cli_->Put(path, items);
}
inline Result Client::Put(const std::string &path, const Headers &headers,
const MultipartFormDataItems &items) {
return cli_->Put(path, headers, items);
}
inline Result Client::Put(const std::string &path, const Headers &headers,
const MultipartFormDataItems &items,
const std::string &boundary) {
return cli_->Put(path, headers, items, boundary);
}
inline Result Client::Patch(const std::string &path) {
return cli_->Patch(path);
}
+1
View File
@@ -77,6 +77,7 @@ if get_option('cpp-httplib_compile')
dependencies: deps,
cpp_args: args,
version: version,
soversion: version.split('.')[0] + '.' + version.split('.')[1],
install: true
)
cpp_httplib_dep = declare_dependency(compile_args: args, dependencies: deps, link_with: lib, sources: httplib_ch[1])
+299 -1
View File
@@ -1643,7 +1643,8 @@ protected:
res.set_content_provider(
size_t(-1), "text/plain",
[](size_t /*offset*/, size_t /*length*/, DataSink &sink) {
EXPECT_TRUE(sink.is_writable());
if (!sink.is_writable()) return false;
sink.os << "data_chunk";
return true;
});
@@ -5062,6 +5063,241 @@ TEST(MultipartFormDataTest, WithPreamble) {
t.join();
}
TEST(MultipartFormDataTest, PostCustomBoundary) {
SSLServer svr(SERVER_CERT_FILE, SERVER_PRIVATE_KEY_FILE);
svr.Post("/post_customboundary", [&](const Request &req, Response & /*res*/,
const ContentReader &content_reader) {
if (req.is_multipart_form_data()) {
MultipartFormDataItems files;
content_reader(
[&](const MultipartFormData &file) {
files.push_back(file);
return true;
},
[&](const char *data, size_t data_length) {
files.back().content.append(data, data_length);
return true;
});
EXPECT_TRUE(std::string(files[0].name) == "document");
EXPECT_EQ(size_t(1024 * 1024 * 2), files[0].content.size());
EXPECT_TRUE(files[0].filename == "2MB_data");
EXPECT_TRUE(files[0].content_type == "application/octet-stream");
EXPECT_TRUE(files[1].name == "hello");
EXPECT_TRUE(files[1].content == "world");
EXPECT_TRUE(files[1].filename == "");
EXPECT_TRUE(files[1].content_type == "");
} else {
std::string body;
content_reader([&](const char *data, size_t data_length) {
body.append(data, data_length);
return true;
});
}
});
auto t = std::thread([&]() { svr.listen("localhost", 8080); });
while (!svr.is_running()) {
std::this_thread::sleep_for(std::chrono::milliseconds(1));
}
std::this_thread::sleep_for(std::chrono::seconds(1));
{
std::string data(1024 * 1024 * 2, '.');
std::stringstream buffer;
buffer << data;
Client cli("https://localhost:8080");
cli.enable_server_certificate_verification(false);
MultipartFormDataItems items{
{"document", buffer.str(), "2MB_data", "application/octet-stream"},
{"hello", "world", "", ""},
};
auto res = cli.Post("/post_customboundary", {}, items, "abc-abc");
ASSERT_TRUE(res);
ASSERT_EQ(200, res->status);
}
svr.stop();
t.join();
}
TEST(MultipartFormDataTest, PostInvalidBoundaryChars) {
std::this_thread::sleep_for(std::chrono::seconds(1));
std::string data(1024 * 1024 * 2, '&');
std::stringstream buffer;
buffer << data;
Client cli("https://localhost:8080");
MultipartFormDataItems items{
{"document", buffer.str(), "2MB_data", "application/octet-stream"},
{"hello", "world", "", ""},
};
for (const char& c: " \t\r\n") {
auto res = cli.Post("/invalid_boundary", {}, items, string("abc123").append(1, c));
ASSERT_EQ(Error::UnsupportedMultipartBoundaryChars, res.error());
ASSERT_FALSE(res);
}
}
TEST(MultipartFormDataTest, PutFormData) {
SSLServer svr(SERVER_CERT_FILE, SERVER_PRIVATE_KEY_FILE);
svr.Put("/put", [&](const Request &req, const Response & /*res*/,
const ContentReader &content_reader) {
if (req.is_multipart_form_data()) {
MultipartFormDataItems files;
content_reader(
[&](const MultipartFormData &file) {
files.push_back(file);
return true;
},
[&](const char *data, size_t data_length) {
files.back().content.append(data, data_length);
return true;
});
EXPECT_TRUE(std::string(files[0].name) == "document");
EXPECT_EQ(size_t(1024 * 1024 * 2), files[0].content.size());
EXPECT_TRUE(files[0].filename == "2MB_data");
EXPECT_TRUE(files[0].content_type == "application/octet-stream");
EXPECT_TRUE(files[1].name == "hello");
EXPECT_TRUE(files[1].content == "world");
EXPECT_TRUE(files[1].filename == "");
EXPECT_TRUE(files[1].content_type == "");
} else {
std::string body;
content_reader([&](const char *data, size_t data_length) {
body.append(data, data_length);
return true;
});
}
});
auto t = std::thread([&]() { svr.listen("localhost", 8080); });
while (!svr.is_running()) {
std::this_thread::sleep_for(std::chrono::milliseconds(1));
}
std::this_thread::sleep_for(std::chrono::seconds(1));
{
std::string data(1024 * 1024 * 2, '&');
std::stringstream buffer;
buffer << data;
Client cli("https://localhost:8080");
cli.enable_server_certificate_verification(false);
MultipartFormDataItems items{
{"document", buffer.str(), "2MB_data", "application/octet-stream"},
{"hello", "world", "", ""},
};
auto res = cli.Put("/put", items);
ASSERT_TRUE(res);
ASSERT_EQ(200, res->status);
}
svr.stop();
t.join();
}
TEST(MultipartFormDataTest, PutFormDataCustomBoundary) {
SSLServer svr(SERVER_CERT_FILE, SERVER_PRIVATE_KEY_FILE);
svr.Put("/put_customboundary", [&](const Request &req, const Response & /*res*/,
const ContentReader &content_reader) {
if (req.is_multipart_form_data()) {
MultipartFormDataItems files;
content_reader(
[&](const MultipartFormData &file) {
files.push_back(file);
return true;
},
[&](const char *data, size_t data_length) {
files.back().content.append(data, data_length);
return true;
});
EXPECT_TRUE(std::string(files[0].name) == "document");
EXPECT_EQ(size_t(1024 * 1024 * 2), files[0].content.size());
EXPECT_TRUE(files[0].filename == "2MB_data");
EXPECT_TRUE(files[0].content_type == "application/octet-stream");
EXPECT_TRUE(files[1].name == "hello");
EXPECT_TRUE(files[1].content == "world");
EXPECT_TRUE(files[1].filename == "");
EXPECT_TRUE(files[1].content_type == "");
} else {
std::string body;
content_reader([&](const char *data, size_t data_length) {
body.append(data, data_length);
return true;
});
}
});
auto t = std::thread([&]() { svr.listen("localhost", 8080); });
while (!svr.is_running()) {
std::this_thread::sleep_for(std::chrono::milliseconds(1));
}
std::this_thread::sleep_for(std::chrono::seconds(1));
{
std::string data(1024 * 1024 * 2, '&');
std::stringstream buffer;
buffer << data;
Client cli("https://localhost:8080");
cli.enable_server_certificate_verification(false);
MultipartFormDataItems items{
{"document", buffer.str(), "2MB_data", "application/octet-stream"},
{"hello", "world", "", ""},
};
auto res = cli.Put("/put_customboundary", {}, items, "abc-abc_");
ASSERT_TRUE(res);
ASSERT_EQ(200, res->status);
}
svr.stop();
t.join();
}
TEST(MultipartFormDataTest, PutInvalidBoundaryChars) {
std::this_thread::sleep_for(std::chrono::seconds(1));
std::string data(1024 * 1024 * 2, '&');
std::stringstream buffer;
buffer << data;
Client cli("https://localhost:8080");
cli.enable_server_certificate_verification(false);
MultipartFormDataItems items{
{"document", buffer.str(), "2MB_data", "application/octet-stream"},
{"hello", "world", "", ""},
};
for (const char& c: " \t\r\n") {
auto res = cli.Put("/put", {}, items, string("abc123").append(1, c));
ASSERT_EQ(Error::UnsupportedMultipartBoundaryChars, res.error());
ASSERT_FALSE(res);
}
}
#endif
#ifndef _WIN32
@@ -5131,4 +5367,66 @@ TEST_F(UnixSocketTest, abstract) {
t.join();
}
#endif
TEST(SocketStream, is_writable_UNIX) {
int fd[2];
ASSERT_EQ(0, socketpair(AF_UNIX, SOCK_STREAM, 0, fd));
const auto asSocketStream =
[&] (socket_t fd, std::function<bool(Stream &)> func) {
return detail::process_client_socket(fd, 0, 0, 0, 0, func);
};
asSocketStream(fd[0], [&] (Stream &s0) {
EXPECT_EQ(s0.socket(), fd[0]);
EXPECT_TRUE(s0.is_writable());
EXPECT_EQ(0, close(fd[1]));
EXPECT_FALSE(s0.is_writable());
return true;
});
EXPECT_EQ(0, close(fd[0]));
}
TEST(SocketStream, is_writable_INET) {
sockaddr_in addr;
memset(&addr, 0, sizeof(addr));
addr.sin_family = AF_INET;
addr.sin_port = htons(PORT+1);
addr.sin_addr.s_addr = htonl(INADDR_LOOPBACK);
int disconnected_svr_sock = -1;
std::thread svr {[&] {
const int s = socket(AF_INET, SOCK_STREAM, 0);
ASSERT_LE(0, s);
ASSERT_EQ(0, ::bind(s, reinterpret_cast<sockaddr*>(&addr), sizeof(addr)));
ASSERT_EQ(0, listen(s, 1));
ASSERT_LE(0, disconnected_svr_sock = accept(s, nullptr, nullptr));
ASSERT_EQ(0, close(s));
}};
std::this_thread::sleep_for(std::chrono::milliseconds(100));
std::thread cli {[&] {
const int s = socket(AF_INET, SOCK_STREAM, 0);
ASSERT_LE(0, s);
ASSERT_EQ(0, connect(s, reinterpret_cast<sockaddr*>(&addr), sizeof(addr)));
ASSERT_EQ(0, close(s));
}};
cli.join();
svr.join();
ASSERT_NE(disconnected_svr_sock, -1);
const auto asSocketStream =
[&] (socket_t fd, std::function<bool(Stream &)> func) {
return detail::process_client_socket(fd, 0, 0, 0, 0, func);
};
asSocketStream(disconnected_svr_sock, [&] (Stream &ss) {
EXPECT_EQ(ss.socket(), disconnected_svr_sock);
EXPECT_FALSE(ss.is_writable());
return true;
});
ASSERT_EQ(0, close(disconnected_svr_sock));
}
#endif // #ifndef _WIN32