Compare commits

...

8 Commits

Author SHA1 Message Date
yhirose 041122908c Fix problem with invalid range 2020-11-05 00:20:59 -05:00
yhirose 401de608df Fixed debug option problem. 2020-11-05 00:20:59 -05:00
yhirose 726c64cf10 Code format 2020-11-05 00:20:59 -05:00
Omkar Jadhav e1f781a21a [oss-fuzz] Enable msan for fuzz tests (#734)
* Disable openssl as we're not using it in fuzz tests

* Increase timeout in ClientStop test
2020-11-04 11:30:56 -05:00
Omkar Jadhav 72b81badad Fix issues reported by oss-fuzz (#729)
* Fix oss-fuzz issue #26529

* Add test for oss-fuzz issue  #26598

* Fix oss-fuzz issue  #26632

* Revert change and add new test cases
2020-11-03 11:36:02 -05:00
yhirose 17428a8fbf Fixed warning 2020-11-03 09:16:28 -05:00
Daniel Ottiger 6e1879dfae ssl-verify-host: fix verifying ip addresses containing zero's (#732)
* ssl-verify-host: fix verifying ip addresses containing zero's

If the subject alternate name contained an ip address with an zero
(like 10.42.0.1) it could not successfully verify.
It is because in c++ strings are null-terminated
and therefore strlen(name) would return a wrong result.
As I can not see why we can not trust the length returned by openssl,
lets drop this check.

* ssl-verify-host: add test case

lets try to validate against 127.0.0.1

Co-authored-by: Daniel Ottiger <daniel.ottiger@ch.schindler.com>
2020-11-02 20:27:34 -05:00
miketsts eb1d2e04bc SSL_connect and SSL_accept in non-blocking mode (#728)
SSL connection is performed in two steps:
First, a regular socket connection is established.
Then, SSL_connect/SSL_accept is called to establish SSL handshake.

If a network problem occurs during the second stage, SSL_connect on
the client may hang indefinitely.

The non-blocking mode solves this problem.

Co-authored-by: Michael Tseitlin <michael.tseitlin@concertio.com>
2020-11-02 17:05:08 -05:00
5 changed files with 260 additions and 153 deletions
+198 -135
View File
@@ -676,8 +676,14 @@ private:
const HandlersForContentReader &handlers);
bool parse_request_line(const char *s, Request &req);
void apply_ranges(const Request &req, Response &res,
std::string &content_type, std::string &boundary);
bool write_response(Stream &strm, bool close_connection, const Request &req,
Response &res);
bool write_response_with_content(Stream &strm, bool close_connection,
const Request &req, Response &res,
std::string &content_type,
std::string &boundary);
bool write_content_with_provider(Stream &strm, const Request &req,
Response &res, const std::string &boundary,
const std::string &content_type);
@@ -1140,7 +1146,7 @@ private:
#ifdef CPPHTTPLIB_OPENSSL_SUPPORT
bool is_ssl_ = false;
#endif
}; // namespace httplib
};
#ifdef CPPHTTPLIB_OPENSSL_SUPPORT
class SSLServer : public Server {
@@ -3172,8 +3178,7 @@ get_range_offset_and_length(const Request &req, size_t content_length,
}
if (r.second == -1) { r.second = slen - 1; }
return std::make_pair(r.first, r.second - r.first + 1);
return std::make_pair(r.first, static_cast<size_t>(r.second - r.first) + 1);
}
inline std::string make_content_range_header_field(size_t offset, size_t length,
@@ -3222,21 +3227,21 @@ bool process_multipart_ranges_data(const Request &req, Response &res,
return true;
}
inline std::string make_multipart_ranges_data(const Request &req, Response &res,
const std::string &boundary,
const std::string &content_type) {
std::string data;
process_multipart_ranges_data(
inline bool make_multipart_ranges_data(const Request &req, Response &res,
const std::string &boundary,
const std::string &content_type,
std::string &data) {
return process_multipart_ranges_data(
req, res, boundary, content_type,
[&](const std::string &token) { data += token; },
[&](const char *token) { data += token; },
[&](size_t offset, size_t length) {
data += res.body.substr(offset, length);
return true;
if (offset < res.body.size()) {
data += res.body.substr(offset, length);
return true;
}
return false;
});
return data;
}
inline size_t
@@ -4005,18 +4010,19 @@ inline bool Server::parse_request_line(const char *s, Request &req) {
inline bool Server::write_response(Stream &strm, bool close_connection,
const Request &req, Response &res) {
std::string content_type;
std::string boundary;
return write_response_with_content(strm, close_connection, req, res,
content_type, boundary);
}
inline bool Server::write_response_with_content(
Stream &strm, bool close_connection, const Request &req, Response &res,
std::string &content_type, std::string &boundary) {
assert(res.status != -1);
if (400 <= res.status && error_handler_) { error_handler_(req, res); }
detail::BufferStream bstrm;
// Response line
if (!bstrm.write_format("HTTP/1.1 %d %s\r\n", res.status,
detail::status_message(res.status))) {
return false;
}
// Headers
if (close_connection || req.get_header_value("Connection") == "close") {
res.set_header("Connection", "close");
@@ -4032,109 +4038,21 @@ inline bool Server::write_response(Stream &strm, bool close_connection,
res.set_header("Content-Type", "text/plain");
}
if (!res.has_header("Content-Length") && res.body.empty() &&
!res.content_length_ && !res.content_provider_) {
res.set_header("Content-Length", "0");
}
if (!res.has_header("Accept-Ranges") && req.method == "HEAD") {
res.set_header("Accept-Ranges", "bytes");
}
std::string content_type;
std::string boundary;
detail::BufferStream bstrm;
if (req.ranges.size() > 1) {
boundary = detail::make_multipart_data_boundary();
auto it = res.headers.find("Content-Type");
if (it != res.headers.end()) {
content_type = it->second;
res.headers.erase(it);
}
res.headers.emplace("Content-Type",
"multipart/byteranges; boundary=" + boundary);
}
auto type = detail::encoding_type(req, res);
if (res.body.empty()) {
if (res.content_length_ > 0) {
size_t length = 0;
if (req.ranges.empty()) {
length = res.content_length_;
} else if (req.ranges.size() == 1) {
auto offsets =
detail::get_range_offset_and_length(req, res.content_length_, 0);
auto offset = offsets.first;
length = offsets.second;
auto content_range = detail::make_content_range_header_field(
offset, length, res.content_length_);
res.set_header("Content-Range", content_range);
} else {
length = detail::get_multipart_ranges_data_length(req, res, boundary,
content_type);
}
res.set_header("Content-Length", std::to_string(length));
} else {
if (res.content_provider_) {
if (res.is_chunked_content_provider) {
res.set_header("Transfer-Encoding", "chunked");
if (type == detail::EncodingType::Gzip) {
res.set_header("Content-Encoding", "gzip");
} else if (type == detail::EncodingType::Brotli) {
res.set_header("Content-Encoding", "br");
}
}
} else {
res.set_header("Content-Length", "0");
}
}
} else {
if (req.ranges.empty()) {
;
} else if (req.ranges.size() == 1) {
auto offsets =
detail::get_range_offset_and_length(req, res.body.size(), 0);
auto offset = offsets.first;
auto length = offsets.second;
auto content_range = detail::make_content_range_header_field(
offset, length, res.body.size());
res.set_header("Content-Range", content_range);
res.body = res.body.substr(offset, length);
} else {
res.body =
detail::make_multipart_ranges_data(req, res, boundary, content_type);
}
if (type != detail::EncodingType::None) {
std::unique_ptr<detail::compressor> compressor;
if (type == detail::EncodingType::Gzip) {
#ifdef CPPHTTPLIB_ZLIB_SUPPORT
compressor = detail::make_unique<detail::gzip_compressor>();
res.set_header("Content-Encoding", "gzip");
#endif
} else if (type == detail::EncodingType::Brotli) {
#ifdef CPPHTTPLIB_BROTLI_SUPPORT
compressor = detail::make_unique<detail::brotli_compressor>();
res.set_header("Content-Encoding", "brotli");
#endif
}
if (compressor) {
std::string compressed;
if (!compressor->compress(res.body.data(), res.body.size(), true,
[&](const char *data, size_t data_len) {
compressed.append(data, data_len);
return true;
})) {
return false;
}
res.body.swap(compressed);
}
}
auto length = std::to_string(res.body.size());
res.set_header("Content-Length", length);
// Response line
if (!bstrm.write_format("HTTP/1.1 %d %s\r\n", res.status,
detail::status_message(res.status))) {
return false;
}
if (!detail::write_headers(bstrm, res, Headers())) { return false; }
@@ -4534,6 +4452,116 @@ inline bool Server::dispatch_request(Request &req, Response &res,
return false;
}
inline void Server::apply_ranges(const Request &req, Response &res,
std::string &content_type,
std::string &boundary) {
if (req.ranges.size() > 1) {
boundary = detail::make_multipart_data_boundary();
auto it = res.headers.find("Content-Type");
if (it != res.headers.end()) {
content_type = it->second;
res.headers.erase(it);
}
res.headers.emplace("Content-Type",
"multipart/byteranges; boundary=" + boundary);
}
auto type = detail::encoding_type(req, res);
if (res.body.empty()) {
if (res.content_length_ > 0) {
size_t length = 0;
if (req.ranges.empty()) {
length = res.content_length_;
} else if (req.ranges.size() == 1) {
auto offsets =
detail::get_range_offset_and_length(req, res.content_length_, 0);
auto offset = offsets.first;
length = offsets.second;
auto content_range = detail::make_content_range_header_field(
offset, length, res.content_length_);
res.set_header("Content-Range", content_range);
} else {
length = detail::get_multipart_ranges_data_length(req, res, boundary,
content_type);
}
res.set_header("Content-Length", std::to_string(length));
} else {
if (res.content_provider_) {
if (res.is_chunked_content_provider) {
res.set_header("Transfer-Encoding", "chunked");
if (type == detail::EncodingType::Gzip) {
res.set_header("Content-Encoding", "gzip");
} else if (type == detail::EncodingType::Brotli) {
res.set_header("Content-Encoding", "br");
}
}
}
}
} else {
if (req.ranges.empty()) {
;
} else if (req.ranges.size() == 1) {
auto offsets =
detail::get_range_offset_and_length(req, res.body.size(), 0);
auto offset = offsets.first;
auto length = offsets.second;
auto content_range = detail::make_content_range_header_field(
offset, length, res.body.size());
res.set_header("Content-Range", content_range);
if (offset < res.body.size()) {
res.body = res.body.substr(offset, length);
} else {
res.body.clear();
res.status = 416;
}
} else {
std::string data;
if (detail::make_multipart_ranges_data(req, res, boundary, content_type,
data)) {
res.body.swap(data);
} else {
res.body.clear();
res.status = 416;
}
}
if (type != detail::EncodingType::None) {
std::unique_ptr<detail::compressor> compressor;
std::string content_encoding;
if (type == detail::EncodingType::Gzip) {
#ifdef CPPHTTPLIB_ZLIB_SUPPORT
compressor = detail::make_unique<detail::gzip_compressor>();
content_encoding = "gzip";
#endif
} else if (type == detail::EncodingType::Brotli) {
#ifdef CPPHTTPLIB_BROTLI_SUPPORT
compressor = detail::make_unique<detail::brotli_compressor>();
content_encoding = "brotli";
#endif
}
if (compressor) {
std::string compressed;
if (compressor->compress(res.body.data(), res.body.size(), true,
[&](const char *data, size_t data_len) {
compressed.append(data, data_len);
return true;
})) {
res.body.swap(compressed);
res.set_header("Content-Encoding", content_encoding);
}
}
}
auto length = std::to_string(res.body.size());
res.set_header("Content-Length", length);
}
}
inline bool Server::dispatch_request_for_content_reader(
Request &req, Response &res, ContentReader content_reader,
const HandlersForContentReader &handlers) {
@@ -4625,7 +4653,12 @@ Server::process_request(Stream &strm, bool close_connection,
if (res.status == -1) { res.status = 404; }
}
return write_response(strm, close_connection, req, res);
std::string content_type;
std::string boundary;
apply_ranges(req, res, content_type, boundary);
return write_response_with_content(strm, close_connection, req, res,
content_type, boundary);
}
inline bool Server::is_valid() const { return true; }
@@ -5627,7 +5660,9 @@ inline SSL *ssl_new(socket_t sock, SSL_CTX *ctx, std::mutex &ctx_mutex,
}
if (ssl) {
set_nonblocking(sock, true);
auto bio = BIO_new_socket(static_cast<int>(sock), BIO_NOCLOSE);
BIO_set_nbio(bio, 1);
SSL_set_bio(ssl, bio, bio);
if (!setup(ssl) || SSL_connect_or_accept(ssl) != 1) {
@@ -5636,8 +5671,11 @@ inline SSL *ssl_new(socket_t sock, SSL_CTX *ctx, std::mutex &ctx_mutex,
std::lock_guard<std::mutex> guard(ctx_mutex);
SSL_free(ssl);
}
set_nonblocking(sock, false);
return nullptr;
}
BIO_set_nbio(bio, 0);
set_nonblocking(sock, false);
}
return ssl;
@@ -5653,6 +5691,28 @@ inline void ssl_delete(std::mutex &ctx_mutex, SSL *ssl,
SSL_free(ssl);
}
template <typename U>
bool ssl_connect_or_accept_nonblocking(socket_t sock, SSL *ssl,
U ssl_connect_or_accept,
time_t timeout_sec,
time_t timeout_usec) {
int res = 0;
while ((res = ssl_connect_or_accept(ssl)) != 1) {
auto err = SSL_get_error(ssl, res);
switch (err) {
case SSL_ERROR_WANT_READ:
if (select_read(sock, timeout_sec, timeout_usec) > 0) { continue; }
break;
case SSL_ERROR_WANT_WRITE:
if (select_write(sock, timeout_sec, timeout_usec) > 0) { continue; }
break;
default: break;
}
return false;
}
return true;
}
template <typename T>
inline bool
process_server_socket_ssl(SSL *ssl, socket_t sock, size_t keep_alive_max_count,
@@ -5766,9 +5826,7 @@ inline ssize_t SSLSocketStream::read(char *ptr, size_t size) {
return SSL_read(ssl_, ptr, static_cast<int>(size));
} else if (is_readable()) {
ret = SSL_read(ssl_, ptr, static_cast<int>(size));
if (ret >= 0) {
return ret;
}
if (ret >= 0) { return ret; }
err = SSL_get_error(ssl_, ret);
} else {
return -1;
@@ -5867,8 +5925,13 @@ inline SSLServer::~SSLServer() {
inline bool SSLServer::is_valid() const { return ctx_; }
inline bool SSLServer::process_and_close_socket(socket_t sock) {
auto ssl = detail::ssl_new(sock, ctx_, ctx_mutex_, SSL_accept,
[](SSL * /*ssl*/) { return true; });
auto ssl = detail::ssl_new(
sock, ctx_, ctx_mutex_,
[&](SSL *ssl) {
return detail::ssl_connect_or_accept_nonblocking(
sock, ssl, SSL_accept, read_timeout_sec_, read_timeout_usec_);
},
[](SSL * /*ssl*/) { return true; });
if (ssl) {
auto ret = detail::process_server_socket_ssl(
@@ -6062,7 +6125,9 @@ inline bool SSLClient::initialize_ssl(Socket &socket) {
SSL_set_verify(ssl, SSL_VERIFY_NONE, nullptr);
}
if (SSL_connect(ssl) != 1) {
if (!detail::ssl_connect_or_accept_nonblocking(
socket.sock, ssl, SSL_connect, connection_timeout_sec_,
connection_timeout_usec_)) {
error_ = Error::SSLConnection;
return false;
}
@@ -6187,17 +6252,15 @@ SSLClient::verify_host_with_subject_alt_name(X509 *server_cert) const {
auto name = (const char *)ASN1_STRING_get0_data(val->d.ia5);
auto name_len = (size_t)ASN1_STRING_length(val->d.ia5);
if (strlen(name) == name_len) {
switch (type) {
case GEN_DNS: dsn_matched = check_host_name(name, name_len); break;
switch (type) {
case GEN_DNS: dsn_matched = check_host_name(name, name_len); break;
case GEN_IPADD:
if (!memcmp(&addr6, name, addr_len) ||
!memcmp(&addr, name, addr_len)) {
ip_mached = true;
}
break;
case GEN_IPADD:
if (!memcmp(&addr6, name, addr_len) ||
!memcmp(&addr, name, addr_len)) {
ip_mached = true;
}
break;
}
}
}
+2 -1
View File
@@ -1,5 +1,5 @@
#CXX = clang++
CXXFLAGS = -ggdb -O0 -std=c++11 -DGTEST_USE_OWN_TR1_TUPLE -I.. -I. -Wall -Wextra -Wtype-limits -Wconversion #-fsanitize=address
CXXFLAGS = -g -std=c++11 -DGTEST_USE_OWN_TR1_TUPLE -I.. -I. -Wall -Wextra -Wtype-limits -Wconversion #-fsanitize=address
OPENSSL_DIR = /usr/local/opt/openssl@1.1
OPENSSL_SUPPORT = -DCPPHTTPLIB_OPENSSL_SUPPORT -I$(OPENSSL_DIR)/include -L$(OPENSSL_DIR)/lib -lssl -lcrypto
@@ -24,6 +24,7 @@ test_proxy : test_proxy.cc ../httplib.h Makefile cert.pem
cert.pem:
openssl genrsa 2048 > key.pem
openssl req -new -batch -config test.conf -key key.pem | openssl x509 -days 3650 -req -signkey key.pem > cert.pem
openssl req -x509 -config test.conf -key key.pem -sha256 -days 3650 -nodes -out cert2.pem -extensions SAN
openssl genrsa 2048 > rootCA.key.pem
openssl req -x509 -new -batch -config test.rootCA.conf -key rootCA.key.pem -days 1024 > rootCA.cert.pem
openssl genrsa 2048 > client.key.pem
+2 -1
View File
@@ -19,7 +19,8 @@ all : server_fuzzer
# Fuzz target, so that you can choose which $(LIB_FUZZING_ENGINE) to use.
server_fuzzer : server_fuzzer.cc ../../httplib.h
$(CXX) $(CXXFLAGS) -o $@ $< -Wl,-Bstatic $(OPENSSL_SUPPORT) -Wl,-Bdynamic -ldl $(ZLIB_SUPPORT) $(LIB_FUZZING_ENGINE) -pthread
# $(CXX) $(CXXFLAGS) -o $@ $< -Wl,-Bstatic $(OPENSSL_SUPPORT) -Wl,-Bdynamic -ldl $(ZLIB_SUPPORT) $(LIB_FUZZING_ENGINE) -pthread
$(CXX) $(CXXFLAGS) -o $@ $< $(ZLIB_SUPPORT) $(LIB_FUZZING_ENGINE) -pthread
zip -q -r server_fuzzer_seed_corpus.zip corpus
clean:
+55 -16
View File
@@ -7,6 +7,7 @@
#include <thread>
#define SERVER_CERT_FILE "./cert.pem"
#define SERVER_CERT2_FILE "./cert2.pem"
#define SERVER_PRIVATE_KEY_FILE "./key.pem"
#define CA_CERT_FILE "./ca-bundle.crt"
#define CLIENT_CA_CERT_FILE "./rootCA.cert.pem"
@@ -1898,9 +1899,7 @@ TEST_F(ServerTest, GetStreamedWithRange2) {
}
TEST_F(ServerTest, GetStreamedWithRangeSuffix1) {
auto res = cli_.Get("/streamed-with-range", {
{"Range", "bytes=-3"}
});
auto res = cli_.Get("/streamed-with-range", {{"Range", "bytes=-3"}});
ASSERT_TRUE(res);
EXPECT_EQ(206, res->status);
EXPECT_EQ("3", res->get_header_value("Content-Length"));
@@ -1908,11 +1907,8 @@ TEST_F(ServerTest, GetStreamedWithRangeSuffix1) {
EXPECT_EQ(std::string("efg"), res->body);
}
TEST_F(ServerTest, GetStreamedWithRangeSuffix2) {
auto res = cli_.Get("/streamed-with-range", {
{"Range", "bytes=-9999"}
});
auto res = cli_.Get("/streamed-with-range", {{"Range", "bytes=-9999"}});
ASSERT_TRUE(res);
EXPECT_EQ(206, res->status);
EXPECT_EQ("7", res->get_header_value("Content-Length"));
@@ -1920,15 +1916,23 @@ TEST_F(ServerTest, GetStreamedWithRangeSuffix2) {
EXPECT_EQ(std::string("abcdefg"), res->body);
}
TEST_F(ServerTest, GetStreamedWithRangeError) {
auto res = cli_.Get("/streamed-with-range", {
{"Range", "bytes=92233720368547758079223372036854775806-92233720368547758079223372036854775807"}
});
auto res = cli_.Get("/streamed-with-range",
{{"Range", "bytes=92233720368547758079223372036854775806-"
"92233720368547758079223372036854775807"}});
ASSERT_TRUE(res);
EXPECT_EQ(416, res->status);
}
TEST_F(ServerTest, GetRangeWithMaxLongLength) {
auto res =
cli_.Get("/with-range", {{"Range", "bytes=0-9223372036854775807"}});
EXPECT_EQ(206, res->status);
EXPECT_EQ("7", res->get_header_value("Content-Length"));
EXPECT_EQ(true, res->has_header("Content-Range"));
EXPECT_EQ(std::string("abcdefg"), res->body);
}
TEST_F(ServerTest, GetStreamedWithRangeMultipart) {
auto res =
cli_.Get("/streamed-with-range", {{make_range_header({{1, 2}, {4, 5}})}});
@@ -1964,8 +1968,7 @@ TEST_F(ServerTest, ClientStop) {
}));
}
std::this_thread::sleep_for(std::chrono::seconds(1));
std::this_thread::sleep_for(std::chrono::seconds(2));
while (cli_.is_socket_open()) {
cli_.stop();
std::this_thread::sleep_for(std::chrono::milliseconds(10));
@@ -2011,6 +2014,12 @@ TEST_F(ServerTest, GetWithRange4) {
EXPECT_EQ(std::string("fg"), res->body);
}
TEST_F(ServerTest, GetWithRangeOffsetGreaterThanContent) {
auto res = cli_.Get("/with-range", {{make_range_header({{10000, 20000}})}});
ASSERT_TRUE(res);
EXPECT_EQ(416, res->status);
}
TEST_F(ServerTest, GetWithRangeMultipart) {
auto res = cli_.Get("/with-range", {{make_range_header({{1, 2}, {4, 5}})}});
ASSERT_TRUE(res);
@@ -2020,6 +2029,13 @@ TEST_F(ServerTest, GetWithRangeMultipart) {
EXPECT_EQ(269, res->body.size());
}
TEST_F(ServerTest, GetWithRangeMultipartOffsetGreaterThanContent) {
auto res =
cli_.Get("/with-range", {{make_range_header({{-1, 2}, {10000, 30000}})}});
ASSERT_TRUE(res);
EXPECT_EQ(416, res->status);
}
TEST_F(ServerTest, GetStreamedChunked) {
auto res = cli_.Get("/streamed-chunked");
ASSERT_TRUE(res);
@@ -3037,9 +3053,7 @@ TEST(KeepAliveTest, ReadTimeoutSSL) {
res.set_content("b", "text/plain");
});
auto listen_thread = std::thread([&svr]() {
svr.listen("localhost", PORT);
});
auto listen_thread = std::thread([&svr]() { svr.listen("localhost", PORT); });
while (!svr.is_running()) {
std::this_thread::sleep_for(std::chrono::milliseconds(1));
}
@@ -3245,6 +3259,31 @@ TEST(SSLClientTest, ServerCertificateVerification3) {
ASSERT_EQ(301, res->status);
}
TEST(SSLClientTest, ServerCertificateVerification4) {
SSLServer svr(SERVER_CERT2_FILE, SERVER_PRIVATE_KEY_FILE);
ASSERT_TRUE(svr.is_valid());
svr.Get("/test", [&](const Request &, Response &res) {
res.set_content("test", "text/plain");
svr.stop();
ASSERT_TRUE(true);
});
thread t = thread([&]() { ASSERT_TRUE(svr.listen("127.0.0.1", PORT)); });
std::this_thread::sleep_for(std::chrono::milliseconds(1));
SSLClient cli("127.0.0.1", PORT);
cli.set_ca_cert_path(SERVER_CERT2_FILE);
cli.enable_server_certificate_verification(true);
cli.set_connection_timeout(30);
auto res = cli.Get("/test");
ASSERT_TRUE(res);
ASSERT_EQ(200, res->status);
t.join();
}
TEST(SSLClientTest, WildcardHostNameMatch) {
SSLClient cli("www.youtube.com");
+3
View File
@@ -16,3 +16,6 @@ emailAddress = test@email.address
[req_attributes]
challengePassword = 1234
[SAN]
subjectAltName=IP:127.0.0.1