Code cleanup and format

In case we want to send a lot of data,
and the receiver is slower than the sender.

This will first fill up the receivers queues and after this
eventually also the senders queues,
until the socket is temporarily unable to accept more data to send.

select_write is done with an timeout of zero,
which makes the select call used always return immediately:
(see http://man7.org/linux/man-pages/man2/select.2.html)

This means that every marginal unavailability will make it return false
for is_writable and therefore httplib will immediately abort the transfer.

Therefore make this values configurable in the same way
as the read timeout already is.

Set the default write timeout to 5 seconds,
the same default value used for the read timeout.

README.md

@@ -181,6 +181,7 @@ svr.Get("/stream", [&](const Request &req, Response &res) {
     [data](size_t offset, size_t length, DataSink &sink) {
       const auto &d = *data;
       sink.write(&d[offset], std::min(length, DATA_CHUNK_SIZE));
+      return true; // return 'false' if you want to cancel the process.
     },
     [data] { delete data; });
 });
@@ -192,10 +193,11 @@ svr.Get("/stream", [&](const Request &req, Response &res) {
 svr.Get("/chunked", [&](const Request& req, Response& res) {
   res.set_chunked_content_provider(
     [](size_t offset, DataSink &sink) {
-       sink.write("123", 3);
-       sink.write("345", 3);
-       sink.write("789", 3);
-       sink.done();
+      sink.write("123", 3);
+      sink.write("345", 3);
+      sink.write("789", 3);
+      sink.done();
+      return true; // return 'false' if you want to cancel the process.
     }
   );
 });
@@ -363,6 +365,35 @@ res = cli.Options("/resource/foo");
 ```c++
 cli.set_timeout_sec(5); // timeouts in 5 seconds
 ```
+### Receive content with Content receiver
+
+```cpp
+std::string body;
+auto res = cli.Get(
+  "/stream", Headers(),
+  [&](const Response &response) {
+    EXPECT_EQ(200, response.status);
+    return true; // return 'false' if you want to cancel the request.
+  },
+  [&](const char *data, size_t data_length) {
+    body.append(data, data_length);
+    return true; // return 'false' if you want to cancel the request.
+  });
+```
+
+### Send content with Content provider
+
+```cpp
+std::string body = ...;
+auto res = cli_.Post(
+  "/stream", body.size(),
+  [](size_t offset, size_t length, DataSink &sink) {
+    sink.write(body.data() + offset, length);
+    return true; // return 'false' if you want to cancel the request.
+  },
+  "text/plain");
+```
+
 ### With Progress Callback
 
 ```cpp
@@ -494,7 +525,7 @@ cli.enable_server_certificate_verification(true);
 Zlib Support
 ------------
 
-'gzip' compression is available with `CPPHTTPLIB_ZLIB_SUPPORT`.
+'gzip' compression is available with `CPPHTTPLIB_ZLIB_SUPPORT`. `libz` should be linked.
 
 The server applies gzip compression to the following MIME type contents:
 

example/simplecli.cc

@@ -14,13 +14,12 @@ using namespace std;
 
 int main(void) {
 #ifdef CPPHTTPLIB_OPENSSL_SUPPORT
-  httplib::url::Options options;
-  options.ca_cert_file_path = CA_CERT_FILE;
-  // options.server_certificate_verification = true;
-
-  auto res = httplib::url::Get("https://localhost:8080/hi", options);
+  auto res = httplib::Client2("https://localhost:8080")
+    .set_ca_cert_path(CA_CERT_FILE)
+    // .enable_server_certificate_verification(true)
+    .Get("/hi");
 #else
-  auto res = httplib::url::Get("http://localhost:8080/hi");
+  auto res = httplib::Client2("http://localhost:8080").Get("/hi");
 #endif
 
   if (res) {

example/sse.cc

@@ -80,15 +80,19 @@ int main(void) {
   svr.Get("/event1", [&](const Request & /*req*/, Response &res) {
     cout << "connected to event1..." << endl;
     res.set_header("Content-Type", "text/event-stream");
-    res.set_chunked_content_provider(
-        [&](uint64_t /*offset*/, DataSink &sink) { ed.wait_event(&sink); });
+    res.set_chunked_content_provider([&](size_t /*offset*/, DataSink &sink) {
+      ed.wait_event(&sink);
+      return true;
+    });
   });
 
   svr.Get("/event2", [&](const Request & /*req*/, Response &res) {
     cout << "connected to event2..." << endl;
     res.set_header("Content-Type", "text/event-stream");
-    res.set_chunked_content_provider(
-        [&](uint64_t /*offset*/, DataSink &sink) { ed.wait_event(&sink); });
+    res.set_chunked_content_provider([&](size_t /*offset*/, DataSink &sink) {
+      ed.wait_event(&sink);
+      return true;
+    });
   });
 
   thread t([&] {

test/test.cc

@@ -334,7 +334,7 @@ TEST(RangeTest, FromHTTPBin) {
     httplib::Headers headers;
     auto res = cli.Get("/range/32", headers);
     ASSERT_TRUE(res != nullptr);
-    EXPECT_EQ(res->body, "abcdefghijklmnopqrstuvwxyzabcdef");
+    EXPECT_EQ("abcdefghijklmnopqrstuvwxyzabcdef", res->body);
     EXPECT_EQ(200, res->status);
   }
 
@@ -342,7 +342,7 @@ TEST(RangeTest, FromHTTPBin) {
     httplib::Headers headers = {httplib::make_range_header({{1, -1}})};
     auto res = cli.Get("/range/32", headers);
     ASSERT_TRUE(res != nullptr);
-    EXPECT_EQ(res->body, "bcdefghijklmnopqrstuvwxyzabcdef");
+    EXPECT_EQ("bcdefghijklmnopqrstuvwxyzabcdef", res->body);
     EXPECT_EQ(206, res->status);
   }
 
@@ -350,7 +350,7 @@ TEST(RangeTest, FromHTTPBin) {
     httplib::Headers headers = {httplib::make_range_header({{1, 10}})};
     auto res = cli.Get("/range/32", headers);
     ASSERT_TRUE(res != nullptr);
-    EXPECT_EQ(res->body, "bcdefghijk");
+    EXPECT_EQ("bcdefghijk", res->body);
     EXPECT_EQ(206, res->status);
   }
 
@@ -358,7 +358,7 @@ TEST(RangeTest, FromHTTPBin) {
     httplib::Headers headers = {httplib::make_range_header({{0, 31}})};
     auto res = cli.Get("/range/32", headers);
     ASSERT_TRUE(res != nullptr);
-    EXPECT_EQ(res->body, "abcdefghijklmnopqrstuvwxyzabcdef");
+    EXPECT_EQ("abcdefghijklmnopqrstuvwxyzabcdef", res->body);
     EXPECT_EQ(200, res->status);
   }
 
@@ -366,7 +366,7 @@ TEST(RangeTest, FromHTTPBin) {
     httplib::Headers headers = {httplib::make_range_header({{0, -1}})};
     auto res = cli.Get("/range/32", headers);
     ASSERT_TRUE(res != nullptr);
-    EXPECT_EQ(res->body, "abcdefghijklmnopqrstuvwxyzabcdef");
+    EXPECT_EQ("abcdefghijklmnopqrstuvwxyzabcdef", res->body);
     EXPECT_EQ(200, res->status);
   }
 
@@ -440,7 +440,7 @@ TEST(CancelTest, NoCancel) {
 
   auto res = cli.Get("/range/32", [](uint64_t, uint64_t) { return true; });
   ASSERT_TRUE(res != nullptr);
-  EXPECT_EQ(res->body, "abcdefghijklmnopqrstuvwxyzabcdef");
+  EXPECT_EQ("abcdefghijklmnopqrstuvwxyzabcdef", res->body);
   EXPECT_EQ(200, res->status);
 }
 
@@ -501,8 +501,8 @@ TEST(BaseAuthTest, FromHTTPWatch) {
         cli.Get("/basic-auth/hello/world",
                 {httplib::make_basic_authentication_header("hello", "world")});
     ASSERT_TRUE(res != nullptr);
-    EXPECT_EQ(res->body,
-              "{\n  \"authenticated\": true, \n  \"user\": \"hello\"\n}\n");
+    EXPECT_EQ("{\n  \"authenticated\": true, \n  \"user\": \"hello\"\n}\n",
+              res->body);
     EXPECT_EQ(200, res->status);
   }
 
@@ -510,8 +510,8 @@ TEST(BaseAuthTest, FromHTTPWatch) {
     cli.set_basic_auth("hello", "world");
     auto res = cli.Get("/basic-auth/hello/world");
     ASSERT_TRUE(res != nullptr);
-    EXPECT_EQ(res->body,
-              "{\n  \"authenticated\": true, \n  \"user\": \"hello\"\n}\n");
+    EXPECT_EQ("{\n  \"authenticated\": true, \n  \"user\": \"hello\"\n}\n",
+              res->body);
     EXPECT_EQ(200, res->status);
   }
 
@@ -554,8 +554,8 @@ TEST(DigestAuthTest, FromHTTPWatch) {
     for (auto path : paths) {
       auto res = cli.Get(path.c_str());
       ASSERT_TRUE(res != nullptr);
-      EXPECT_EQ(res->body,
-                "{\n  \"authenticated\": true, \n  \"user\": \"hello\"\n}\n");
+      EXPECT_EQ("{\n  \"authenticated\": true, \n  \"user\": \"hello\"\n}\n",
+                res->body);
       EXPECT_EQ(200, res->status);
     }
 
@@ -651,19 +651,6 @@ TEST(YahooRedirectTest, Redirect) {
   EXPECT_EQ(200, res->status);
 }
 
-TEST(YahooRedirectTestWithURL, Redirect) {
-  auto res = httplib::url::Get("http://yahoo.com");
-  ASSERT_TRUE(res != nullptr);
-  EXPECT_EQ(301, res->status);
-
-  httplib::url::Options options;
-  options.follow_location = true;
-
-  res = httplib::url::Get("http://yahoo.com", options);
-  ASSERT_TRUE(res != nullptr);
-  EXPECT_EQ(200, res->status);
-}
-
 TEST(HttpsToHttpRedirectTest, Redirect) {
   httplib::SSLClient cli("httpbin.org");
   cli.set_follow_location(true);
@@ -673,19 +660,6 @@ TEST(HttpsToHttpRedirectTest, Redirect) {
   EXPECT_EQ(200, res->status);
 }
 
-TEST(HttpsToHttpRedirectTestWithURL, Redirect) {
-  httplib::url::Options options;
-  options.follow_location = true;
-
-  auto res = httplib::url::Get(
-      "https://httpbin.org/"
-      "redirect-to?url=http%3A%2F%2Fwww.google.com&status_code=302",
-      options);
-
-  ASSERT_TRUE(res != nullptr);
-  EXPECT_EQ(200, res->status);
-}
-
 TEST(RedirectToDifferentPort, Redirect) {
   Server svr8080;
   Server svr8081;
@@ -715,7 +689,7 @@ TEST(RedirectToDifferentPort, Redirect) {
   auto res = cli.Get("/1");
   ASSERT_TRUE(res != nullptr);
   EXPECT_EQ(200, res->status);
-  EXPECT_EQ(res->body, "Hello World!");
+  EXPECT_EQ("Hello World!", res->body);
 
   svr8080.stop();
   svr8081.stop();
@@ -744,7 +718,7 @@ TEST(Server, BindDualStack) {
     auto res = cli.Get("/1");
     ASSERT_TRUE(res != nullptr);
     EXPECT_EQ(200, res->status);
-    EXPECT_EQ(res->body, "Hello World!");
+    EXPECT_EQ("Hello World!", res->body);
   }
   {
     Client cli("::1", PORT);
@@ -752,7 +726,7 @@ TEST(Server, BindDualStack) {
     auto res = cli.Get("/1");
     ASSERT_TRUE(res != nullptr);
     EXPECT_EQ(200, res->status);
-    EXPECT_EQ(res->body, "Hello World!");
+    EXPECT_EQ("Hello World!", res->body);
   }
   svr.stop();
   thread.join();
@@ -833,6 +807,11 @@ protected:
                std::this_thread::sleep_for(std::chrono::seconds(2));
                res.set_content("slow", "text/plain");
              })
+        .Post("/slowpost",
+              [&](const Request & /*req*/, Response &res) {
+                std::this_thread::sleep_for(std::chrono::seconds(2));
+                res.set_content("slow", "text/plain");
+              })
         .Get("/remote_addr",
              [&](const Request &req, Response &res) {
                auto remote_addr = req.headers.find("REMOTE_ADDR")->second;
@@ -899,35 +878,38 @@ protected:
         .Get("/streamed-chunked",
              [&](const Request & /*req*/, Response &res) {
                res.set_chunked_content_provider(
-                   [](uint64_t /*offset*/, DataSink &sink) {
-                     ASSERT_TRUE(sink.is_writable());
-                     sink.write("123", 3);
-                     sink.write("456", 3);
-                     sink.write("789", 3);
+                   [](size_t /*offset*/, DataSink &sink) {
+                     EXPECT_TRUE(sink.is_writable());
+                     sink.os << "123";
+                     sink.os << "456";
+                     sink.os << "789";
                      sink.done();
+                     return true;
                    });
              })
         .Get("/streamed-chunked2",
              [&](const Request & /*req*/, Response &res) {
                auto i = new int(0);
                res.set_chunked_content_provider(
-                   [i](uint64_t /*offset*/, DataSink &sink) {
-                     ASSERT_TRUE(sink.is_writable());
+                   [i](size_t /*offset*/, DataSink &sink) {
+                     EXPECT_TRUE(sink.is_writable());
                      switch (*i) {
-                     case 0: sink.write("123", 3); break;
-                     case 1: sink.write("456", 3); break;
-                     case 2: sink.write("789", 3); break;
+                     case 0: sink.os << "123"; break;
+                     case 1: sink.os << "456"; break;
+                     case 2: sink.os << "789"; break;
                      case 3: sink.done(); break;
                      }
                      (*i)++;
+                     return true;
                    },
                    [i] { delete i; });
              })
         .Get("/streamed",
              [&](const Request & /*req*/, Response &res) {
                res.set_content_provider(
-                   6, [](uint64_t offset, uint64_t /*length*/, DataSink &sink) {
-                     sink.write(offset < 3 ? "a" : "b", 1);
+                   6, [](size_t offset, size_t /*length*/, DataSink &sink) {
+                     sink.os << (offset < 3 ? "a" : "b");
+                     return true;
                    });
              })
         .Get("/streamed-with-range",
@@ -935,25 +917,26 @@ protected:
                auto data = new std::string("abcdefg");
                res.set_content_provider(
                    data->size(),
-                   [data](uint64_t offset, uint64_t length, DataSink &sink) {
-                     ASSERT_TRUE(sink.is_writable());
+                   [data](size_t offset, size_t length, DataSink &sink) {
+                     EXPECT_TRUE(sink.is_writable());
                      size_t DATA_CHUNK_SIZE = 4;
                      const auto &d = *data;
                      auto out_len =
                          std::min(static_cast<size_t>(length), DATA_CHUNK_SIZE);
                      sink.write(&d[static_cast<size_t>(offset)], out_len);
+                     return true;
                    },
                    [data] { delete data; });
              })
         .Get("/streamed-cancel",
              [&](const Request & /*req*/, Response &res) {
-               res.set_content_provider(size_t(-1), [](uint64_t /*offset*/,
-                                                       uint64_t /*length*/,
-                                                       DataSink &sink) {
-                 ASSERT_TRUE(sink.is_writable());
-                 std::string data = "data_chunk";
-                 sink.write(data.data(), data.size());
-               });
+               res.set_content_provider(
+                   size_t(-1),
+                   [](size_t /*offset*/, size_t /*length*/, DataSink &sink) {
+                     EXPECT_TRUE(sink.is_writable());
+                     sink.os << "data_chunk";
+                     return true;
+                   });
              })
         .Get("/with-range",
              [&](const Request & /*req*/, Response &res) {
@@ -1907,6 +1890,33 @@ TEST_F(ServerTest, SlowRequest) {
   std::this_thread::sleep_for(std::chrono::milliseconds(100));
 }
 
+TEST_F(ServerTest, SlowPost) {
+  char buffer[64 * 1024];
+  memset(buffer, 0x42, sizeof(buffer));
+
+  auto res = cli_.Post(
+      "/slowpost", 64 * 1024 * 1024,
+      [&](size_t /*offset*/, size_t /*length*/, DataSink &sink) {
+        sink.write(buffer, sizeof(buffer));
+        return true;
+      },
+      "text/plain");
+
+  ASSERT_TRUE(res != nullptr);
+  EXPECT_EQ(200, res->status);
+
+  cli_.set_write_timeout(0, 0);
+  res = cli_.Post(
+      "/slowpost", 64 * 1024 * 1024,
+      [&](size_t /*offset*/, size_t /*length*/, DataSink &sink) {
+        sink.write(buffer, sizeof(buffer));
+        return true;
+      },
+      "text/plain");
+
+  ASSERT_FALSE(res != nullptr);
+}
+
 TEST_F(ServerTest, Put) {
   auto res = cli_.Put("/put", "PUT", "text/plain");
   ASSERT_TRUE(res != nullptr);
@@ -1918,8 +1928,9 @@ TEST_F(ServerTest, PutWithContentProvider) {
   auto res = cli_.Put(
       "/put", 3,
       [](size_t /*offset*/, size_t /*length*/, DataSink &sink) {
-        ASSERT_TRUE(sink.is_writable());
-        sink.write("PUT", 3);
+        EXPECT_TRUE(sink.is_writable());
+        sink.os << "PUT";
+        return true;
       },
       "text/plain");
 
@@ -1928,14 +1939,26 @@ TEST_F(ServerTest, PutWithContentProvider) {
   EXPECT_EQ("PUT", res->body);
 }
 
+TEST_F(ServerTest, PostWithContentProviderAbort) {
+  auto res = cli_.Post(
+      "/post", 42,
+      [](size_t /*offset*/, size_t /*length*/, DataSink & /*sink*/) {
+        return false;
+      },
+      "text/plain");
+
+  ASSERT_TRUE(res == nullptr);
+}
+
 #ifdef CPPHTTPLIB_ZLIB_SUPPORT
 TEST_F(ServerTest, PutWithContentProviderWithGzip) {
   cli_.set_compress(true);
   auto res = cli_.Put(
       "/put", 3,
       [](size_t /*offset*/, size_t /*length*/, DataSink &sink) {
-        ASSERT_TRUE(sink.is_writable());
-        sink.write("PUT", 3);
+        EXPECT_TRUE(sink.is_writable());
+        sink.os << "PUT";
+        return true;
       },
       "text/plain");
 
@@ -1944,6 +1967,18 @@ TEST_F(ServerTest, PutWithContentProviderWithGzip) {
   EXPECT_EQ("PUT", res->body);
 }
 
+TEST_F(ServerTest, PostWithContentProviderWithGzipAbort) {
+  cli_.set_compress(true);
+  auto res = cli_.Post(
+      "/post", 42,
+      [](size_t /*offset*/, size_t /*length*/, DataSink & /*sink*/) {
+        return false;
+      },
+      "text/plain");
+
+  ASSERT_TRUE(res == nullptr);
+}
+
 TEST_F(ServerTest, PutLargeFileWithGzip) {
   cli_.set_compress(true);
   auto res = cli_.Put("/put-large", LARGE_DATA, "text/plain");
@@ -2091,8 +2126,8 @@ TEST_F(ServerTest, KeepAlive) {
   Get(requests, "/not-exist");
   Post(requests, "/empty", "", "text/plain");
   Post(
-      requests, "/empty", 0, [&](size_t, size_t, httplib::DataSink &) {},
-      "text/plain");
+      requests, "/empty", 0,
+      [&](size_t, size_t, httplib::DataSink &) { return true; }, "text/plain");
 
   std::vector<Response> responses;
   auto ret = cli_.send(requests, responses);
@@ -2250,7 +2285,7 @@ static bool send_request(time_t read_timeout_sec, const std::string &req,
   if (client_sock == INVALID_SOCKET) { return false; }
 
   return detail::process_and_close_socket(
-      true, client_sock, 1, read_timeout_sec, 0,
+      true, client_sock, 1, read_timeout_sec, 0, 0, 0,
       [&](Stream &strm, bool /*last_connection*/, bool &
           /*connection_close*/) -> bool {
         if (req.size() !=
@@ -2435,11 +2470,12 @@ TEST(ServerStopTest, StopServerWithChunkedTransmission) {
   svr.Get("/events", [](const Request & /*req*/, Response &res) {
     res.set_header("Content-Type", "text/event-stream");
     res.set_header("Cache-Control", "no-cache");
-    res.set_chunked_content_provider([](size_t offset, const DataSink &sink) {
+    res.set_chunked_content_provider([](size_t offset, DataSink &sink) {
       char buffer[27];
       auto size = static_cast<size_t>(sprintf(buffer, "data:%ld\n\n", offset));
       sink.write(buffer, size);
       std::this_thread::sleep_for(std::chrono::seconds(1));
+      return true;
     });
   });
 
@@ -2859,13 +2895,6 @@ TEST(SSLClientServerTest, TrustDirOptional) {
 
   t.join();
 }
-
-/* Cannot test this case as there is no external access to SSL object to check
-SSL_get_peer_certificate() == NULL TEST(SSLClientServerTest,
-ClientCAPathRequired) { SSLServer svr(SERVER_CERT_FILE, SERVER_PRIVATE_KEY_FILE,
-nullptr, CLIENT_CA_CERT_DIR);
-}
-*/
 #endif
 
 #ifdef _WIN32
@@ -2874,3 +2903,51 @@ TEST(CleanupTest, WSACleanup) {
   ASSERT_EQ(0, ret);
 }
 #endif
+
+#ifdef CPPHTTPLIB_OPENSSL_SUPPORT
+TEST(InvalidScheme, SimpleInterface) {
+  httplib::Client2 cli("scheme://yahoo.com");
+  ASSERT_FALSE(cli.is_valid());
+}
+
+TEST(NoScheme, SimpleInterface) {
+  httplib::Client2 cli("yahoo.com");
+  ASSERT_FALSE(cli.is_valid());
+}
+
+TEST(YahooRedirectTest2, SimpleInterface) {
+  httplib::Client2 cli("http://yahoo.com");
+
+  auto res = cli.Get("/");
+  ASSERT_TRUE(res != nullptr);
+  EXPECT_EQ(301, res->status);
+
+  cli.set_follow_location(true);
+  res = cli.Get("/");
+  ASSERT_TRUE(res != nullptr);
+  EXPECT_EQ(200, res->status);
+}
+
+TEST(YahooRedirectTest3, SimpleInterface) {
+  httplib::Client2 cli("https://yahoo.com");
+
+  auto res = cli.Get("/");
+  ASSERT_TRUE(res != nullptr);
+  EXPECT_EQ(301, res->status);
+
+  cli.set_follow_location(true);
+  res = cli.Get("/");
+  ASSERT_TRUE(res != nullptr);
+  EXPECT_EQ(200, res->status);
+}
+
+TEST(HttpsToHttpRedirectTest2, SimpleInterface) {
+  auto res =
+      httplib::Client2("https://httpbin.org")
+          .set_follow_location(true)
+          .Get("/redirect-to?url=http%3A%2F%2Fwww.google.com&status_code=302");
+
+  ASSERT_TRUE(res != nullptr);
+  EXPECT_EQ(200, res->status);
+}
+#endif

test/test_proxy.cc

@@ -185,15 +185,17 @@ void DigestAuthTestFromHTTPWatch(Client& cli) {
     for (auto path : paths) {
       auto res = cli.Get(path.c_str());
       ASSERT_TRUE(res != nullptr);
-      EXPECT_EQ(400, res->status);
+      EXPECT_EQ(401, res->status);
     }
 
-    cli.set_digest_auth("bad", "world");
-    for (auto path : paths) {
-      auto res = cli.Get(path.c_str());
-      ASSERT_TRUE(res != nullptr);
-      EXPECT_EQ(400, res->status);
-    }
+    // NOTE: Until httpbin.org fixes issue #46, the following test is commented
+    // out. Plese see https://httpbin.org/digest-auth/auth/hello/world
+    // cli.set_digest_auth("bad", "world");
+    // for (auto path : paths) {
+    //   auto res = cli.Get(path.c_str());
+    //   ASSERT_TRUE(res != nullptr);
+    //   EXPECT_EQ(401, res->status);
+    // }
   }
 }
 
@@ -266,7 +268,7 @@ void KeepAliveTest(Client& cli, bool basic) {
 
 
   {
-    int count = paths.size();
+    int count = static_cast<int>(paths.size());
     while (count--) {
       auto &res = responses[i++];
       EXPECT_EQ("{\n  \"authenticated\": true, \n  \"user\": \"hello\"\n}\n", res.body);