From 5dbcdf148440e9895383724dbda4fb4036db5ceb Mon Sep 17 00:00:00 2001
From: Daniel Lemire <lemire@gmail.com>
Date: Sun, 21 Jun 2020 17:52:30 -0400
Subject: [PATCH] Ok

---
 include/simdjson/implementation.h          | 18 ++++++++++++++
 include/simdjson/simdjson.h                |  1 -
 src/arm64/dom_parser_implementation.cpp    |  3 +++
 src/implementation.cpp                     |  3 +++
 src/westmere/dom_parser_implementation.cpp |  3 +++
 tests/basictests.cpp                       | 28 +++++++++++++++++++++-
 6 files changed, 54 insertions(+), 2 deletions(-)

diff --git a/include/simdjson/implementation.h b/include/simdjson/implementation.h
index 80dad2491..f4146fcf1 100644
--- a/include/simdjson/implementation.h
+++ b/include/simdjson/implementation.h
@@ -10,6 +10,24 @@
 
 namespace simdjson {
 
+/**
+ * Validate the UTF-8 string.
+ *
+ * @param buf the string to validate.
+ * @param len the length of the string in bytes.
+ * @return true if the string is valid UTF-8.
+ */
+WARN_UNUSED bool validate_utf8(const char * buf, size_t length) noexcept;
+
+
+/**
+ * Validate the UTF-8 string.
+ *
+ * @param p the string_view to validate.
+ * @return true if the string is valid UTF-8.
+ */
+WARN_UNUSED bool validate_utf8(std::string_view& p) noexcept;
+
 namespace dom {
   class document;
 } // namespace dom
diff --git a/include/simdjson/simdjson.h b/include/simdjson/simdjson.h
index 1c5cc55a3..1e348e13d 100644
--- a/include/simdjson/simdjson.h
+++ b/include/simdjson/simdjson.h
@@ -7,5 +7,4 @@
 
 #include "simdjson/compiler_check.h"
 #include "simdjson/error.h"
-
 #endif // SIMDJSON_H
diff --git a/src/arm64/dom_parser_implementation.cpp b/src/arm64/dom_parser_implementation.cpp
index f9dfa3922..f187beb06 100644
--- a/src/arm64/dom_parser_implementation.cpp
+++ b/src/arm64/dom_parser_implementation.cpp
@@ -106,6 +106,9 @@ WARN_UNUSED error_code dom_parser_implementation::stage1(const uint8_t *_buf, si
   return arm64::stage1::json_structural_indexer::index<64>(buf, len, *this, streaming);
 }
 #include "generic/stage1/utf8_validator.h"
+WARN_UNUSED bool implementation::validate_utf8(const char *buf, size_t len) const noexcept {
+  return simdjson::arm64::stage1::utf8_validate(buf,len);
+}
 } // namespace arm64
 } // namespace simdjson
 
diff --git a/src/implementation.cpp b/src/implementation.cpp
index 5f9c774e4..e9cd9250d 100644
--- a/src/implementation.cpp
+++ b/src/implementation.cpp
@@ -141,6 +141,9 @@ SIMDJSON_DLLIMPORTEXPORT internal::atomic_ptr<const implementation> active_imple
 WARN_UNUSED error_code minify(const char *buf, size_t len, char *dst, size_t &dst_len) noexcept {
   return active_implementation->minify((const uint8_t *)buf, len, (uint8_t *)dst, dst_len);
 }
+WARN_UNUSED bool validate_utf8(const char *buf, size_t len) noexcept {
+  return active_implementation->validate_utf8(buf, len);
+}
 
 
 } // namespace simdjson
diff --git a/src/westmere/dom_parser_implementation.cpp b/src/westmere/dom_parser_implementation.cpp
index f996277d7..13ba5f7cb 100644
--- a/src/westmere/dom_parser_implementation.cpp
+++ b/src/westmere/dom_parser_implementation.cpp
@@ -95,6 +95,9 @@ WARN_UNUSED error_code dom_parser_implementation::stage1(const uint8_t *_buf, si
   return westmere::stage1::json_structural_indexer::index<64>(_buf, _len, *this, streaming);
 }
 #include "generic/stage1/utf8_validator.h"
+WARN_UNUSED bool implementation::validate_utf8(const char *buf, size_t len) const noexcept {
+  return simdjson::westmere::stage1::utf8_validate(buf,len);
+}
 } // namespace westmere
 } // namespace simdjson
 UNTARGET_REGION
diff --git a/tests/basictests.cpp b/tests/basictests.cpp
index 99a04d409..222643857 100644
--- a/tests/basictests.cpp
+++ b/tests/basictests.cpp
@@ -1649,6 +1649,31 @@ namespace type_tests {
 }
 
 
+namespace validate_tests {
+  bool test_validate() {
+    std::cout << "Running " << __func__ << std::endl;
+    const std::string test = R"({ "foo" : 1, "bar" : [ 1, 2, 3 ], "baz": { "a": 1, "b": 2, "c": 3 } })";
+    if(!simdjson::validate_utf8(test.data(), test.size())) {
+      return false;
+    }
+    return true;
+  }
+
+  bool test_bad_validate() {
+    std::cout << "Running " << __func__ << std::endl;
+    const std::string test = "\x80\x81";
+    if(simdjson::validate_utf8(test.data(), test.size())) {
+      return false;
+    }
+    return true;
+  }
+  bool run() {
+    return test_validate() &&
+           test_bad_validate();
+  }
+}
+
+
 
 namespace minify_tests {
 
@@ -1960,7 +1985,8 @@ int main(int argc, char *argv[]) {
     printf("unsupported CPU\n");
   }
   std::cout << "Running basic tests." << std::endl;
-  if (minify_tests::run() &&
+  if (validate_tests::run() &&
+      minify_tests::run() &&
       parse_api_tests::run() &&
       dom_api_tests::run() &&
       type_tests::run() &&