diff --git a/CMakeLists.txt b/CMakeLists.txt index 295b5e3f4..56650edd5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,6 +1,6 @@ cmake_minimum_required(VERSION 2.8) - - +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED ON) set(CMAKE_MACOSX_RPATH OFF) if (NOT CMAKE_BUILD_TYPE) message(STATUS "No build type selected, default to Release") @@ -10,8 +10,11 @@ endif() project(simdjson) set(SIMDJSON_LIB_NAME simdjson) - +if(NOT MSVC) option(SIMDJSON_BUILD_STATIC "Build a static library" OFF) # turning it on disables the production of a dynamic library +else() +option(SIMDJSON_BUILD_STATIC "Build a static library" ON) # turning it on disables the production of a dynamic library +endif() option(SIMDJSON_BUILD_LTO "Build library with Link Time Optimization" OFF) option(SIMDJSON_SANITIZE "Sanitize addresses" OFF) diff --git a/README.md b/README.md index b57d03711..157e8fa9d 100644 --- a/README.md +++ b/README.md @@ -39,15 +39,17 @@ twitter.json: ## Requirements -- Linux or macOS (currently) +- We support platforms like Linux or macOS, as well as Windows through Visual Studio 2017 or better. - A processor with AVX2 (i.e., Intel processors starting with the Haswell microarchitecture released 2013, and processors from AMD starting with the Rizen) -- A recent C++ compiler (e.g., GNU GCC or LLVM CLANG), we assume C++17 -- Bash (for benchmark scripts) and other common utilities (optional) +- A recent C++ compiler (e.g., GNU GCC or LLVM CLANG or Visual Studio 2017), we assume C++17 +- Some benchmark scripts assume bash and other common utilities, but they are optional. ## License This code is made available under the Apache License 2.0. +Under Windows, we build some tools using the windows/dirent_portable.h file (which is outside our library code): it under the liberal (business-friendly) MIT license. + ## Code example ```C @@ -80,14 +82,14 @@ of memory allocation with each new JSON document: const char * filename = ... // std::string_view p = get_corpus(filename); ParsedJson pj = build_parsed_json(p); // do the parsing -// you no longer need p at this point, can do free((void*)p.data()) +// you no longer need p at this point, can do aligned_free((void*)p.data()) if( ! pj.isValid() ) { // something went wrong } ``` -## Usage (old-school Makefile) +## Usage (old-school Makefile on platforms like Linux or macOS) Requirements: clang or gcc and make. A system like Linux or macOS is expected. @@ -112,7 +114,7 @@ To run comparative benchmarks (with other parsers): make benchmark ``` -## Usage (CMake) +## Usage (CMake on platforms like Linux or macOS) While in the project repository, do the following: @@ -136,6 +138,19 @@ make make test ``` +## Usage (CMake on Windows using Visual Studio) + + +We are assuming that you have a common Windows PC with at least Visual Studio 2017, and an x64 processor with AVX2 support (2013 Haswell or better). + +- Grab the simdjosn code from GitHub, e.g., by cloning it using [GitHub Desktop](https://desktop.github.com/). +- Install [CMake](https://cmake.org/download/). When you install it, make sure to ask that ``cmake`` be made available from the command line. +- Create a subdirectory within simdjson, such as ``VisualStudio``. +- Using a shell, go to this newly created directory. +- Type ``cmake -DCMAKE_GENERATOR_PLATFORM=x64 ..`` in the shell while in the ``VisualStudio`` repository. (Alternatively, if you want to build a DLL, you may use the command line ``cmake -DCMAKE_GENERATOR_PLATFORM=x64 -DSIMDJSON_BUILD_STATIC=OFF ..``.) +- This last command created a Visual Studio solution file in the newly created directory (e.g., ``simdjson.sln``). Open this file in Visual Studio. You should now be able to build the project and run the tests. For example, in the ``Solution Explorer`` window (available from the ``View`` menu), right-click ``ALL_BUILD`` and select ``Build``. To test the code, still in the ``Solution Explorer`` window, select ``RUN_TESTS`` and select ``Build``. + + ## Tools - `json2json mydoc.json` parses the document, constructs a model and then dumps back the result to standard output. diff --git a/benchmark/benchmark.h b/benchmark/benchmark.h index dde4a67ed..ae78ae934 100644 --- a/benchmark/benchmark.h +++ b/benchmark/benchmark.h @@ -106,7 +106,7 @@ double diff(timespec start, timespec end) { clock_gettime(CLOCK_REALTIME, &time1); \ RDTSC_START(cycles_start); \ if (test != expected) { \ - printf("not expected (%d , %d )", (int)test, (int)expected); \ + fprintf(stderr, "not expected (%d , %d )", (int)test, (int)expected); \ break; \ } \ RDTSC_STOP(cycles_final); \ diff --git a/benchmark/distinctuseridcompetition.cpp b/benchmark/distinctuseridcompetition.cpp index d1af4bcfa..cb91f8f9c 100644 --- a/benchmark/distinctuseridcompetition.cpp +++ b/benchmark/distinctuseridcompetition.cpp @@ -279,5 +279,5 @@ int main(int argc, char *argv[]) { !justdata); BEST_TIME("sasjon ", sasjon_computestats(p).size(), size, , repeat, volume, !justdata); - free((void*)p.data()); + aligned_free((void*)p.data()); } diff --git a/benchmark/minifiercompetition.cpp b/benchmark/minifiercompetition.cpp index ca86e2517..c4b921c64 100644 --- a/benchmark/minifiercompetition.cpp +++ b/benchmark/minifiercompetition.cpp @@ -137,7 +137,7 @@ int main(int argc, char *argv[]) { ParsedJson pj; bool isallocok = pj.allocateCapacity(p.size(), 1024); if(!isallocok) { - printf("failed to allocate memory\n"); + fprintf(stderr, "failed to allocate memory\n"); return EXIT_FAILURE; } BEST_TIME("simdjson orig", json_parse((const uint8_t*)buffer, p.size(), pj), true, memcpy(buffer, p.data(), p.size()), repeat, volume, !justdata); @@ -145,12 +145,12 @@ int main(int argc, char *argv[]) { ParsedJson pj2; bool isallocok2 = pj2.allocateCapacity(p.size(), 1024); if(!isallocok2) { - printf("failed to allocate memory\n"); + fprintf(stderr, "failed to allocate memory\n"); return EXIT_FAILURE; } BEST_TIME("simdjson despaced", json_parse((const uint8_t*)buffer, minisize, pj2), true, memcpy(buffer, minibuffer, p.size()), repeat, volume, !justdata); - free((void*)p.data()); + aligned_free((void*)p.data()); free(buffer); free(ast_buffer); free(minibuffer); diff --git a/benchmark/parse.cpp b/benchmark/parse.cpp index 24e2be285..e53848e36 100644 --- a/benchmark/parse.cpp +++ b/benchmark/parse.cpp @@ -1,13 +1,17 @@ #include #include +#ifndef _MSC_VER +#include +#include #include +#else +#include +#endif #include #include #include #include #include -#include -#include #include #include @@ -41,7 +45,7 @@ int main(int argc, char *argv[]) { bool jsonoutput = false; bool forceoneiteration = false; bool justdata = false; - +#ifndef _MSC_VER int c; while ((c = getopt(argc, argv, "1vdt")) != -1) @@ -64,6 +68,9 @@ int main(int argc, char *argv[]) { default: abort(); } +#else + int optind = 1; +#endif if (optind >= argc) { cerr << "Usage: " << argv[0] << " " << endl; exit(1); @@ -278,9 +285,9 @@ int main(int argc, char *argv[]) { if (dump) { isok = isok && pj.dump_raw_tape(std::cout); } - free((void *)p.data()); + aligned_free((void *)p.data()); if (!isok) { - printf(" Parsing failed. \n "); + fprintf(stderr, " Parsing failed. \n "); return EXIT_FAILURE; } return EXIT_SUCCESS; diff --git a/benchmark/parseandstatcompetition.cpp b/benchmark/parseandstatcompetition.cpp index 2c80db0b8..db401341d 100644 --- a/benchmark/parseandstatcompetition.cpp +++ b/benchmark/parseandstatcompetition.cpp @@ -300,5 +300,5 @@ int main(int argc, char *argv[]) { !justdata); BEST_TIME("sasjon ", sasjon_computestats(p).valid, true, , repeat, volume, !justdata); - free((void*)p.data()); + aligned_free((void*)p.data()); } diff --git a/benchmark/parsingcompetition.cpp b/benchmark/parsingcompetition.cpp index 7e2ef4b4f..f707d08fd 100644 --- a/benchmark/parsingcompetition.cpp +++ b/benchmark/parsingcompetition.cpp @@ -158,7 +158,7 @@ int main(int argc, char *argv[]) { if(!justdata) BEST_TIME("memcpy ", (memcpy(buffer, p.data(), p.size()) == buffer), true, , repeat, volume, !justdata); - free((void *)p.data()); + aligned_free((void *)p.data()); free(ast_buffer); free(buffer); } diff --git a/benchmark/statisticalmodel.cpp b/benchmark/statisticalmodel.cpp index 5547d0c48..1097332c4 100644 --- a/benchmark/statisticalmodel.cpp +++ b/benchmark/statisticalmodel.cpp @@ -1,6 +1,7 @@ #include +#ifndef _MSC_VER #include - +#endif #include "simdjson/jsonioutil.h" #include "simdjson/jsonparser.h" #ifdef __linux__ @@ -112,14 +113,17 @@ stat_t simdjson_computestats(const std::string_view &p) { } int main(int argc, char *argv[]) { - int c; - - while ((c = getopt(argc, argv, "")) != -1) +#ifndef _MSC_VER + int c; + while ((c = getopt(argc, argv, "")) != -1) switch (c) { default: abort(); } +#else + int optind = 1; +#endif if (optind >= argc) { cerr << "Reads json, prints stats. " << endl; cerr << "Usage: " << argv[0] << " " << endl; diff --git a/include/simdjson/common_defs.h b/include/simdjson/common_defs.h index 408dd6b56..d1e8bf845 100644 --- a/include/simdjson/common_defs.h +++ b/include/simdjson/common_defs.h @@ -1,27 +1,46 @@ -#pragma once +#ifndef SIMDJSON_COMMON_DEFS_H +#define SIMDJSON_COMMON_DEFS_H + +#include "simdjson/portability.h" #include // the input buf should be readable up to buf + SIMDJSON_PADDING -#define SIMDJSON_PADDING sizeof(__m256i) +#define SIMDJSON_PADDING sizeof(__m256i) - - - -#ifdef _MSC_VER -/* Microsoft C/C++-compatible compiler */ -#include -#else -#include +#ifndef _MSC_VER +// Implemented using Labels as Values which works in GCC and CLANG (and maybe +// also in Intel's compiler), but won't work in MSVC. +#define SIMDJSON_USE_COMPUTED_GOTO #endif + + // Align to N-byte boundary #define ROUNDUP_N(a, n) (((a) + ((n)-1)) & ~((n)-1)) #define ROUNDDOWN_N(a, n) ((a) & ~((n)-1)) #define ISALIGNED_N(ptr, n) (((uintptr_t)(ptr) & ((n)-1)) == 0) +#ifdef _MSC_VER + + +#define really_inline inline +#define never_inline inline + +#define UNUSED +#define WARN_UNUSED + +#ifndef likely +#define likely(x) x +#endif +#ifndef unlikely +#define unlikely(x) x +#endif + +#else + #define really_inline inline __attribute__((always_inline, unused)) #define never_inline inline __attribute__((noinline, unused)) @@ -35,3 +54,6 @@ #define unlikely(x) __builtin_expect(!!(x), 0) #endif +#endif // MSC_VER + +#endif // COMMON_DEFS_H diff --git a/include/simdjson/jsoncharutils.h b/include/simdjson/jsoncharutils.h index cbbe1391a..42876a40a 100644 --- a/include/simdjson/jsoncharutils.h +++ b/include/simdjson/jsoncharutils.h @@ -1,4 +1,5 @@ -#pragma once +#ifndef SIMDJSON_JSONCHARUTILS_H +#define SIMDJSON_JSONCHARUTILS_H #include "simdjson/common_defs.h" #include "simdjson/parsedjson.h" @@ -96,7 +97,7 @@ uint32_t hex_to_u32_nocheck(const uint8_t *src) { // and clz and table lookups, but JSON documents // have few escaped code points, and the following // function looks cheap. -// +// // Note: we assume that surrogates are treated separately // inline size_t codepoint_to_utf8(uint32_t cp, uint8_t *c) { @@ -126,3 +127,4 @@ inline size_t codepoint_to_utf8(uint32_t cp, uint8_t *c) { return 0; // bad r } +#endif diff --git a/include/simdjson/jsonformatutils.h b/include/simdjson/jsonformatutils.h index 4039ef0d4..6406cc3ba 100644 --- a/include/simdjson/jsonformatutils.h +++ b/include/simdjson/jsonformatutils.h @@ -1,4 +1,5 @@ -#pragma once +#ifndef SIMDJSON_JSONFORMATUTILS_H +#define SIMDJSON_JSONFORMATUTILS_H #include #include @@ -79,7 +80,7 @@ static inline void print_with_escapes(const unsigned char *src, std::ostream &os default: if (*src <= 0x1F) { std::ios::fmtflags f(os.flags()); - os << std::hex << std::setw(4) << std::setfill('0') << (int) *src; + os << std::hex << std::setw(4) << std::setfill('0') << (int) *src; os.flags(f); } else os << *src; @@ -89,5 +90,7 @@ static inline void print_with_escapes(const unsigned char *src, std::ostream &os } static inline void print_with_escapes(const char *src, std::ostream &os) { - print_with_escapes((const unsigned char *)src, os); -} \ No newline at end of file + print_with_escapes((const unsigned char *)src, os); +} + +#endif diff --git a/include/simdjson/jsonioutil.h b/include/simdjson/jsonioutil.h index da88cc48d..9e0742c6d 100644 --- a/include/simdjson/jsonioutil.h +++ b/include/simdjson/jsonioutil.h @@ -1,12 +1,11 @@ -#ifndef JSONIOUTIL_H -#define JSONIOUTIL_H +#ifndef SIMDJSON_JSONIOUTIL_H +#define SIMDJSON_JSONIOUTIL_H #include #include #include #include #include -#include #include "simdjson/common_defs.h" @@ -32,7 +31,7 @@ char * allocate_padded_buffer(size_t length); // try { // p = get_corpus(filename); // } catch (const std::exception& e) { -// free((void*)p.data()); +// free((void*)p.data());//use aligned_free if you plan to use VisualStudio // std::cout << "Could not load the file " << filename << std::endl; // } std::string_view get_corpus(std::string filename); diff --git a/include/simdjson/jsonminifier.h b/include/simdjson/jsonminifier.h index 64389f870..acc8d79f8 100644 --- a/include/simdjson/jsonminifier.h +++ b/include/simdjson/jsonminifier.h @@ -1,10 +1,11 @@ -#pragma once +#ifndef SIMDJSON_JSONMINIFIER_H +#define SIMDJSON_JSONMINIFIER_H #include #include // Take input from buf and remove useless whitespace, write it to out; buf and -// out can be the same pointer. Result is null terminated, +// out can be the same pointer. Result is null terminated, // return the string length (minus the null termination). size_t jsonminify(const uint8_t *buf, size_t len, uint8_t *out); @@ -16,4 +17,6 @@ static inline size_t jsonminify(const char *buf, size_t len, char *out) { static inline size_t jsonminify(const std::string_view & p, char *out) { return jsonminify(p.data(), p.size(), out); -} \ No newline at end of file +} + +#endif diff --git a/include/simdjson/jsonparser.h b/include/simdjson/jsonparser.h index 574142a62..27b8f5051 100644 --- a/include/simdjson/jsonparser.h +++ b/include/simdjson/jsonparser.h @@ -1,4 +1,5 @@ -#pragma once +#ifndef SIMDJSON_JSONPARSER_H +#define SIMDJSON_JSONPARSER_H #include "simdjson/common_defs.h" #include "simdjson/jsonioutil.h" @@ -11,7 +12,7 @@ // Parse a document found in buf, need to preallocate ParsedJson. -// Return false in case of a failure. You can also check validity +// Return false in case of a failure. You can also check validity // by calling pj.isValid(). The same ParsedJson can be reused for other documents. // // If reallocifneeded is true (default) then a temporary buffer is created when needed during processing @@ -22,7 +23,7 @@ WARN_UNUSED bool json_parse(const uint8_t *buf, size_t len, ParsedJson &pj, bool reallocifneeded = true); // Parse a document found in buf, need to preallocate ParsedJson. -// Return false in case of a failure. You can also check validity +// Return false in case of a failure. You can also check validity // by calling pj.isValid(). The same ParsedJson can be reused for other documents. // // If reallocifneeded is true (default) then a temporary buffer is created when needed during processing @@ -30,12 +31,12 @@ bool json_parse(const uint8_t *buf, size_t len, ParsedJson &pj, bool reallocifne // The input buf should be readable up to buf + len + SIMDJSON_PADDING if reallocifneeded is false, // all bytes at and after buf + len are ignored (can be garbage). WARN_UNUSED -static inline bool json_parse(const char * buf, size_t len, ParsedJson &pj, bool reallocifneeded = true) { +inline bool json_parse(const char * buf, size_t len, ParsedJson &pj, bool reallocifneeded = true) { return json_parse((const uint8_t *) buf, len, pj, reallocifneeded); } // Parse a document found in buf, need to preallocate ParsedJson. -// Return false in case of a failure. You can also check validity +// Return false in case of a failure. You can also check validity // by calling pj.isValid(). The same ParsedJson can be reused for other documents. // // If reallocifneeded is true (default) then a temporary buffer is created when needed during processing @@ -43,12 +44,12 @@ static inline bool json_parse(const char * buf, size_t len, ParsedJson &pj, bool // the input s should be readable up to s.data() + s.size() + SIMDJSON_PADDING if reallocifneeded is false, // all bytes at and after s.data()+s.size() are ignored (can be garbage). WARN_UNUSED -static inline bool json_parse(const std::string_view &s, ParsedJson &pj, bool reallocifneeded = true) { +inline bool json_parse(const std::string_view &s, ParsedJson &pj, bool reallocifneeded = true) { return json_parse(s.data(), s.size(), pj, reallocifneeded); } -// Build a ParsedJson object. You can check validity +// Build a ParsedJson object. You can check validity // by calling pj.isValid(). This does the memory allocation needed for ParsedJson. // If reallocifneeded is true (default) then a temporary buffer is created when needed during processing // (a copy of the input string is made). @@ -59,24 +60,26 @@ WARN_UNUSED ParsedJson build_parsed_json(const uint8_t *buf, size_t len, bool reallocifneeded = true); WARN_UNUSED -// Build a ParsedJson object. You can check validity +// Build a ParsedJson object. You can check validity // by calling pj.isValid(). This does the memory allocation needed for ParsedJson. // If reallocifneeded is true (default) then a temporary buffer is created when needed during processing // (a copy of the input string is made). // The input buf should be readable up to buf + len + SIMDJSON_PADDING if reallocifneeded is false, // all bytes at and after buf + len are ignored (can be garbage). -static inline ParsedJson build_parsed_json(const char * buf, size_t len, bool reallocifneeded = true) { +inline ParsedJson build_parsed_json(const char * buf, size_t len, bool reallocifneeded = true) { return build_parsed_json((const uint8_t *) buf, len, reallocifneeded); } // convenience function WARN_UNUSED -// Build a ParsedJson object. You can check validity +// Build a ParsedJson object. You can check validity // by calling pj.isValid(). This does the memory allocation needed for ParsedJson. // If reallocifneeded is true (default) then a temporary buffer is created when needed during processing // (a copy of the input string is made). // The input s should be readable up to s.data() + s.size() + SIMDJSON_PADDING if reallocifneeded is false, // all bytes at and after s.data()+s.size() are ignored (can be garbage). -static inline ParsedJson build_parsed_json(const std::string_view &s, bool reallocifneeded = true) { +inline ParsedJson build_parsed_json(const std::string_view &s, bool reallocifneeded = true) { return build_parsed_json(s.data(), s.size(), reallocifneeded); -} \ No newline at end of file +} + +#endif diff --git a/include/simdjson/numberparsing.h b/include/simdjson/numberparsing.h index 6ec56532a..61fb6c68b 100644 --- a/include/simdjson/numberparsing.h +++ b/include/simdjson/numberparsing.h @@ -1,5 +1,7 @@ -#pragma once +#ifndef SIMDJSON_NUMBERPARSING_H +#define SIMDJSON_NUMBERPARSING_H +#include "simdjson/portability.h" #include "simdjson/common_defs.h" #include "simdjson/jsoncharutils.h" #include "simdjson/parsedjson.h" @@ -105,10 +107,11 @@ is_not_structural_or_whitespace_or_exponent_or_decimal(unsigned char c) { #ifdef SWAR_NUMBER_PARSING +#ifdef _MSC_VER // check quickly whether the next 8 chars are made of digits // at a glance, it looks better than Mula's // http://0x80.pl/articles/swar-digits-validate.html -/*static inline bool is_made_of_eight_digits_fast(const char *chars) { +static inline bool is_made_of_eight_digits_fast(const char *chars) { uint64_t val; memcpy(&val, chars, 8); // a branchy method might be faster: @@ -118,8 +121,8 @@ is_not_structural_or_whitespace_or_exponent_or_decimal(unsigned char c) { return (((val & 0xF0F0F0F0F0F0F0F0) | (((val + 0x0606060606060606) & 0xF0F0F0F0F0F0F0F0) >> 4)) == 0x3333333333333333); -}*/ - +} +#else // this is more efficient apparently than the scalar code above (fewer instructions) static inline bool is_made_of_eight_digits_fast(const char *chars) { __m64 val; @@ -128,6 +131,7 @@ static inline bool is_made_of_eight_digits_fast(const char *chars) { __m64 basecmp = _mm_subs_pu8(base,_mm_set1_pi8(9)); return _mm_cvtm64_si64(basecmp) == 0; } +#endif static inline uint32_t parse_eight_digits_unrolled(const char *chars) { // this actually computes *16* values so we are being wasteful. @@ -159,7 +163,7 @@ static inline uint32_t parse_eight_digits_unrolled(const char *chars) { // Note: a redesign could avoid this function entirely. // static never_inline bool -parse_float(const uint8_t *const buf, +parse_float(const uint8_t *const buf, ParsedJson &pj, const uint32_t offset, bool found_minus) { const char *p = (const char *)(buf + offset); @@ -284,13 +288,13 @@ static never_inline bool parse_large_integer(const uint8_t *const buf, // we rarely see large integer parts like 123456789 while (is_integer(*p)) { digit = *p - '0'; - if (__builtin_umulll_overflow(i, 10, (unsigned long long *)&i)) { + if (mul_overflow(i, 10, &i)) { #ifdef JSON_TEST_NUMBERS // for unit testing foundInvalidNumber(buf + offset); #endif return false; // overflow } - if (__builtin_uaddll_overflow(i, digit, (unsigned long long *)&i)) { + if (add_overflow(i, digit, &i)) { #ifdef JSON_TEST_NUMBERS // for unit testing foundInvalidNumber(buf + offset); #endif @@ -324,21 +328,14 @@ static never_inline bool parse_large_integer(const uint8_t *const buf, return is_structural_or_whitespace(*p); } -#ifndef likely -#define likely(x) __builtin_expect(!!(x), 1) -#endif - -#ifndef unlikely -#define unlikely(x) __builtin_expect(!!(x), 0) -#endif // parse the number at buf + offset // define JSON_TEST_NUMBERS for unit testing -static really_inline bool parse_number(const uint8_t *const buf, - ParsedJson &pj, - const uint32_t offset, +static really_inline bool parse_number(const uint8_t *const buf, + ParsedJson &pj, + const uint32_t offset, bool found_minus) { #ifdef SIMDJSON_SKIPNUMBERPARSING // for performance analysis, it is sometimes useful to skip parsing pj.write_tape_s64(0); // always write zero @@ -459,7 +456,7 @@ static really_inline bool parse_number(const uint8_t *const buf, if (unlikely(digitcount >= 19)) { // this is uncommon!!! // this is almost never going to get called!!! // we start anew, going slowly!!! - return parse_float(buf, pj, offset, + return parse_float(buf, pj, offset, found_minus); } /////////// @@ -488,7 +485,7 @@ static really_inline bool parse_number(const uint8_t *const buf, } } else { if (unlikely(digitcount >= 18)) { // this is uncommon!!! - return parse_large_integer(buf, pj, offset, + return parse_large_integer(buf, pj, offset, found_minus); } pj.write_tape_s64(i); @@ -499,3 +496,5 @@ static really_inline bool parse_number(const uint8_t *const buf, return is_structural_or_whitespace(*p); #endif // SIMDJSON_SKIPNUMBERPARSING } + +#endif diff --git a/include/simdjson/parsedjson.h b/include/simdjson/parsedjson.h index 9ca3ab4cd..15ee69cb2 100644 --- a/include/simdjson/parsedjson.h +++ b/include/simdjson/parsedjson.h @@ -1,17 +1,13 @@ -#pragma once - #include +#ifndef SIMDJSON_PARSEDJSON_H +#define SIMDJSON_PARSEDJSON_H + +#include #include #include -#ifdef _MSC_VER -/* Microsoft C/C++-compatible compiler */ -#include -#else -#include -#endif - #include #include +#include "simdjson/portability.h" #include "simdjson/jsonformatutils.h" #define JSONVALUEMASK 0xFFFFFFFFFFFFFF @@ -49,7 +45,8 @@ public: } isvalid = false; bytecapacity = 0; // will only set it to len after allocations are a success - if (posix_memalign((void **)&structurals, 8, ROUNDUP_N(len, 64) / 8)) { + structurals = (uint8_t *)aligned_malloc(8, ROUNDUP_N(len, 64) / 8); + if (structurals == NULL) { std::cerr << "Could not allocate memory for structurals" << std::endl; return false; }; @@ -61,17 +58,20 @@ public: string_buf = new uint8_t[localstringcapacity]; tape = new uint64_t[localtapecapacity]; containing_scope_offset = new uint32_t[maxdepth]; +#ifdef SIMDJSON_USE_COMPUTED_GOTO ret_address = new void *[maxdepth]; - +#else + ret_address = new char[maxdepth]; +#endif if ((string_buf == NULL) || (tape == NULL) || (containing_scope_offset == NULL) || (ret_address == NULL) || (structural_indexes == NULL)) { std::cerr << "Could not allocate memory" << std::endl; - delete[] ret_address; - delete[] containing_scope_offset; - delete[] tape; - delete[] string_buf; - delete[] structural_indexes; - free(structurals); + if(ret_address != NULL) delete[] ret_address; + if(containing_scope_offset != NULL) delete[] containing_scope_offset; + if(tape != NULL) delete[] tape; + if(string_buf != NULL) delete[] string_buf; + if(structural_indexes != NULL) delete[] structural_indexes; + aligned_free(structurals); return false; } @@ -93,12 +93,12 @@ public: depthcapacity = 0; tapecapacity = 0; stringcapacity = 0; - delete[] ret_address; - delete[] containing_scope_offset; - delete[] tape; - delete[] string_buf; - delete[] structural_indexes; - free(structurals); + if(ret_address != NULL) delete[] ret_address; + if(containing_scope_offset != NULL) delete[] containing_scope_offset; + if(tape != NULL) delete[] tape; + if(string_buf != NULL) delete[] string_buf; + if(structural_indexes != NULL) delete[] structural_indexes; + aligned_free(structurals); isvalid = false; } @@ -124,11 +124,11 @@ public: if (type == 'r') { howmany = tape_val & JSONVALUEMASK; } else { - printf("Error: no starting root node?"); + fprintf(stderr, "Error: no starting root node?"); return false; } if (howmany > tapecapacity) { - printf( + fprintf(stderr, "We may be exceeding the tape capacity. Is this a valid document?\n"); return false; } @@ -192,22 +192,22 @@ public: os << '}'; break; case '[': // we start an array - os << '['; + os << '['; depth++; inobject[depth] = false; inobjectidx[depth] = 0; break; case ']': // we end an array depth--; - os << ']'; + os << ']'; break; case 'r': // we start and end with the root node - printf("should we be hitting the root node?\n"); + fprintf(stderr, "should we be hitting the root node?\n"); delete[] inobject; delete[] inobjectidx; return false; default: - printf("bug %c\n", type); + fprintf(stderr, "bug %c\n", type); delete[] inobject; delete[] inobjectidx; return false; @@ -224,25 +224,25 @@ public: size_t tapeidx = 0; uint64_t tape_val = tape[tapeidx]; uint8_t type = (tape_val >> 56); - os << tapeidx << " : " << type; + os << tapeidx << " : " << type; tapeidx++; size_t howmany = 0; if (type == 'r') { howmany = tape_val & JSONVALUEMASK; } else { - printf("Error: no starting root node?"); + fprintf(stderr, "Error: no starting root node?"); return false; } os << "\t// pointing to " << howmany <<" (right after last node)\n"; uint64_t payload; for (; tapeidx < howmany; tapeidx++) { - os << tapeidx << " : "; + os << tapeidx << " : "; tape_val = tape[tapeidx]; payload = tape_val & JSONVALUEMASK; type = (tape_val >> 56); switch (type) { case '"': // we have a string - os << "string \""; + os << "string \""; print_with_escapes((const unsigned char *)(string_buf + payload)); os << '"'; os << '\n'; @@ -362,9 +362,9 @@ public: delete[] depthindex; } - iterator(const iterator &o): - pj(o.pj), depth(o.depth), location(o.location), - tape_length(o.tape_length), current_type(o.current_type), + iterator(const iterator &o): + pj(o.pj), depth(o.depth), location(o.location), + tape_length(o.tape_length), current_type(o.current_type), current_val(o.current_val), depthindex(NULL) { depthindex = new scopeindex_t[pj.depthcapacity]; if(depthindex != NULL) { @@ -374,18 +374,18 @@ public: } } - iterator(iterator &&o): - pj(o.pj), depth(o.depth), location(o.location), - tape_length(o.tape_length), current_type(o.current_type), - current_val(o.current_val), depthindex(o.depthindex) { - o.depthindex = NULL;// we take ownship + iterator(iterator &&o): + pj(o.pj), depth(std::move(o.depth)), location(std::move(o.location)), + tape_length(std::move(o.tape_length)), current_type(std::move(o.current_type)), + current_val(std::move(o.current_val)), depthindex(std::move(o.depthindex)) { + o.depthindex = NULL;// we take ownership } WARN_UNUSED bool isOk() const { return location < tape_length; } - + // useful for debuging purposes size_t get_tape_location() const { return location; @@ -417,7 +417,7 @@ public: depth++; depthindex[depth].start_of_scope = location; depthindex[depth].scope_type = current_type; - } + } location = location + 1; current_val = pj.tape[location]; current_type = (current_val >> 56); @@ -456,7 +456,7 @@ public: double answer; memcpy(&answer, & pj.tape[location + 1], sizeof(answer)); return answer; - } + } bool is_object_or_array() const { return is_object_or_array(get_type()); @@ -473,15 +473,15 @@ public: bool is_string() const { return get_type() == '"'; } - + bool is_integer() const { return get_type() == 'l'; } - + bool is_double() const { return get_type() == 'd'; } - + static bool is_object_or_array(uint8_t type) { return (type == '[' || (type == '{')); } @@ -509,7 +509,7 @@ public: really_inline const char * get_string() const { return (const char *)(pj.string_buf + (current_val & JSONVALUEMASK)) ; } - + // throughout return true if we can do the navigation, false // otherwise @@ -518,10 +518,10 @@ public: // Thus, given [true, null, {"a":1}, [1,2]], we would visit true, null, { and [. // At the object ({) or at the array ([), you can issue a "down" to visit their content. // valid if we're not at the end of a scope (returns true). - really_inline bool next() { + really_inline bool next() { if ((current_type == '[') || (current_type == '{')){ // we need to jump - size_t npos = ( current_val & JSONVALUEMASK); + size_t npos = ( current_val & JSONVALUEMASK); if(npos >= tape_length) { return false; // shoud never happen unless at the root } @@ -548,14 +548,14 @@ public: return true; } } - + // Withing a given scope (series of nodes at the same depth within either an // array or an object), we move backward. // Thus, given [true, null, {"a":1}, [1,2]], we would visit ], }, null, true when starting at the end // of the scope. - // At the object ({) or at the array ([), you can issue a "down" to visit their content. + // At the object ({) or at the array ([), you can issue a "down" to visit their content. really_inline bool prev() { if(location - 1 < depthindex[depth].start_of_scope) return false; location -= 1; @@ -563,9 +563,9 @@ public: current_type = (current_val >> 56); if ((current_type == ']') || (current_type == '}')){ // we need to jump - size_t new_location = ( current_val & JSONVALUEMASK); + size_t new_location = ( current_val & JSONVALUEMASK); if(new_location < depthindex[depth].start_of_scope) { - return false; // shoud never happen + return false; // shoud never happen } location = new_location; current_val = pj.tape[location]; @@ -574,7 +574,7 @@ public: return true; } - // Moves back to either the containing array or object (type { or [) from + // Moves back to either the containing array or object (type { or [) from // within a contained scope. // Valid unless we are at the first level of the document // @@ -590,8 +590,8 @@ public: current_type = (current_val >> 56); return true; } - - + + // Valid if we're at a [ or { and it starts a non-empty scope; moves us to start of // that deeper scope if it not empty. // Thus, given [true, null, {"a":1}, [1,2]], if we are at the { node, we would move to the @@ -599,7 +599,7 @@ public: really_inline bool down() { if(location + 1 >= tape_length) return false; if ((current_type == '[') || (current_type == '{')) { - size_t npos = (current_val & JSONVALUEMASK); + size_t npos = (current_val & JSONVALUEMASK); if(npos == location + 2) { return false; // we have an empty scope } @@ -610,13 +610,13 @@ public: current_val = pj.tape[location]; current_type = (current_val >> 56); return true; - } + } return false; } // move us to the start of our current scope, // a scope is a series of nodes at the same level - void to_start_scope() { + void to_start_scope() { location = depthindex[depth].start_of_scope; current_val = pj.tape[location]; current_type = (current_val >> 56); @@ -641,7 +641,7 @@ public: case 'l': // we have a long int os << get_integer(); break; - case 'd': + case 'd': os << get_double(); break; case 'n': // we have a null @@ -674,7 +674,7 @@ private: ParsedJson &pj; size_t depth; size_t location; // our current location on a tape - size_t tape_length; + size_t tape_length; uint8_t current_type; uint64_t current_val; scopeindex_t *depthindex; @@ -696,12 +696,39 @@ private: uint64_t *tape; uint32_t *containing_scope_offset; +#ifdef SIMDJSON_USE_COMPUTED_GOTO void **ret_address; +#else + char *ret_address; +#endif uint8_t *string_buf; // should be at least bytecapacity uint8_t *current_string_buf_loc; bool isvalid; - ParsedJson(const ParsedJson && p); + + ParsedJson(ParsedJson && p) + : bytecapacity(std::move(p.bytecapacity)), + depthcapacity(std::move(p.depthcapacity)), + tapecapacity(std::move(p.tapecapacity)), + stringcapacity(std::move(p.stringcapacity)), + current_loc(std::move(p.current_loc)), + structurals(std::move(p.structurals)), + n_structural_indexes(std::move(p.n_structural_indexes)), + structural_indexes(std::move(p.structural_indexes)), + tape(std::move(p.tape)), + containing_scope_offset(std::move(p.containing_scope_offset)), + ret_address(std::move(p.ret_address)), + string_buf(std::move(p.string_buf)), + current_string_buf_loc(std::move(p.current_string_buf_loc)), + isvalid(std::move(p.isvalid)) { + p.structurals=NULL; + p.structural_indexes=NULL; + p.tape=NULL; + p.containing_scope_offset=NULL; + p.ret_address=NULL; + p.string_buf=NULL; + p.current_string_buf_loc=NULL; + } private : @@ -719,12 +746,15 @@ inline void dumpbits_always(uint64_t v, const std::string &msg) { for (uint32_t i = 0; i < 64; i++) { std::cout << (((v >> (uint64_t)i) & 0x1ULL) ? "1" : "_"); } - std::cout << " " << msg << "\n"; + std::cout << " " << msg.c_str() << "\n"; } inline void dumpbits32_always(uint32_t v, const std::string &msg) { for (uint32_t i = 0; i < 32; i++) { std::cout << (((v >> (uint32_t)i) & 0x1ULL) ? "1" : "_"); } - std::cout << " " << msg << "\n"; + std::cout << " " << msg.c_str() << "\n"; } + + +#endif diff --git a/include/simdjson/portability.h b/include/simdjson/portability.h new file mode 100644 index 000000000..77187f2ef --- /dev/null +++ b/include/simdjson/portability.h @@ -0,0 +1,122 @@ +#ifndef SIMDJSON_PORTABILITY_H +#define SIMDJSON_PORTABILITY_H + +#if defined(_MSC_VER) +#include +#else +#include +#endif + +#ifdef _MSC_VER +/* Microsoft C/C++-compatible compiler */ +#include +#include +#include + +static inline bool add_overflow(uint64_t value1, uint64_t value2, uint64_t *result) { + return _addcarry_u64(0, value1, value2, reinterpret_cast(result)); +} + +# pragma intrinsic(_umul128) +static inline bool mul_overflow(uint64_t value1, uint64_t value2, uint64_t *result) { + uint64_t high; + *result = _umul128(value1, value2, &high); + return high; +} + +static inline int trailingzeroes(uint64_t input_num) { + return _tzcnt_u64(input_num); +} + +static inline int leadingzeroes(uint64_t input_num) { + return _lzcnt_u64(input_num); +} + +static inline int hamming(uint64_t input_num) { +#ifdef _WIN64 // highly recommended!!! + return (int)__popcnt64(input_num); +#else // if we must support 32-bit Windows + return (int)(__popcnt((uint32_t)input_num) + + __popcnt((uint32_t)(input_num >> 32))); +#endif +} + +#else +#include +#include + +static inline bool add_overflow(uint64_t value1, uint64_t value2, uint64_t *result) { + return __builtin_uaddll_overflow(value1, value2, (unsigned long long*)result); +} +static inline bool mul_overflow(uint64_t value1, uint64_t value2, uint64_t *result) { + return __builtin_umulll_overflow(value1, value2, (unsigned long long *)result); +} + +/* result might be undefined when input_num is zero */ +static inline int trailingzeroes(uint64_t input_num) { + return _tzcnt_u64(input_num); +} + +/* result might be undefined when input_num is zero */ +static inline int leadingzeroes(uint64_t input_num) { + return _lzcnt_u64(input_num); +} + +/* result might be undefined when input_num is zero */ +static inline int hamming(uint64_t input_num) { + return _popcnt64(input_num); +} + +#endif // _MSC_VER + + + +// portable version of posix_memalign +static inline void *aligned_malloc(size_t alignment, size_t size) { + void *p; +#ifdef _MSC_VER + p = _aligned_malloc(size, alignment); +#elif defined(__MINGW32__) || defined(__MINGW64__) + p = __mingw_aligned_malloc(size, alignment); +#else + // somehow, if this is used before including "x86intrin.h", it creates an + // implicit defined warning. + if (posix_memalign(&p, alignment, size) != 0) return NULL; +#endif + return p; +} + + +#ifndef __clang__ +#ifndef _MSC_VER +static __m256i inline _mm256_loadu2_m128i(__m128i const *__addr_hi, + __m128i const *__addr_lo) { + __m256i __v256 = _mm256_castsi128_si256(_mm_loadu_si128(__addr_lo)); + return _mm256_insertf128_si256(__v256, _mm_loadu_si128(__addr_hi), 1); +} + +static inline void _mm256_storeu2_m128i(__m128i *__addr_hi, __m128i *__addr_lo, + __m256i __a) { + __m128i __v128; + + __v128 = _mm256_castsi256_si128(__a); + _mm_storeu_si128(__addr_lo, __v128); + __v128 = _mm256_extractf128_si256(__a, 1); + _mm_storeu_si128(__addr_hi, __v128); +} +#endif +#endif + + +static inline void aligned_free(void *memblock) { + if(memblock == NULL) return; +#ifdef _MSC_VER + _aligned_free(memblock); +#elif defined(__MINGW32__) || defined(__MINGW64__) + __mingw_aligned_free(memblock); +#else + free(memblock); +#endif +} + +#endif /* end of include PORTABILITY_H */ diff --git a/include/simdjson/simdprune_tables.h b/include/simdjson/simdprune_tables.h index 26d4d2fd0..1ae92adf4 100644 --- a/include/simdjson/simdprune_tables.h +++ b/include/simdjson/simdprune_tables.h @@ -1,19 +1,9 @@ -/** - * (c) Daniel Lemire - * License: Apache License 2.0 - */ +#ifndef SIMDJSON_SIMDPRUNE_TABLES_H +#define SIMDJSON_SIMDPRUNE_TABLES_H -#ifndef SIMDPRUNE_TABLES_H -#define SIMDPRUNE_TABLES_H +#include "simdjson/portability.h" -#if defined(_MSC_VER) -#include -#include -#else -#include -#endif -#include -#ifdef __SSE3__ +#ifdef __AVX__ static const unsigned char mask128_epi8[] = { 0x0, 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, 0x8, 0x9, 0xa, 0xb, 0xc, 0xd, 0xe, diff --git a/include/simdjson/simdutf8check.h b/include/simdjson/simdutf8check.h index 5d2b19183..cf5ddc83a 100644 --- a/include/simdjson/simdutf8check.h +++ b/include/simdjson/simdutf8check.h @@ -1,16 +1,12 @@ -#ifndef SIMDUTF8CHECK_H -#define SIMDUTF8CHECK_H +#ifndef SIMDJSON_SIMDUTF8CHECK_H +#define SIMDJSON_SIMDUTF8CHECK_H + #include #include #include -#ifdef _MSC_VER -/* Microsoft C/C++-compatible compiler */ -#include -#else -#include -#endif #include +#include "simdjson/portability.h" /* * legal utf-8 byte sequence * http://www.unicode.org/versions/Unicode6.0.0/ch03.pdf - page 94 diff --git a/include/simdjson/stage1_find_marks.h b/include/simdjson/stage1_find_marks.h index 1071b4e6a..684d5502d 100644 --- a/include/simdjson/stage1_find_marks.h +++ b/include/simdjson/stage1_find_marks.h @@ -1,7 +1,8 @@ -#pragma once +#ifndef SIMDJSON_STAGE1_FIND_MARKS_H +#define SIMDJSON_STAGE1_FIND_MARKS_H -#include "common_defs.h" -#include "parsedjson.h" +#include "simdjson/common_defs.h" +#include "simdjson/parsedjson.h" WARN_UNUSED bool find_structural_bits(const uint8_t *buf, size_t len, ParsedJson &pj); @@ -10,3 +11,5 @@ WARN_UNUSED static inline bool find_structural_bits(const char *buf, size_t len, ParsedJson &pj) { return find_structural_bits((const uint8_t *)buf, len, pj); } + +#endif diff --git a/include/simdjson/stage2_flatten.h b/include/simdjson/stage2_flatten.h index 449581b3c..cdba4ee7a 100644 --- a/include/simdjson/stage2_flatten.h +++ b/include/simdjson/stage2_flatten.h @@ -1,7 +1,10 @@ -#pragma once +#ifndef SIMDJSON_STAGE2_FLATTEN_H +#define SIMDJSON_STAGE2_FLATTEN_H #include "simdjson/common_defs.h" #include "simdjson/parsedjson.h" WARN_UNUSED bool flatten_indexes(size_t len, ParsedJson &pj); + +#endif diff --git a/include/simdjson/stage34_unified.h b/include/simdjson/stage34_unified.h index 4cc11349d..3cd1b66a7 100644 --- a/include/simdjson/stage34_unified.h +++ b/include/simdjson/stage34_unified.h @@ -1,4 +1,5 @@ -#pragma once +#ifndef SIMDJSON_STAGE34_UNIFIED_H +#define SIMDJSON_STAGE34_UNIFIED_H #include "simdjson/common_defs.h" #include "simdjson/parsedjson.h" @@ -13,3 +14,4 @@ static inline bool unified_machine(const char *buf, size_t len, ParsedJson &pj) return unified_machine((const uint8_t *)buf,len,pj); } +#endif diff --git a/include/simdjson/stringparsing.h b/include/simdjson/stringparsing.h index 61712a827..f2e1288ba 100644 --- a/include/simdjson/stringparsing.h +++ b/include/simdjson/stringparsing.h @@ -1,4 +1,5 @@ -#pragma once +#ifndef SIMDJSON_STRINGPARSING_H +#define SIMDJSON_STRINGPARSING_H #include "simdjson/common_defs.h" #include "simdjson/parsedjson.h" @@ -53,7 +54,7 @@ really_inline bool handle_unicode_codepoint(const uint8_t **src_ptr, uint8_t **d (((code_point - 0xd800) << 10) | (code_point_2 - 0xdc00)) + 0x10000; *src_ptr += 6; } - size_t offset = codepoint_to_utf8(code_point, *dst_ptr); + size_t offset = codepoint_to_utf8(code_point, *dst_ptr); *dst_ptr += offset; return offset > 0; } @@ -87,8 +88,8 @@ really_inline bool parse_string(const uint8_t *buf, UNUSED size_t len, __m256i unescaped_vec = _mm256_cmpeq_epi8(_mm256_max_epu8(unitsep,v),unitsep);// could do it with saturated subtraction #endif // CHECKUNESCAPED - uint32_t quote_dist = __tzcnt_u64(quote_bits); - uint32_t bs_dist = __tzcnt_u64(bs_bits); + uint32_t quote_dist = trailingzeroes(quote_bits); + uint32_t bs_dist = trailingzeroes(bs_bits); // store to dest unconditionally - we can overwrite the bits we don't like // later _mm256_storeu_si256((__m256i *)(dst), v); @@ -104,7 +105,7 @@ really_inline bool parse_string(const uint8_t *buf, UNUSED size_t len, uint32_t unescaped_bits = (uint32_t)_mm256_movemask_epi8(unescaped_vec); bool is_ok = ((quote_bits - 1) & (~ quote_bits) & unescaped_bits) == 0; #ifdef JSON_TEST_STRINGS // for unit testing - if(is_ok) foundString(buf + offset,start_of_string,pj.current_string_buf_loc - 1); + if(is_ok) foundString(buf + offset,start_of_string,pj.current_string_buf_loc - 1); else foundBadString(buf + offset); #endif // JSON_TEST_STRINGS return is_ok; @@ -176,3 +177,4 @@ really_inline bool parse_string(const uint8_t *buf, UNUSED size_t len, } +#endif diff --git a/src/jsonioutil.cpp b/src/jsonioutil.cpp index 8a7aa738a..5b74d1080 100644 --- a/src/jsonioutil.cpp +++ b/src/jsonioutil.cpp @@ -1,6 +1,6 @@ #include "simdjson/jsonioutil.h" #include - +#include char * allocate_padded_buffer(size_t length) { // we could do a simple malloc @@ -8,10 +8,14 @@ char * allocate_padded_buffer(size_t length) { // However, we might as well align to cache lines... char *padded_buffer; size_t totalpaddedlength = length + SIMDJSON_PADDING; - if (posix_memalign((void **)&padded_buffer, 64, totalpaddedlength)) { - return NULL; - }; - return padded_buffer; +#ifdef _MSC_VER + padded_buffer = (char*) _aligned_malloc(totalpaddedlength, 64); +#elif defined(__MINGW32__) || defined(__MINGW64__) + padded_buffer = __mingw_aligned_malloc(totalpaddedlength, 64); +#else + if (posix_memalign((void **)&padded_buffer, 64, totalpaddedlength) != 0) return NULL; +#endif + return padded_buffer; } std::string_view get_corpus(std::string filename) { @@ -28,7 +32,7 @@ std::string_view get_corpus(std::string filename) { size_t readb = std::fread(buf, 1, len, fp); std::fclose(fp); if(readb != len) { - free(buf); + aligned_free(buf); throw std::runtime_error("could not read the data"); } return std::string_view(buf,len); diff --git a/src/jsonminifier.cpp b/src/jsonminifier.cpp index 83be98325..60f049999 100644 --- a/src/jsonminifier.cpp +++ b/src/jsonminifier.cpp @@ -1,7 +1,7 @@ - +#include +#include "simdjson/portability.h" #ifndef __AVX2__ -#include static uint8_t jump_table[256 * 3] = { 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, @@ -59,33 +59,8 @@ size_t jsonminify(const unsigned char *bytes, size_t howmany, #else -#ifdef _MSC_VER -/* Microsoft C/C++-compatible compiler */ -#include -#else -#include -#include -#endif // _MSC_VER - #include "simdjson/simdprune_tables.h" #include -#ifndef __clang__ -static inline __m256i _mm256_loadu2_m128i(__m128i const *__addr_hi, - __m128i const *__addr_lo) { - __m256i __v256 = _mm256_castsi128_si256(_mm_loadu_si128(__addr_lo)); - return _mm256_insertf128_si256(__v256, _mm_loadu_si128(__addr_hi), 1); -} - -static inline void _mm256_storeu2_m128i(__m128i *__addr_hi, __m128i *__addr_lo, - __m256i __a) { - __m128i __v128; - - __v128 = _mm256_castsi256_si128(__a); - _mm_storeu_si128(__addr_lo, __v128); - __v128 = _mm256_extractf128_si256(__a, 1); - _mm_storeu_si128(__addr_hi, __v128); -} -#endif // a straightforward comparison of a mask against input. static uint64_t cmp_mask_against_input_mini(__m256i input_lo, __m256i input_hi, @@ -122,7 +97,7 @@ size_t jsonminify(const uint8_t *buf, size_t len, uint8_t *out) { uint64_t odd_starts = start_edges & ~even_start_mask; uint64_t even_carries = bs_bits + even_starts; uint64_t odd_carries; - bool iter_ends_odd_backslash = __builtin_uaddll_overflow( + bool iter_ends_odd_backslash = add_overflow( bs_bits, odd_starts, (unsigned long long *)&odd_carries); odd_carries |= prev_iter_ends_odd_backslash; prev_iter_ends_odd_backslash = iter_ends_odd_backslash ? 0x1ULL : 0x0ULL; @@ -171,10 +146,10 @@ size_t jsonminify(const uint8_t *buf, size_t len, uint8_t *out) { int mask2 = (whitespace >> 16) & 0xFFFF; int mask3 = (whitespace >> 32) & 0xFFFF; int mask4 = (whitespace >> 48) & 0xFFFF; - int pop1 = _popcnt64((~whitespace) & 0xFFFF); - int pop2 = _popcnt64((~whitespace) & UINT64_C(0xFFFFFFFF)); - int pop3 = _popcnt64((~whitespace) & UINT64_C(0xFFFFFFFFFFFF)); - int pop4 = _popcnt64((~whitespace)); + int pop1 = hamming((~whitespace) & 0xFFFF); + int pop2 = hamming((~whitespace) & UINT64_C(0xFFFFFFFF)); + int pop3 = hamming((~whitespace) & UINT64_C(0xFFFFFFFFFFFF)); + int pop4 = hamming((~whitespace)); __m256i vmask1 = _mm256_loadu2_m128i((const __m128i *)mask128_epi8 + (mask2 & 0x7FFF), (const __m128i *)mask128_epi8 + (mask1 & 0x7FFF)); @@ -206,7 +181,7 @@ size_t jsonminify(const uint8_t *buf, size_t len, uint8_t *out) { uint64_t even_carries = bs_bits + even_starts; uint64_t odd_carries; //bool iter_ends_odd_backslash = - __builtin_uaddll_overflow( bs_bits, odd_starts, (unsigned long long *)&odd_carries); + add_overflow( bs_bits, odd_starts, &odd_carries); odd_carries |= prev_iter_ends_odd_backslash; //prev_iter_ends_odd_backslash = iter_ends_odd_backslash ? 0x1ULL : 0x0ULL; // we never use it uint64_t even_carry_ends = even_carries & ~bs_bits; @@ -250,10 +225,10 @@ size_t jsonminify(const uint8_t *buf, size_t len, uint8_t *out) { int mask2 = (whitespace >> 16) & 0xFFFF; int mask3 = (whitespace >> 32) & 0xFFFF; int mask4 = (whitespace >> 48) & 0xFFFF; - int pop1 = _popcnt64((~whitespace) & 0xFFFF); - int pop2 = _popcnt64((~whitespace) & UINT64_C(0xFFFFFFFF)); - int pop3 = _popcnt64((~whitespace) & UINT64_C(0xFFFFFFFFFFFF)); - int pop4 = _popcnt64((~whitespace)); + int pop1 = hamming((~whitespace) & 0xFFFF); + int pop2 = hamming((~whitespace) & UINT64_C(0xFFFFFFFF)); + int pop3 = hamming((~whitespace) & UINT64_C(0xFFFFFFFFFFFF)); + int pop4 = hamming((~whitespace)); __m256i vmask1 = _mm256_loadu2_m128i((const __m128i *)mask128_epi8 + (mask2 & 0x7FFF), (const __m128i *)mask128_epi8 + (mask1 & 0x7FFF)); diff --git a/src/jsonparser.cpp b/src/jsonparser.cpp index 6e92c73d2..65af48929 100644 --- a/src/jsonparser.cpp +++ b/src/jsonparser.cpp @@ -1,5 +1,17 @@ #include "simdjson/jsonparser.h" +#ifdef _MSC_VER +#include +#include +#else #include +#endif + + +extern bool json_parse(const char * buf, size_t len, ParsedJson &pj, bool reallocifneeded); +extern bool json_parse(const std::string_view &s, ParsedJson &pj, bool reallocifneeded); +extern ParsedJson build_parsed_json(const char * buf, size_t len, bool reallocifneeded); +extern ParsedJson build_parsed_json(const std::string_view &s, bool reallocifneeded); + // parse a document found in buf, need to preallocate ParsedJson. WARN_UNUSED @@ -12,8 +24,14 @@ bool json_parse(const uint8_t *buf, size_t len, ParsedJson &pj, bool reallocifne bool reallocated = false; if(reallocifneeded) { // realloc is needed if the end of the memory crosses a page - long pagesize = sysconf (_SC_PAGESIZE); // on windows this should be SYSTEM_INFO sysInfo; GetSystemInfo(&sysInfo); sysInfo.dwPageSize - if ( (reinterpret_cast(buf + len - 1) % pagesize ) < SIMDJSON_PADDING ) { +#ifdef _MSC_VER + SYSTEM_INFO sysInfo; + GetSystemInfo(&sysInfo); + long pagesize = sysInfo.dwPageSize; +#else + long pagesize = sysconf (_SC_PAGESIZE); +#endif + if ( (reinterpret_cast(buf + len - 1) % pagesize ) < SIMDJSON_PADDING ) { const uint8_t *tmpbuf = buf; buf = (uint8_t *) allocate_padded_buffer(len); if(buf == NULL) return false; diff --git a/src/stage1_find_marks.cpp b/src/stage1_find_marks.cpp index 1e58491fe..37b0b6f30 100644 --- a/src/stage1_find_marks.cpp +++ b/src/stage1_find_marks.cpp @@ -1,12 +1,5 @@ -#ifdef _MSC_VER -/* Microsoft C/C++-compatible compiler */ -#include -#else -#include -#endif - +#include "simdjson/portability.h" #include - #include "simdjson/common_defs.h" #include "simdjson/parsedjson.h" @@ -16,7 +9,7 @@ // It seems that many parsers do UTF-8 validation. // RapidJSON does not do it by default, but a flag -// allows it. +// allows it. #ifdef SIMDJSON_UTF8VALIDATE #include "simdjson/simdutf8check.h" #endif @@ -42,10 +35,10 @@ WARN_UNUSED } #ifdef SIMDJSON_UTF8VALIDATE __m256i has_error = _mm256_setzero_si256(); - struct avx_processed_utf_bytes previous = { - .rawbytes = _mm256_setzero_si256(), - .high_nibbles = _mm256_setzero_si256(), - .carried_continuations = _mm256_setzero_si256()}; + struct avx_processed_utf_bytes previous; + previous.rawbytes = _mm256_setzero_si256(); + previous.high_nibbles = _mm256_setzero_si256(); + previous.carried_continuations = _mm256_setzero_si256(); #endif // Useful constant masks @@ -63,11 +56,13 @@ WARN_UNUSED // effectively the very first char is considered to follow "whitespace" for the // purposes of psuedo-structural character detection uint64_t prev_iter_ends_pseudo_pred = 1ULL; - size_t lenminus64 = len < 64 ? 0 : len - 64; + size_t lenminus64 = len < 64 ? 0 : len - 64; size_t idx = 0; for (; idx < lenminus64; idx += 64) { +#ifndef _MSC_VER __builtin_prefetch(buf + idx + 128); - __m256i input_lo = _mm256_loadu_si256((const __m256i *)(buf + idx + 0)); +#endif + __m256i input_lo = _mm256_loadu_si256((const __m256i *)(buf + idx + 0)); __m256i input_hi = _mm256_loadu_si256((const __m256i *)(buf + idx + 32)); #ifdef SIMDJSON_UTF8VALIDATE __m256i highbit = _mm256_set1_epi8(0x80); @@ -78,7 +73,7 @@ WARN_UNUSED _mm256_setr_epi8(9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 1)),has_error); - + } else { // it is not ascii so we have to do heavy work previous = avxcheckUTF8Bytes(input_lo, &previous, &has_error); @@ -104,7 +99,7 @@ WARN_UNUSED // indicates whether the sense of any edge going to the next iteration // should be flipped bool iter_ends_odd_backslash = - __builtin_uaddll_overflow(bs_bits, odd_starts, (unsigned long long *) &odd_carries); + add_overflow(bs_bits, odd_starts, &odd_carries); odd_carries |= prev_iter_ends_odd_backslash; // push in bit zero as a potential end @@ -214,7 +209,7 @@ WARN_UNUSED /// but otherwise the string needs to be properly padded or else we /// risk invalidating the UTF-8 checks. //////////// - if (idx < len) { + if (idx < len) { uint8_t tmpbuf[64]; memset(tmpbuf,0x20,64); memcpy(tmpbuf,buf+idx,len - idx); @@ -229,7 +224,7 @@ WARN_UNUSED _mm256_setr_epi8(9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 1)),has_error); - + } else { // it is not ascii so we have to do heavy work previous = avxcheckUTF8Bytes(input_lo, &previous, &has_error); @@ -255,7 +250,7 @@ WARN_UNUSED // indicates whether the sense of any edge going to the next iteration // should be flipped //bool iter_ends_odd_backslash = - __builtin_uaddll_overflow(bs_bits, odd_starts, (unsigned long long *) &odd_carries); + add_overflow(bs_bits, odd_starts, &odd_carries); odd_carries |= prev_iter_ends_odd_backslash; // push in bit zero as a potential end diff --git a/src/stage2_flatten.cpp b/src/stage2_flatten.cpp index eb0668566..00f8bcbeb 100644 --- a/src/stage2_flatten.cpp +++ b/src/stage2_flatten.cpp @@ -1,12 +1,6 @@ -#ifdef _MSC_VER -/* Microsoft C/C++-compatible compiler */ -#include -#else -#include -#endif #include - +#include "simdjson/portability.h" #include "simdjson/common_defs.h" #include "simdjson/parsedjson.h" @@ -20,7 +14,7 @@ #endif #define SET_BIT(i) \ - base_ptr[base + i] = (uint32_t)idx + __tzcnt_u64(s); \ + base_ptr[base + i] = (uint32_t)idx + trailingzeroes(s); \ s = s & (s - 1); #define SET_BIT1 SET_BIT(0) @@ -59,7 +53,7 @@ bool flatten_indexes(size_t len, ParsedJson &pj) { counters[k] = 0; for (size_t idx = 0; idx < len; idx += 64) { uint64_t s = *(uint64_t *)(pj.structurals + idx / 8); - uint32_t cnt = __builtin_popcountll(s); + uint32_t cnt = hamming(s); total++; counters[cnt]++; } @@ -74,23 +68,23 @@ bool flatten_indexes(size_t len, ParsedJson &pj) { uint64_t s = *(uint64_t *)(pj.structurals + idx / 8); #ifdef SUPPRESS_CHEESY_FLATTEN while (s) { - base_ptr[base++] = (uint32_t)idx + __builtin_ctzll(s); + base_ptr[base++] = (uint32_t)idx + trailingzeroes(s); s &= s - 1ULL; } #elif defined(NO_PDEP_PLEASE) - uint32_t cnt = __builtin_popcountll(s); + uint32_t cnt = hamming(s); uint32_t next_base = base + cnt; while (s) { CALL(SET_BITLOOPN, NO_PDEP_WIDTH) /*for(size_t i = 0; i < NO_PDEP_WIDTH; i++) { - base_ptr[base+i] = (uint32_t)idx + __builtin_ctzll(s); + base_ptr[base+i] = (uint32_t)idx + trailingzeroes(s); s = s & (s - 1); }*/ base += NO_PDEP_WIDTH; } base = next_base; #else - uint32_t cnt = __builtin_popcountll(s); + uint32_t cnt = hamming(s); uint32_t next_base = base + cnt; while (s) { // spoil the suspense by reducing dependency chains; actually a win even @@ -98,18 +92,18 @@ bool flatten_indexes(size_t len, ParsedJson &pj) { uint64_t s3 = _pdep_u64(~0x7ULL, s); // s3 will have bottom 3 1-bits unset uint64_t s5 = _pdep_u64(~0x1fULL, s); // s5 will have bottom 5 1-bits unset - base_ptr[base + 0] = (uint32_t)idx + __builtin_ctzll(s); + base_ptr[base + 0] = (uint32_t)idx + trailingzeroes(s); uint64_t s1 = s & (s - 1ULL); - base_ptr[base + 1] = (uint32_t)idx + __builtin_ctzll(s1); + base_ptr[base + 1] = (uint32_t)idx + trailingzeroes(s1); uint64_t s2 = s1 & (s1 - 1ULL); base_ptr[base + 2] = - (uint32_t)idx + __builtin_ctzll(s2); // uint64_t s3 = s2 & (s2 - 1ULL); - base_ptr[base + 3] = (uint32_t)idx + __builtin_ctzll(s3); + (uint32_t)idx + trailingzeroes(s2); // uint64_t s3 = s2 & (s2 - 1ULL); + base_ptr[base + 3] = (uint32_t)idx + trailingzeroes(s3); uint64_t s4 = s3 & (s3 - 1ULL); base_ptr[base + 4] = - (uint32_t)idx + __builtin_ctzll(s4); // uint64_t s5 = s4 & (s4 - 1ULL); - base_ptr[base + 5] = (uint32_t)idx + __builtin_ctzll(s5); + (uint32_t)idx + trailingzeroes(s4); // uint64_t s5 = s4 & (s4 - 1ULL); + base_ptr[base + 5] = (uint32_t)idx + trailingzeroes(s5); uint64_t s6 = s5 & (s5 - 1ULL); s = s6; base += 6; diff --git a/src/stage34_unified.cpp b/src/stage34_unified.cpp index 9226ee7f3..4e6021caf 100644 --- a/src/stage34_unified.cpp +++ b/src/stage34_unified.cpp @@ -61,9 +61,6 @@ really_inline bool is_valid_null_atom(const uint8_t *loc) { * The JSON is parsed to a tape, see the accompanying tape.md file * for documentation. ***********/ -// Implemented using Labels as Values which works in GCC and CLANG (and maybe -// also in Intel's compiler), but won't work in MSVC. This would need to be -// reimplemented differently if one wants to be standard compliant. WARN_UNUSED bool unified_machine(const uint8_t *buf, size_t len, ParsedJson &pj) { uint32_t i = 0; // index of the structural character (0,1,2,3...) @@ -73,7 +70,7 @@ bool unified_machine(const uint8_t *buf, size_t len, ParsedJson &pj) { uint32_t depth = 0; // could have an arbitrary starting depth pj.init(); if(pj.bytecapacity < len) { - printf("insufficient capacity\n"); + fprintf(stderr, "insufficient capacity\n"); return false; } // this macro reads the next structural character, updating idx, i and c. @@ -85,7 +82,11 @@ bool unified_machine(const uint8_t *buf, size_t len, ParsedJson &pj) { ////////////////////////////// START STATE ///////////////////////////// +#ifdef SIMDJSON_USE_COMPUTED_GOTO pj.ret_address[depth] = &&start_continue; +#else + pj.ret_address[depth] = 's'; +#endif pj.containing_scope_offset[depth] = pj.get_current_loc(); pj.write_tape(0, 'r'); // r for root, 0 is going to get overwritten // the root is used, if nothing else, to capture the size of the tape @@ -98,7 +99,11 @@ bool unified_machine(const uint8_t *buf, size_t len, ParsedJson &pj) { switch (c) { case '{': pj.containing_scope_offset[depth] = pj.get_current_loc(); +#ifdef SIMDJSON_USE_COMPUTED_GOTO pj.ret_address[depth] = &&start_continue; +#else + pj.ret_address[depth] = 's'; +#endif depth++; if (depth > pj.depthcapacity) { goto fail; @@ -107,7 +112,11 @@ bool unified_machine(const uint8_t *buf, size_t len, ParsedJson &pj) { goto object_begin; case '[': pj.containing_scope_offset[depth] = pj.get_current_loc(); +#ifdef SIMDJSON_USE_COMPUTED_GOTO pj.ret_address[depth] = &&start_continue; +#else + pj.ret_address[depth] = 's'; +#endif depth++; if (depth > pj.depthcapacity) { goto fail; @@ -299,7 +308,11 @@ object_key_state: pj.containing_scope_offset[depth] = pj.get_current_loc(); pj.write_tape(0, c); // here the compilers knows what c is so this gets optimized // we have not yet encountered } so we need to come back for it +#ifdef SIMDJSON_USE_COMPUTED_GOTO pj.ret_address[depth] = &&object_continue; +#else + pj.ret_address[depth] = 'o'; +#endif // we found an object inside an object, so we need to increment the depth depth++; if (depth > pj.depthcapacity) { @@ -312,7 +325,11 @@ object_key_state: pj.containing_scope_offset[depth] = pj.get_current_loc(); pj.write_tape(0, c); // here the compilers knows what c is so this gets optimized // we have not yet encountered } so we need to come back for it +#ifdef SIMDJSON_USE_COMPUTED_GOTO pj.ret_address[depth] = &&object_continue; +#else + pj.ret_address[depth] = 'o'; +#endif // we found an array inside an object, so we need to increment the depth depth++; if (depth > pj.depthcapacity) { @@ -352,7 +369,15 @@ scope_end: pj.annotate_previousloc(pj.containing_scope_offset[depth], pj.get_current_loc()); // goto saved_state +#ifdef SIMDJSON_USE_COMPUTED_GOTO goto *pj.ret_address[depth]; +#else + if(pj.ret_address[depth] == 'a') { + goto array_continue; + } else if (pj.ret_address[depth] == 'o') { + goto object_continue; + } else goto start_continue; +#endif ////////////////////////////// ARRAY STATES ///////////////////////////// array_begin: @@ -415,7 +440,11 @@ main_array_switch: // we have not yet encountered ] so we need to come back for it pj.containing_scope_offset[depth] = pj.get_current_loc(); pj.write_tape(0, c); // here the compilers knows what c is so this gets optimized +#ifdef SIMDJSON_USE_COMPUTED_GOTO pj.ret_address[depth] = &&array_continue; +#else + pj.ret_address[depth] = 'a'; +#endif // we found an object inside an array, so we need to increment the depth depth++; if (depth > pj.depthcapacity) { @@ -428,7 +457,11 @@ main_array_switch: // we have not yet encountered ] so we need to come back for it pj.containing_scope_offset[depth] = pj.get_current_loc(); pj.write_tape(0, c); // here the compilers knows what c is so this gets optimized +#ifdef SIMDJSON_USE_COMPUTED_GOTO pj.ret_address[depth] = &&array_continue; +#else + pj.ret_address[depth] = 'a'; +#endif // we found an array inside an array, so we need to increment the depth depth++; if (depth > pj.depthcapacity) { @@ -457,11 +490,11 @@ array_continue: succeed: depth --; if(depth != 0) { - printf("internal bug\n"); + fprintf(stderr, "internal bug\n"); abort(); } if(pj.containing_scope_offset[depth] != 0) { - printf("internal bug\n"); + fprintf(stderr, "internal bug\n"); abort(); } pj.annotate_previousloc(pj.containing_scope_offset[depth], diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 44a2dda92..c88fc500c 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -1,3 +1,8 @@ +if(MSVC) +target_include_directories(${SIMDJSON_LIB_NAME} + PUBLIC ${PROJECT_SOURCE_DIR}/windows +) +endif() add_cpp_test(jsoncheck) add_test(jsoncheck jsoncheck) diff --git a/tests/allparserscheckfile.cpp b/tests/allparserscheckfile.cpp index cc915a83e..edd2468df 100644 --- a/tests/allparserscheckfile.cpp +++ b/tests/allparserscheckfile.cpp @@ -108,7 +108,7 @@ int main(int argc, char *argv[]) { printf("fastjson : %s \n", fastjson_correct ? "correct":"invalid"); printf("gason : %s \n", gason_correct ? "correct":"invalid"); printf("ultrajson : %s \n", ultrajson_correct ? "correct":"invalid"); - free((void*)p.data()); + aligned_free((void*)p.data()); free(buffer); return EXIT_SUCCESS; } diff --git a/tests/jsoncheck.cpp b/tests/jsoncheck.cpp index 58595af6c..f1fcebbdf 100644 --- a/tests/jsoncheck.cpp +++ b/tests/jsoncheck.cpp @@ -1,11 +1,16 @@ #include #include +#ifndef _MSC_VER #include +#include +#else +// Microsoft can't be bothered to provide standard utils. +#include +#endif #include #include #include #include -#include #include "simdjson/jsonparser.h" @@ -34,7 +39,7 @@ bool validate(const char *dirname) { struct dirent **entry_list; int c = scandir(dirname, &entry_list, 0, alphasort); if (c < 0) { - printf("error accessing %s \n", dirname); + fprintf(stderr, "error accessing %s \n", dirname); return false; } if (c == 0) { @@ -62,19 +67,19 @@ bool validate(const char *dirname) { std::string_view p; try { p = get_corpus(fullpath); - } catch (const std::exception& e) { - std::cout << "Could not load the file " << fullpath << std::endl; + } catch (const std::exception& e) { + std::cerr << "Could not load the file " << fullpath << std::endl; return EXIT_FAILURE; } ParsedJson pj; bool allocok = pj.allocateCapacity(p.size(), 1024); if(!allocok) { - std::cerr<< "can't allocate memory"<d_name); + if(!isfileasexpected[i]) fprintf(stderr, "%s \n", entry_list[i]->d_name); } } for (int i = 0; i < c; ++i) free(entry_list[i]); free(entry_list); delete[] isfileasexpected; - return everythingfine; } diff --git a/tests/numberparsingcheck.cpp b/tests/numberparsingcheck.cpp index 3989179e5..cf5021abc 100644 --- a/tests/numberparsingcheck.cpp +++ b/tests/numberparsingcheck.cpp @@ -57,8 +57,8 @@ inline void foundInteger(int64_t result, const uint8_t *buf) { char *endptr; long long expected = strtoll((const char *)buf, &endptr, 10); if ((endptr == (const char *)buf) || (expected != result)) { - printf("Error: parsed %" PRId64 " out of %.32s, ", result, buf); - printf(" while parsing %s \n", fullpath); + fprintf(stderr, "Error: parsed %" PRId64 " out of %.32s, ", result, buf); + fprintf(stderr, " while parsing %s \n", fullpath); parse_error |= PARSE_ERROR; } } @@ -68,23 +68,23 @@ inline void foundFloat(double result, const uint8_t *buf) { float_count++; double expected = strtod((const char *)buf, &endptr); if (endptr == (const char *)buf) { - printf("parsed %f from %.32s whereas strtod refuses to parse a float, ", + fprintf(stderr, "parsed %f from %.32s whereas strtod refuses to parse a float, ", result, buf); - printf(" while parsing %s \n", fullpath); + fprintf(stderr, " while parsing %s \n", fullpath); parse_error |= PARSE_ERROR; } if( fpclassify(expected) != fpclassify(result) ) { - printf("floats not in the same category expected: %f observed: %f \n", expected, result); - printf("%.128s\n", buf); + fprintf(stderr, "floats not in the same category expected: %f observed: %f \n", expected, result); + fprintf(stderr, "%.128s\n", buf); parse_error |= PARSE_ERROR; } // we want to get some reasonable relative accuracy else if (fabs(expected - result) / fmin(fabs(expected), fabs(result)) > 1e-14) { - printf("parsed %.128e from \n", result); - printf(" %.200s whereas strtod gives\n", buf); - printf(" %.128e,", expected); - printf(" while parsing %s \n", fullpath); + fprintf(stderr, "parsed %.128e from \n", result); + fprintf(stderr, " %.200s whereas strtod gives\n", buf); + fprintf(stderr, " %.128e,", expected); + fprintf(stderr, " while parsing %s \n", fullpath); parse_error |= PARSE_ERROR; } } @@ -154,7 +154,7 @@ bool validate(const char *dirname) { float_count, invalid_count, int_count + float_count + invalid_count); } - free((void*)p.data()); + aligned_free((void*)p.data()); free(fullpath); } } diff --git a/tests/stringparsingcheck.cpp b/tests/stringparsingcheck.cpp index 6ce5fb3a2..c76a0910c 100644 --- a/tests/stringparsingcheck.cpp +++ b/tests/stringparsingcheck.cpp @@ -341,7 +341,7 @@ bool validate(const char *dirname) { bigbuffer = (char *) malloc(p.size()); if(bigbuffer == NULL) { std::cerr << "can't allocate memory" << std::endl; - free((void*)p.data()); + aligned_free((void*)p.data()); return false; } bad_string = 0; @@ -350,7 +350,7 @@ bool validate(const char *dirname) { empty_string = 0; bool isok = json_parse(p, pj); free(bigbuffer); - free((void*)p.data()); + aligned_free((void*)p.data()); if (good_string > 0) { printf("File %40s %s --- bad strings: %10zu \tgood strings: %10zu\t " "empty strings: %10zu " @@ -368,7 +368,7 @@ bool validate(const char *dirname) { } printf("%zu strings checked.\n", total_strings); if (probable_bug) { - printf("STRING PARSING FAILS?\n"); + fprintf(stderr, "STRING PARSING FAILS?\n"); } else { printf("All ok.\n"); } diff --git a/tools/cmake/FindOptions.cmake b/tools/cmake/FindOptions.cmake index ab7f58e9f..a48662802 100644 --- a/tools/cmake/FindOptions.cmake +++ b/tools/cmake/FindOptions.cmake @@ -18,9 +18,10 @@ endif() if(NOT MSVC) set (OPT_FLAGS "${OPT_FLAGS} -mavx2 -mbmi2 -mpclmul") else() - set (OPT_FLAGS "${OPT_FLAGS} /arch:AVX2") + set (OPT_FLAGS "${OPT_FLAGS} /arch:AVX2 /std:c++latest") endif() + if(NOT MSVC) set(CXXSTD_FLAGS "-std=c++17 -fPIC") endif() @@ -41,5 +42,5 @@ endif() set(CMAKE_CXX_FLAGS "${CXXSTD_FLAGS} ${OPT_FLAGS} ${INCLUDE_FLAGS} ${WARNING_FLAGS} ${SIMDJSON_SANITIZE_FLAGS} ") if(MSVC) -add_definitions( "/W3 /D_CRT_SECURE_NO_WARNINGS /wd4005 /wd4996 /wd4267 /wd4244 /wd4113 /nologo") +add_definitions( "${OPT_FLAGS} /W3 /D_CRT_SECURE_NO_WARNINGS /wd4005 /wd4996 /wd4267 /wd4244 /wd4113 /nologo") endif() diff --git a/tools/json2json.cpp b/tools/json2json.cpp index 67596b9fb..54c783532 100644 --- a/tools/json2json.cpp +++ b/tools/json2json.cpp @@ -1,6 +1,7 @@ #include +#ifndef _MSC_VER #include - +#endif #include "simdjson/jsonioutil.h" #include "simdjson/jsonparser.h" @@ -41,9 +42,11 @@ void compute_dump(ParsedJson::iterator &pjh) { } int main(int argc, char *argv[]) { + bool rawdump = false; + bool apidump = false; + +#ifndef _MSC_VER int c; - bool rawdump = false; - bool apidump = false; while ((c = getopt(argc, argv, "da")) != -1) switch (c) { @@ -56,6 +59,9 @@ int main(int argc, char *argv[]) { default: abort(); } +#else + int optind = 1; +#endif if (optind >= argc) { cerr << "Reads json in, out the result of the parsing. " << endl; cerr << "Usage: " << argv[0] << " " << endl; @@ -81,7 +87,7 @@ int main(int argc, char *argv[]) { return EXIT_FAILURE; } bool is_ok = json_parse(p, pj); // do the parsing, return false on error - free((void *)p.data()); + aligned_free((void *)p.data()); if (!is_ok) { std::cerr << " Parsing failed. " << std::endl; return EXIT_FAILURE; diff --git a/tools/jsonstats.cpp b/tools/jsonstats.cpp index 63c37b830..f479c606f 100644 --- a/tools/jsonstats.cpp +++ b/tools/jsonstats.cpp @@ -1,5 +1,4 @@ #include -#include #include "simdjson/jsonioutil.h" #include "simdjson/jsonparser.h" @@ -118,14 +117,7 @@ stat_t simdjson_computestats(const std::string_view &p) { int main(int argc, char *argv[]) { - int c; - - while ((c = getopt(argc, argv, "")) != -1) - switch (c) { - - default: - abort(); - } + int optind = 1; if (optind >= argc) { cerr << "Reads json, prints stats. " << endl; cerr << "Usage: " << argv[0] << " " << endl; diff --git a/tools/minify.cpp b/tools/minify.cpp index 86ab78481..e91ec6a9e 100644 --- a/tools/minify.cpp +++ b/tools/minify.cpp @@ -18,5 +18,5 @@ int main(int argc, char *argv[]) { } jsonminify(p, (char *)p.data()); printf("%s",p.data()); - free((void*)p.data()); + aligned_free((void*)p.data()); } diff --git a/windows/dirent_portable.h b/windows/dirent_portable.h new file mode 100644 index 000000000..4c99eda06 --- /dev/null +++ b/windows/dirent_portable.h @@ -0,0 +1,1043 @@ +// only use under Visual Studio and only for jsoncheck.cpp +/* + * This file was originally: "dirent for Visual C++" from: http://softagalleria.net/dirent.php (version 1.20.1) + * However I've modified it to by adding: + * + * a fallback to if _WIN32 is not defined + * two missing methods: scandir(...) and alphasort(...) + * and some other minor modifications (see below) + * + * + * Original license from http://softagalleria.net/dirent.php + * + *======================================================================== + * + * dirent.h - dirent API for Microsoft Visual Studio + * + * Copyright (C) 2006-2012 Toni Ronkko + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files (the + * ``Software''), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED ``AS IS'', WITHOUT WARRANTY OF ANY KIND, EXPRESS + * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL TONI RONKKO BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * $Id: dirent.h,v 1.20 2014/03/19 17:52:23 tronkko Exp $ + * + * ========================================================================= + * Added: + * -> some undefs to prevent possible compiler warnings + * -> the scandir(...) and alphasort(...) methods + * -> the optional DIRENT_USE_ASCII_SHORT_PATHS_ON_WINDOWS definition (needed for browsing with short ASCII paths instead of long UTF8 paths). + * WARNING: in my tests the usage of the long UTF8 paths is not fully functional (patches are welcome) + * All these additions have been made to made usage for Windows consistent + * with what I get using under my Ubuntu Linux OS. + * ========================================================================= + * + * The code of the scandir(...) method come from the musl library (http://www.musl-libc.org/) + * (MIT licensed, Copyright © 2005-2014 Rich Felker, et al.). + * + * The code of the alphasort(...) method and of all the other minor modifications is in the public domain. + * + */ + +#if (!defined(_WIN32) && !defined(_WIN64)) +# include +#else // #if (!defined(_WIN32) && !defined(_WIN64)) + +#ifndef DIRENT_H +#define DIRENT_H + +/* + * Define architecture flags so we don't need to include windows.h. + * Avoiding windows.h makes it simpler to use windows sockets in conjunction + * with dirent.h. + */ +#if !defined(_68K_) && !defined(_MPPC_) && !defined(_X86_) && !defined(_IA64_) && !defined(_AMD64_) && defined(_M_IX86) +# define _X86_ +#endif +#if !defined(_68K_) && !defined(_MPPC_) && !defined(_X86_) && !defined(_IA64_) && !defined(_AMD64_) && defined(_M_AMD64) +#define _AMD64_ +#endif +#include +#include +#include +#include // MAX_PATH is defined here, +#include +#include +#include +#include +#include +#include +#include +#include +#if (!defined(SIZE_MAX) && !defined(INT_MAX)) +#include // INT_MAX +#endif //(!defined(SIZE_MAX) && !defined(INT_MAX)) + + +//#define DIRENT_USE_ASCII_SHORT_PATHS_ON_WINDOWS // set it globally, not just here] + +/* Indicates that d_type field is available in dirent structure */ +#define _DIRENT_HAVE_D_TYPE + +/* Indicates that d_namlen field is available in dirent structure */ +#define _DIRENT_HAVE_D_NAMLEN + +/* Entries missing from MSVC 6.0 */ +#if !defined(FILE_ATTRIBUTE_DEVICE) +# define FILE_ATTRIBUTE_DEVICE 0x40 +#endif + +/* File type and permission flags for stat() */ +#if !defined(S_IFMT) +# define S_IFMT _S_IFMT /* File type mask */ +#endif +#if !defined(S_IFDIR) +# define S_IFDIR _S_IFDIR /* Directory */ +#endif +#if !defined(S_IFCHR) +# define S_IFCHR _S_IFCHR /* Character device */ +#endif +#if !defined(S_IFFIFO) +# define S_IFFIFO _S_IFFIFO /* Pipe */ +#endif +#if !defined(S_IFREG) +# define S_IFREG _S_IFREG /* Regular file */ +#endif +#if !defined(S_IREAD) +# define S_IREAD _S_IREAD /* Read permission */ +#endif +#if !defined(S_IWRITE) +# define S_IWRITE _S_IWRITE /* Write permission */ +#endif +#if !defined(S_IEXEC) +# define S_IEXEC _S_IEXEC /* Execute permission */ +#endif +#if !defined(S_IFIFO) +# define S_IFIFO _S_IFIFO /* Pipe */ +#endif +#if !defined(S_IFBLK) +# define S_IFBLK 0 /* Block device */ +#endif +#if !defined(S_IFLNK) +# define S_IFLNK 0 /* Link */ +#endif +#if !defined(S_IFSOCK) +# define S_IFSOCK 0 /* Socket */ +#endif + +#if defined(_MSC_VER) +# define S_IRUSR S_IREAD /* Read user */ +# define S_IWUSR S_IWRITE /* Write user */ +# define S_IXUSR 0 /* Execute user */ +# define S_IRGRP 0 /* Read group */ +# define S_IWGRP 0 /* Write group */ +# define S_IXGRP 0 /* Execute group */ +# define S_IROTH 0 /* Read others */ +# define S_IWOTH 0 /* Write others */ +# define S_IXOTH 0 /* Execute others */ +#endif + +/* Maximum length of file name */ +#ifndef DIRENT_MAX_PATH +# ifndef MAX_PATH +# define MAX_PATH PATH_MAX // it should be in AFAIK +# endif //MAX_PATH +# ifndef DIRENT_USE_ASCII_SHORT_PATHS_ON_WINDOWS // utf8 strings can have up to 4 bytes per char +# define DIRENT_MAX_PATH (MAX_PATH*4) +# else //DIRENT_USE_ASCII_SHORT_PATHS_ON_WINDOWS +# define DIRENT_MAX_PATH (MAX_PATH) +# endif //DIRENT_USE_ASCII_SHORT_PATHS_ON_WINDOWS +#endif //DIRENT_MAX_PATH + +/* File type flags for d_type */ +#define DT_UNKNOWN 0 +#define DT_REG S_IFREG +#define DT_DIR S_IFDIR +#define DT_FIFO S_IFIFO +#define DT_SOCK S_IFSOCK +#define DT_CHR S_IFCHR +#define DT_BLK S_IFBLK +#define DT_LNK S_IFLNK + +/* Macros for converting between st_mode and d_type */ +#define IFTODT(mode) ((mode) & S_IFMT) +#define DTTOIF(type) (type) + +/* + * File type macros. Note that block devices, sockets and links cannot be + * distinguished on Windows and the macros S_ISBLK, S_ISSOCK and S_ISLNK are + * only defined for compatibility. These macros should always return false + * on Windows. + */ + +// Added some undefs to prevent possible compiler warnings +#undef S_ISFIFO +#undef S_ISDIR +#undef S_ISREG +#undef S_ISLNK +#undef S_ISSOCK +#undef S_ISSOCK +#undef S_ISCHR +#undef S_ISBLK + +#define S_ISFIFO(mode) (((mode) & S_IFMT) == S_IFIFO) +#define S_ISDIR(mode) (((mode) & S_IFMT) == S_IFDIR) +#define S_ISREG(mode) (((mode) & S_IFMT) == S_IFREG) +#define S_ISLNK(mode) (((mode) & S_IFMT) == S_IFLNK) +#define S_ISSOCK(mode) (((mode) & S_IFMT) == S_IFSOCK) +#define S_ISCHR(mode) (((mode) & S_IFMT) == S_IFCHR) +#define S_ISBLK(mode) (((mode) & S_IFMT) == S_IFBLK) + +/* Return the exact length of d_namlen without zero terminator */ +#define _D_EXACT_NAMLEN(p) ((p)->d_namlen) + +/* Return number of bytes needed to store d_namlen */ +#define _D_ALLOC_NAMLEN(p) (DIRENT_MAX_PATH) + + +#ifdef __cplusplus +extern "C" { +#endif + + +/* Wide-character version */ +struct _wdirent { + long d_ino; /* Always zero */ + unsigned short d_reclen; /* Structure size */ + size_t d_namlen; /* Length of name without \0 */ + int d_type; /* File type */ + wchar_t d_name[DIRENT_MAX_PATH]; /* File name */ +}; +typedef struct _wdirent _wdirent; + +struct _WDIR { + struct _wdirent ent; /* Current directory entry */ + WIN32_FIND_DATAW data; /* Private file data */ + int cached; /* True if data is valid */ + HANDLE handle; /* Win32 search handle */ + wchar_t *patt; /* Initial directory name */ +}; +typedef struct _WDIR _WDIR; + +static _WDIR *_wopendir (const wchar_t *dirname); +static struct _wdirent *_wreaddir (_WDIR *dirp); +static int _wclosedir (_WDIR *dirp); +static void _wrewinddir (_WDIR* dirp); + + +/* For compatibility with Symbian */ +#define wdirent _wdirent +#define WDIR _WDIR +#define wopendir _wopendir +#define wreaddir _wreaddir +#define wclosedir _wclosedir +#define wrewinddir _wrewinddir + + +/* Multi-byte character versions */ +struct dirent { + long d_ino; /* Always zero */ + unsigned short d_reclen; /* Structure size */ + size_t d_namlen; /* Length of name without \0 */ + int d_type; /* File type */ + char d_name[DIRENT_MAX_PATH]; /* File name */ +}; +typedef struct dirent dirent; + +struct DIR { + struct dirent ent; + struct _WDIR *wdirp; +}; +typedef struct DIR DIR; + +static DIR *opendir (const char *dirname); +static struct dirent *readdir (DIR *dirp); +static int closedir (DIR *dirp); +static void rewinddir (DIR* dirp); + + +/* Internal utility functions */ +static WIN32_FIND_DATAW *dirent_first (_WDIR *dirp); +static WIN32_FIND_DATAW *dirent_next (_WDIR *dirp); + +static int dirent_mbstowcs_s( + size_t *pReturnValue, + wchar_t *wcstr, + size_t sizeInWords, + const char *mbstr, + size_t count); + +static int dirent_wcstombs_s( + size_t *pReturnValue, + char *mbstr, + size_t sizeInBytes, + const wchar_t *wcstr, + size_t count); + +static void dirent_set_errno (int error); + +/* + * Open directory stream DIRNAME for read and return a pointer to the + * internal working area that is used to retrieve individual directory + * entries. + */ +static _WDIR* +_wopendir( + const wchar_t *dirname) +{ + _WDIR *dirp = NULL; + int error; + + /* Must have directory name */ + if (dirname == NULL || dirname[0] == '\0') { + dirent_set_errno (ENOENT); + return NULL; + } + + /* Allocate new _WDIR structure */ + dirp = (_WDIR*) malloc (sizeof (struct _WDIR)); + if (dirp != NULL) { + DWORD n; + + /* Reset _WDIR structure */ + dirp->handle = INVALID_HANDLE_VALUE; + dirp->patt = NULL; + dirp->cached = 0; + + /* Compute the length of full path plus zero terminator */ + n = GetFullPathNameW (dirname, 0, NULL, NULL); + + /* Allocate room for absolute directory name and search pattern */ + dirp->patt = (wchar_t*) malloc (sizeof (wchar_t) * n + 16); + if (dirp->patt) { + + /* + * Convert relative directory name to an absolute one. This + * allows rewinddir() to function correctly even when current + * working directory is changed between opendir() and rewinddir(). + */ + n = GetFullPathNameW (dirname, n, dirp->patt, NULL); + if (n > 0) { + wchar_t *p; + + /* Append search pattern \* to the directory name */ + p = dirp->patt + n; + if (dirp->patt < p) { + switch (p[-1]) { + case '\\': + case '/': + case ':': + /* Directory ends in path separator, e.g. c:\temp\ */ + /*NOP*/; + break; + + default: + /* Directory name doesn't end in path separator */ + *p++ = '\\'; + } + } + *p++ = '*'; + *p = '\0'; + + /* Open directory stream and retrieve the first entry */ + if (dirent_first (dirp)) { + /* Directory stream opened successfully */ + error = 0; + } else { + /* Cannot retrieve first entry */ + error = 1; + dirent_set_errno (ENOENT); + } + + } else { + /* Cannot retrieve full path name */ + dirent_set_errno (ENOENT); + error = 1; + } + + } else { + /* Cannot allocate memory for search pattern */ + error = 1; + } + + } else { + /* Cannot allocate _WDIR structure */ + error = 1; + } + + /* Clean up in case of error */ + if (error && dirp) { + _wclosedir (dirp); + dirp = NULL; + } + + return dirp; +} + +/* + * Read next directory entry. The directory entry is returned in dirent + * structure in the d_name field. Individual directory entries returned by + * this function include regular files, sub-directories, pseudo-directories + * "." and ".." as well as volume labels, hidden files and system files. + */ +static struct _wdirent* +_wreaddir( + _WDIR *dirp) +{ + WIN32_FIND_DATAW *datap; + struct _wdirent *entp; + + /* Read next directory entry */ + datap = dirent_next (dirp); + if (datap) { + size_t n; + DWORD attr; + + /* Pointer to directory entry to return */ + entp = &dirp->ent; + + /* + * Copy file name as wide-character string. If the file name is too + * long to fit in to the destination buffer, then truncate file name + * to DIRENT_MAX_PATH characters and zero-terminate the buffer. + */ + n = 0; + while (n + 1 < DIRENT_MAX_PATH && datap->cFileName[n] != 0) { + entp->d_name[n] = datap->cFileName[n]; + n++; + } + dirp->ent.d_name[n] = 0; + + /* Length of file name excluding zero terminator */ + entp->d_namlen = n; + + /* File type */ + attr = datap->dwFileAttributes; + if ((attr & FILE_ATTRIBUTE_DEVICE) != 0) { + entp->d_type = DT_CHR; + } else if ((attr & FILE_ATTRIBUTE_DIRECTORY) != 0) { + entp->d_type = DT_DIR; + } else { + entp->d_type = DT_REG; + } + + /* Reset dummy fields */ + entp->d_ino = 0; + entp->d_reclen = sizeof (struct _wdirent); + + } else { + + /* Last directory entry read */ + entp = NULL; + + } + + return entp; +} + +/* + * Close directory stream opened by opendir() function. This invalidates the + * DIR structure as well as any directory entry read previously by + * _wreaddir(). + */ +static int +_wclosedir( + _WDIR *dirp) +{ + int ok; + if (dirp) { + + /* Release search handle */ + if (dirp->handle != INVALID_HANDLE_VALUE) { + FindClose (dirp->handle); + dirp->handle = INVALID_HANDLE_VALUE; + } + + /* Release search pattern */ + if (dirp->patt) { + free (dirp->patt); + dirp->patt = NULL; + } + + /* Release directory structure */ + free (dirp); + ok = /*success*/0; + + } else { + /* Invalid directory stream */ + dirent_set_errno (EBADF); + ok = /*failure*/-1; + } + return ok; +} + +/* + * Rewind directory stream such that _wreaddir() returns the very first + * file name again. + */ +static void +_wrewinddir( + _WDIR* dirp) +{ + if (dirp) { + /* Release existing search handle */ + if (dirp->handle != INVALID_HANDLE_VALUE) { + FindClose (dirp->handle); + } + + /* Open new search handle */ + dirent_first (dirp); + } +} + +/* Get first directory entry (internal) */ +static WIN32_FIND_DATAW* +dirent_first( + _WDIR *dirp) +{ + WIN32_FIND_DATAW *datap; + + /* Open directory and retrieve the first entry */ + dirp->handle = FindFirstFileW (dirp->patt, &dirp->data); + if (dirp->handle != INVALID_HANDLE_VALUE) { + + /* a directory entry is now waiting in memory */ + datap = &dirp->data; + dirp->cached = 1; + + } else { + + /* Failed to re-open directory: no directory entry in memory */ + dirp->cached = 0; + datap = NULL; + + } + return datap; +} + +/* Get next directory entry (internal) */ +static WIN32_FIND_DATAW* +dirent_next( + _WDIR *dirp) +{ + WIN32_FIND_DATAW *p; + + /* Get next directory entry */ + if (dirp->cached != 0) { + + /* A valid directory entry already in memory */ + p = &dirp->data; + dirp->cached = 0; + + } else if (dirp->handle != INVALID_HANDLE_VALUE) { + + /* Get the next directory entry from stream */ + if (FindNextFileW (dirp->handle, &dirp->data) != FALSE) { + /* Got a file */ + p = &dirp->data; + } else { + /* The very last entry has been processed or an error occured */ + FindClose (dirp->handle); + dirp->handle = INVALID_HANDLE_VALUE; + p = NULL; + } + + } else { + + /* End of directory stream reached */ + p = NULL; + + } + + return p; +} + +/* + * Open directory stream using plain old C-string. + */ +static DIR* +opendir( + const char *dirname) +{ + struct DIR *dirp; + int error; + + /* Must have directory name */ + if (dirname == NULL || dirname[0] == '\0') { + dirent_set_errno (ENOENT); + return NULL; + } + + /* Allocate memory for DIR structure */ + dirp = (DIR*) malloc (sizeof (struct DIR)); + if (dirp) { + wchar_t wname[DIRENT_MAX_PATH]; + size_t n; + + /* Convert directory name to wide-character string */ + error = dirent_mbstowcs_s (&n, wname, DIRENT_MAX_PATH, dirname, DIRENT_MAX_PATH); + if (!error) { + + /* Open directory stream using wide-character name */ + dirp->wdirp = _wopendir (wname); + if (dirp->wdirp) { + /* Directory stream opened */ + error = 0; + } else { + /* Failed to open directory stream */ + error = 1; + } + + } else { + /* + * Cannot convert file name to wide-character string. This + * occurs if the string contains invalid multi-byte sequences or + * the output buffer is too small to contain the resulting + * string. + */ + error = 1; + } + + } else { + /* Cannot allocate DIR structure */ + error = 1; + } + + /* Clean up in case of error */ + if (error && dirp) { + free (dirp); + dirp = NULL; + } + + return dirp; +} + +/* + * Read next directory entry. + * + * When working with text consoles, please note that file names returned by + * readdir() are represented in the default ANSI code page while any output to + * console is typically formatted on another code page. Thus, non-ASCII + * characters in file names will not usually display correctly on console. The + * problem can be fixed in two ways: (1) change the character set of console + * to 1252 using chcp utility and use Lucida Console font, or (2) use + * _cprintf function when writing to console. The _cprinf() will re-encode + * ANSI strings to the console code page so many non-ASCII characters will + * display correcly. + */ +static struct dirent* +readdir( + DIR *dirp) +{ + WIN32_FIND_DATAW *datap; + struct dirent *entp; + + /* Read next directory entry */ + datap = dirent_next (dirp->wdirp); + if (datap) { + size_t n; + int error; + + /* Attempt to convert file name to multi-byte string */ + error = dirent_wcstombs_s( + &n, dirp->ent.d_name, DIRENT_MAX_PATH, datap->cFileName, DIRENT_MAX_PATH); + + /* + * If the file name cannot be represented by a multi-byte string, + * then attempt to use old 8+3 file name. This allows traditional + * Unix-code to access some file names despite of unicode + * characters, although file names may seem unfamiliar to the user. + * + * Be ware that the code below cannot come up with a short file + * name unless the file system provides one. At least + * VirtualBox shared folders fail to do this. + */ + if (error && datap->cAlternateFileName[0] != '\0') { + error = dirent_wcstombs_s( + &n, dirp->ent.d_name, DIRENT_MAX_PATH, + datap->cAlternateFileName, DIRENT_MAX_PATH); + } + + if (!error) { + DWORD attr; + + /* Initialize directory entry for return */ + entp = &dirp->ent; + + /* Length of file name excluding zero terminator */ + entp->d_namlen = n - 1; + + /* File attributes */ + attr = datap->dwFileAttributes; + if ((attr & FILE_ATTRIBUTE_DEVICE) != 0) { + entp->d_type = DT_CHR; + } else if ((attr & FILE_ATTRIBUTE_DIRECTORY) != 0) { + entp->d_type = DT_DIR; + } else { + entp->d_type = DT_REG; + } + + /* Reset dummy fields */ + entp->d_ino = 0; + entp->d_reclen = sizeof (struct dirent); + + } else { + /* + * Cannot convert file name to multi-byte string so construct + * an errornous directory entry and return that. Note that + * we cannot return NULL as that would stop the processing + * of directory entries completely. + */ + entp = &dirp->ent; + entp->d_name[0] = '?'; + entp->d_name[1] = '\0'; + entp->d_namlen = 1; + entp->d_type = DT_UNKNOWN; + entp->d_ino = 0; + entp->d_reclen = 0; + } + + } else { + /* No more directory entries */ + entp = NULL; + } + + return entp; +} + +/* + * Close directory stream. + */ +static int +closedir( + DIR *dirp) +{ + int ok; + if (dirp) { + + /* Close wide-character directory stream */ + ok = _wclosedir (dirp->wdirp); + dirp->wdirp = NULL; + + /* Release multi-byte character version */ + free (dirp); + + } else { + + /* Invalid directory stream */ + dirent_set_errno (EBADF); + ok = /*failure*/-1; + + } + return ok; +} + +/* + * Rewind directory stream to beginning. + */ +static void +rewinddir( + DIR* dirp) +{ + /* Rewind wide-character string directory stream */ + _wrewinddir (dirp->wdirp); +} + +/* Convert multi-byte string to wide character string */ +static int +dirent_mbstowcs_s( + size_t *pReturnValue, + wchar_t *wcstr, + size_t sizeInWords, + const char *mbstr, + size_t count) +{ + int error; +#ifndef DIRENT_USE_ASCII_SHORT_PATHS_ON_WINDOWS + // we don't use "count" at all: we assume mstr is zero terminated: + size_t n = (size_t) MultiByteToWideChar (CP_UTF8, 0, mbstr, -1, wcstr, 0);//sizeInWords); + if (n==0) { + error = 1; + if (sizeInWords>0) wcstr[0]=L'\0'; + if (pReturnValue) *pReturnValue = 0; + } + else if (n<=sizeInWords) { + error = MultiByteToWideChar (CP_UTF8, 0, mbstr, -1, wcstr, n) == 0 ? 1 : 0; + if (pReturnValue) *pReturnValue = n; + } + else { + // Buffer too low: + if (sizeInWords>0) { + if (sizeInWords>1) MultiByteToWideChar (CP_UTF8, 0, mbstr, -1, wcstr, sizeInWords-1); + wcstr[sizeInWords-1] = L'\0'; + } + if (pReturnValue) *pReturnValue = sizeInWords; + error = 1; + } + + /* + if (!wcstr || n < count) { + // Zero-terminate output buffer + if (wcstr && sizeInWords) { + if (n >= sizeInWords) { + n = sizeInWords - 1; + } + wcstr[n] = 0; + } + + // Length of resuting multi-byte string WITH zero terminator + if (pReturnValue) { + *pReturnValue = n + 1; + } + + // Success + error = 0; + + } else { + + // Could not convert string + error = 1; + + } + */ +#else //DIRENT_USE_ASCII_SHORT_PATHS_ON_WINDOWS +#if defined(_MSC_VER) && _MSC_VER >= 1400 + + /* Microsoft Visual Studio 2005 or later */ + error = mbstowcs_s (pReturnValue, wcstr, sizeInWords, mbstr, count); + +#else + + /* Older Visual Studio or non-Microsoft compiler */ + size_t n; + + /* Convert to wide-character string (or count characters) */ + n = mbstowcs (wcstr, mbstr, sizeInWords); + if (!wcstr || n < count) { + + /* Zero-terminate output buffer */ + if (wcstr && sizeInWords) { + if (n >= sizeInWords) { + n = sizeInWords - 1; + } + wcstr[n] = 0; + } + + /* Length of resuting multi-byte string WITH zero terminator */ + if (pReturnValue) { + *pReturnValue = n + 1; + } + + /* Success */ + error = 0; + + } else { + + /* Could not convert string */ + error = 1; + + } + +#endif +#endif //DIRENT_USE_ASCII_SHORT_PATHS_ON_WINDOWS + + return error; +} + +/* Convert wide-character string to multi-byte string */ +static int +dirent_wcstombs_s( + size_t *pReturnValue, + char *mbstr, + size_t sizeInBytes, /* max size of mbstr */ + const wchar_t *wcstr, + size_t count) +{ + int error; + +#ifndef DIRENT_USE_ASCII_SHORT_PATHS_ON_WINDOWS + // we don't use "count" at all: we assume wcstr is zero terminated: + size_t n = (size_t) WideCharToMultiByte (CP_UTF8, 0, wcstr, -1, mbstr, 0,NULL,NULL);//sizeInBytes, NULL, NULL); + if (n==0) { + error = 1; + if (sizeInBytes>0) mbstr[0]='\0'; + if (pReturnValue) *pReturnValue = 0; + } + else if (n<=sizeInBytes) { + error = WideCharToMultiByte (CP_UTF8, 0, wcstr, -1, mbstr, n, NULL, NULL) == 0 ? 1 : 0; + if (pReturnValue) *pReturnValue = n; + } + else { + // Buffer too low: + if (sizeInBytes>0) { + if (sizeInBytes>1) WideCharToMultiByte (CP_UTF8, 0, wcstr, -1, mbstr, sizeInBytes-1, NULL, NULL); + mbstr[sizeInBytes-1] = '\0'; + } + if (pReturnValue) *pReturnValue = sizeInBytes; + error = 1; + } + /* + if (!mbstr || n < count) { + + // Zero-terminate output buffer + if (mbstr && sizeInBytes) { + if (n >= sizeInBytes) { + n = sizeInBytes - 1; + } + mbstr[n] = '\0'; + } + + // Lenght of resulting multi-bytes string WITH zero-terminator + if (pReturnValue) { + *pReturnValue = n + 1; + } + + + // Success + error = 0; + + } else { + + // Cannot convert string + error = 1; + + } + */ +#else //DIRENT_USE_ASCII_SHORT_PATHS_ON_WINDOWS +#if defined(_MSC_VER) && _MSC_VER >= 1400 + + /* Microsoft Visual Studio 2005 or later */ + error = wcstombs_s (pReturnValue, mbstr, sizeInBytes, wcstr, count); + +#else + + /* Older Visual Studio or non-Microsoft compiler */ + size_t n; + + /* Convert to multi-byte string (or count the number of bytes needed) */ + n = wcstombs (mbstr, wcstr, sizeInBytes); + if (!mbstr || n < count) { + + /* Zero-terminate output buffer */ + if (mbstr && sizeInBytes) { + if (n >= sizeInBytes) { + n = sizeInBytes - 1; + } + mbstr[n] = '\0'; + } + + /* Lenght of resulting multi-bytes string WITH zero-terminator */ + if (pReturnValue) { + *pReturnValue = n + 1; + } + + /* Success */ + error = 0; + + } else { + + /* Cannot convert string */ + error = 1; + + } + +#endif +#endif //DIRENT_USE_ASCII_SHORT_PATHS_ON_WINDOWS + return error; +} + +/* Set errno variable */ +static void +dirent_set_errno( + int error) +{ +#if defined(_MSC_VER) && _MSC_VER >= 1400 + + /* Microsoft Visual Studio 2005 and later */ + _set_errno (error); + +#else + + /* Non-Microsoft compiler or older Microsoft compiler */ + errno = error; + +#endif +} + + +// The code of this single method comes from the musl library +// (MIT licensed, Copyright © 2005-2014 Rich Felker, et al.) +# ifndef SIZE_MAX // Can't we just set it to INT_MAX or something like that ? +# define SIZE_MAX_WAS_NOT_DEFINED +# define SIZE_MAX INT_MAX // should be in limits.h AFAIK +# endif //SIZE_MAX +inline static int scandir(const char *path, struct dirent ***res, + int (*sel)(const struct dirent *), + int (*cmp)(const struct dirent **, const struct dirent **)) +{ + DIR *d = opendir(path); + struct dirent *de, **names=0, **tmp; + size_t cnt=0, len=0; + int old_errno = errno; + + if (!d) return -1; + + while ((errno=0), (de = readdir(d))) { + if (sel && !sel(de)) continue; + if (cnt >= len) { + len = 2*len+1; + if (len > SIZE_MAX/sizeof *names) break; + tmp = (dirent**)realloc(names, len * sizeof *names); + if (!tmp) break; + names = tmp; + } + names[cnt] = (dirent*)malloc(de->d_reclen); + if (!names[cnt]) break; + memcpy(names[cnt++], de, de->d_reclen); + } + + closedir(d); + + if (errno) { + if (names) while (cnt-->0) free(names[cnt]); + free(names); + return -1; + } + errno = old_errno; + + if (cmp) qsort(names, cnt, sizeof *names, (int (*)(const void *, const void *))cmp); + *res = names; + return cnt; +} +#ifdef SIZE_MAX_WAS_NOT_DEFINED +# undef SIZE_MAX +# undef SIZE_MAX_WAS_NOT_DEFINED +#endif //SIZE_MAX_WAS_NOT_DEFINED + +// alphasort: Function to compare two `struct dirent's alphabetically. +inline static int alphasort (const struct dirent **e1,const struct dirent **e2) { + return strcmp((*e1)->d_name,(*e2)->d_name); +} + + +#ifdef __cplusplus +} +#endif +#endif /*DIRENT_H*/ + +#endif //#if (!defined(_WIN32) && !defined(_WIN64))