From 1d069e50778a008793412937043fde81683bd129 Mon Sep 17 00:00:00 2001 From: John Keiser Date: Wed, 22 Apr 2020 17:25:45 -0700 Subject: [PATCH 1/2] Split simdjson-flags and cmakecache into include files --- CMakeLists.txt | 143 +-------------------------------- simdjson-flags.cmake | 120 +++++++++++++++++++++++++++ simdjson-user-cmakecache.cmake | 16 ++++ 3 files changed, 138 insertions(+), 141 deletions(-) create mode 100644 simdjson-flags.cmake create mode 100644 simdjson-user-cmakecache.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index eae9261e0..42640bc23 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,11 +1,5 @@ cmake_minimum_required(VERSION 3.9) # CMP0069 NEW -message(STATUS "cmake version ${CMAKE_VERSION}") -if (NOT CMAKE_BUILD_TYPE) - message(STATUS "No build type selected, default to Release") - set(CMAKE_BUILD_TYPE Release CACHE STRING "Choose the type of build." FORCE) -endif() - project(simdjson DESCRIPTION "Parsing gigabytes of JSON per second" LANGUAGES CXX C @@ -18,142 +12,9 @@ set(SIMDJSON_LIB_VERSION "0.3.1" CACHE STRING "simdjson library version") set(SIMDJSON_LIB_SOVERSION "1" CACHE STRING "simdjson library soversion") set(SIMDJSON_GITHUB_REPOSITORY https://github.com/simdjson/simdjson) -if(MSVC) - option(SIMDJSON_BUILD_STATIC "Build a static library" ON) # turning it on disables the production of a dynamic library - option(SIMDJSON_COMPETITION "Compile competitive benchmarks" OFF) -else() - option(SIMDJSON_BUILD_STATIC "Build a static library" OFF) # turning it on disables the production of a dynamic library - option(SIMDJSON_COMPETITION "Compile competitive benchmarks" ON) - option(SIMDJSON_USE_LIBCPP "Use the libc++ library" OFF) -endif() -option(SIMDJSON_GOOGLE_BENCHMARKS "compile the Google Benchmark benchmarks" OFF) - -set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/tools/cmake") - - -# We compile tools, tests, etc. with C++ 17. Override yourself if you need on a target. -set(CMAKE_CXX_STANDARD 17) -set(CMAKE_CXX_STANDARD_REQUIRED ON) -set(CMAKE_CXX_EXTENSIONS OFF) -set(CMAKE_MACOSX_RPATH OFF) -set(CMAKE_THREAD_PREFER_PTHREAD ON) -set(THREADS_PREFER_PTHREAD_FLAG ON) - include(GNUInstallDirs) - -# LTO seems to create all sorts of fun problems. Let us -# disable temporarily. -#include(CheckIPOSupported) -#check_ipo_supported(RESULT ltoresult) -#if(ltoresult) -# set(CMAKE_INTERPROCEDURAL_OPTIMIZATION TRUE) -#endif() - -# -# Flags used by exes and by the simdjson library (project-wide flags) -# -add_library(simdjson-flags INTERFACE) -if(MSVC) - target_compile_options(simdjson-flags INTERFACE /nologo /D_CRT_SECURE_NO_WARNINGS) - target_compile_options(simdjson-flags INTERFACE /WX /W3 /sdl) -else() - target_compile_options(simdjson-flags INTERFACE -fPIC) - if(NOT SIMDJSON_GOOGLE_BENCHMARKS) # Google Benchmark can't be compiled without warnings with -Weffc++ - target_compile_options(simdjson-flags INTERFACE -Weffc++) - endif() - target_compile_options(simdjson-flags INTERFACE -Werror -Wall -Wextra -Wsign-compare -Wshadow -Wwrite-strings -Wpointer-arith -Winit-self -Wconversion -Wno-sign-conversion) -endif() - - -if(SIMDJSON_USE_LIBCPP) - target_link_libraries(simdjson-flags INTERFACE -stdlib=libc++ -lc++abi) - # instead of the above line, we could have used - # set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -stdlib=libc++ -lc++abi") - # The next line is needed empirically. - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -stdlib=libc++") - # we update CMAKE_SHARED_LINKER_FLAGS, this gets updated later as well - set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -lc++abi") -endif(SIMDJSON_USE_LIBCPP) - -# Optional flags -option(SIMDJSON_IMPLEMENTATION_HASWELL "Include the haswell implementation" ON) -if(NOT SIMDJSON_IMPLEMENTATION_HASWELL) - target_compile_definitions(simdjson-flags INTERFACE SIMDJSON_IMPLEMENTATION_HASWELL=0) -endif() -option(SIMDJSON_IMPLEMENTATION_WESTMERE "Include the westmere implementation" ON) -if(NOT SIMDJSON_IMPLEMENTATION_WESTMERE) - target_compile_definitions(simdjson-flags INTERFACE SIMDJSON_IMPLEMENTATION_WESTMERE=0) -endif() -option(SIMDJSON_IMPLEMENTATION_ARM64 "Include the arm64 implementation" ON) -if(NOT SIMDJSON_IMPLEMENTATION_ARM64) - target_compile_definitions(simdjson-flags INTERFACE SIMDJSON_IMPLEMENTATION_ARM64=0) -endif() -option(SIMDJSON_IMPLEMENTATION_FALLBACK "Include the fallback implementation" ON) -if(NOT SIMDJSON_IMPLEMENTATION_FALLBACK) - target_compile_definitions(simdjson-flags INTERFACE SIMDJSON_IMPLEMENTATION_FALLBACK=0) -endif() - -option(SIMDJSON_EXCEPTIONS "Enable simdjson's exception-throwing interface" ON) -if(NOT SIMDJSON_EXCEPTIONS) - message(STATUS "simdjson exception interface turned off. Code that does not check error codes will not compile.") - target_compile_definitions(simdjson-flags INTERFACE SIMDJSON_EXCEPTIONS=0) -endif() - -option(SIMDJSON_ENABLE_THREADS "Enable threaded operation" ON) -if(SIMDJSON_ENABLE_THREADS) - find_package(Threads REQUIRED) - target_link_libraries(simdjson-flags INTERFACE Threads::Threads) -endif() - -option(SIMDJSON_SANITIZE "Sanitize addresses" OFF) -if(SIMDJSON_SANITIZE) - # Not sure which - target_compile_options(simdjson-flags INTERFACE -fsanitize=address -fno-omit-frame-pointer -fsanitize=undefined -fno-sanitize-recover=all) - target_link_libraries(simdjson-flags INTERFACE -fsanitize=address -fno-omit-frame-pointer -fsanitize=undefined -fno-sanitize-recover=all) - - # Ubuntu bug for GCC 5.0+ (safe for all versions) - if (CMAKE_COMPILER_IS_GNUCC) - target_link_libraries(simdjson-flags INTERFACE -fuse-ld=gold) - endif() -endif() - -# prevent shared libraries from depending on Intel provided libraries -if(${CMAKE_C_COMPILER_ID} MATCHES "Intel") # icc / icpc - set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -static-intel") -endif() - -# Workaround for https://gitlab.kitware.com/cmake/cmake/issues/15415#note_633938: -function(export_private_library NAME) - install(TARGETS ${NAME} - EXPORT ${NAME}-config - ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} - LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} - RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} - ) - install(EXPORT ${NAME}-config - FILE ${NAME}-config.cmake - DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/simdjson-private - ) -endfunction() - -export_private_library(simdjson-flags) - -# -# ${SIMDJSON_USER_CMAKECACHE} contains the *user-specified* simdjson options so you can call cmake on -# another branch or repository with the same options. -# -# Not supported on Windows at present, because the only thing that uses it is checkperf, which we -# don't run on Windows. -# -if (NOT MSVC) - set(SIMDJSON_USER_CMAKECACHE ${CMAKE_CURRENT_BINARY_DIR}/.simdjson-user-CMakeCache.txt) - add_custom_command( - OUTPUT ${SIMDJSON_USER_CMAKECACHE} - COMMAND bash -c "grep SIMDJSON_ ${PROJECT_BINARY_DIR}/CMakeCache.txt | grep -v SIMDJSON_LIB_ > ${SIMDJSON_USER_CMAKECACHE}" - VERBATIM # Makes it not do weird escaping with the command - ) - add_custom_target(simdjson-user-cmakecache ALL DEPENDS ${SIMDJSON_USER_CMAKECACHE}) -endif() +include(simdjson-flags.cmake) +include(simdjson-user-cmakecache.cmake) # # Set up test data diff --git a/simdjson-flags.cmake b/simdjson-flags.cmake new file mode 100644 index 000000000..c2d308083 --- /dev/null +++ b/simdjson-flags.cmake @@ -0,0 +1,120 @@ +if (NOT CMAKE_BUILD_TYPE) + message(STATUS "No build type selected, default to Release") + set(CMAKE_BUILD_TYPE Release CACHE STRING "Choose the type of build." FORCE) +endif() + +if(MSVC) + option(SIMDJSON_BUILD_STATIC "Build a static library" ON) # turning it on disables the production of a dynamic library + option(SIMDJSON_COMPETITION "Compile competitive benchmarks" OFF) +else() + option(SIMDJSON_BUILD_STATIC "Build a static library" OFF) # turning it on disables the production of a dynamic library + option(SIMDJSON_COMPETITION "Compile competitive benchmarks" ON) + option(SIMDJSON_USE_LIBCPP "Use the libc++ library" OFF) +endif() +option(SIMDJSON_GOOGLE_BENCHMARKS "compile the Google Benchmark benchmarks" OFF) + +set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/tools/cmake") + +# We compile tools, tests, etc. with C++ 17. Override yourself if you need on a target. +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED ON) +set(CMAKE_CXX_EXTENSIONS OFF) +set(CMAKE_MACOSX_RPATH OFF) +set(CMAKE_THREAD_PREFER_PTHREAD ON) +set(THREADS_PREFER_PTHREAD_FLAG ON) + +# LTO seems to create all sorts of fun problems. Let us +# disable temporarily. +#include(CheckIPOSupported) +#check_ipo_supported(RESULT ltoresult) +#if(ltoresult) +# set(CMAKE_INTERPROCEDURAL_OPTIMIZATION TRUE) +#endif() + +# +# Flags used by exes and by the simdjson library (project-wide flags) +# +add_library(simdjson-flags INTERFACE) +if(MSVC) + target_compile_options(simdjson-flags INTERFACE /nologo /D_CRT_SECURE_NO_WARNINGS) + target_compile_options(simdjson-flags INTERFACE /WX /W3 /sdl) +else() + target_compile_options(simdjson-flags INTERFACE -fPIC) + if (NOT SIMDJSON_GOOGLE_BENCHMARKS) # Google Benchmark can't be compiled without warnings with -Weffc++ + target_compile_options(simdjson-flags INTERFACE -Weffc++) + endif() + target_compile_options(simdjson-flags INTERFACE -Werror -Wall -Wextra -Wsign-compare -Wshadow -Wwrite-strings -Wpointer-arith -Winit-self -Wconversion -Wno-sign-conversion) +endif() + +# Optional flags +option(SIMDJSON_IMPLEMENTATION_HASWELL "Include the haswell implementation" ON) +if(NOT SIMDJSON_IMPLEMENTATION_HASWELL) + target_compile_definitions(simdjson-flags INTERFACE SIMDJSON_IMPLEMENTATION_HASWELL=0) +endif() +option(SIMDJSON_IMPLEMENTATION_WESTMERE "Include the westmere implementation" ON) +if(NOT SIMDJSON_IMPLEMENTATION_WESTMERE) + target_compile_definitions(simdjson-flags INTERFACE SIMDJSON_IMPLEMENTATION_WESTMERE=0) +endif() +option(SIMDJSON_IMPLEMENTATION_ARM64 "Include the arm64 implementation" ON) +if(NOT SIMDJSON_IMPLEMENTATION_ARM64) + target_compile_definitions(simdjson-flags INTERFACE SIMDJSON_IMPLEMENTATION_ARM64=0) +endif() +option(SIMDJSON_IMPLEMENTATION_FALLBACK "Include the fallback implementation" ON) +if(NOT SIMDJSON_IMPLEMENTATION_FALLBACK) + target_compile_definitions(simdjson-flags INTERFACE SIMDJSON_IMPLEMENTATION_FALLBACK=0) +endif() + +option(SIMDJSON_EXCEPTIONS "Enable simdjson's exception-throwing interface" ON) +if(NOT SIMDJSON_EXCEPTIONS) + message(STATUS "simdjson exception interface turned off. Code that does not check error codes will not compile.") + target_compile_definitions(simdjson-flags INTERFACE SIMDJSON_EXCEPTIONS=0) +endif() + +option(SIMDJSON_ENABLE_THREADS "Enable threaded operation" ON) +if(SIMDJSON_ENABLE_THREADS) + find_package(Threads REQUIRED) + target_link_libraries(simdjson-flags INTERFACE Threads::Threads) +endif() + +option(SIMDJSON_SANITIZE "Sanitize addresses" OFF) +if(SIMDJSON_SANITIZE) + # Not sure which + target_compile_options(simdjson-flags INTERFACE -fsanitize=address -fno-omit-frame-pointer -fsanitize=undefined -fno-sanitize-recover=all) + target_link_libraries(simdjson-flags INTERFACE -fsanitize=address -fno-omit-frame-pointer -fsanitize=undefined -fno-sanitize-recover=all) + + # Ubuntu bug for GCC 5.0+ (safe for all versions) + if (CMAKE_COMPILER_IS_GNUCC) + target_link_libraries(simdjson-flags INTERFACE -fuse-ld=gold) + endif() +endif() + +if(SIMDJSON_USE_LIBCPP) + target_link_libraries(simdjson-flags INTERFACE -stdlib=libc++ -lc++abi) + # instead of the above line, we could have used + # set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -stdlib=libc++ -lc++abi") + # The next line is needed empirically. + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -stdlib=libc++") + # we update CMAKE_SHARED_LINKER_FLAGS, this gets updated later as well + set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -lc++abi") +endif(SIMDJSON_USE_LIBCPP) + +# prevent shared libraries from depending on Intel provided libraries +if(${CMAKE_C_COMPILER_ID} MATCHES "Intel") # icc / icpc + set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -static-intel") +endif() + +# Workaround for https://gitlab.kitware.com/cmake/cmake/issues/15415#note_633938: +function(export_private_library NAME) + install(TARGETS ${NAME} + EXPORT ${NAME}-config + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} + ) + install(EXPORT ${NAME}-config + FILE ${NAME}-config.cmake + DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/simdjson-private + ) +endfunction() + +export_private_library(simdjson-flags) diff --git a/simdjson-user-cmakecache.cmake b/simdjson-user-cmakecache.cmake new file mode 100644 index 000000000..cdc6a5808 --- /dev/null +++ b/simdjson-user-cmakecache.cmake @@ -0,0 +1,16 @@ +# +# ${SIMDJSON_USER_CMAKECACHE} contains the *user-specified* simdjson options so you can call cmake on +# another branch or repository with the same options. +# +# Not supported on Windows at present, because the only thing that uses it is checkperf, which we +# don't run on Windows. +# +if (NOT MSVC) + set(SIMDJSON_USER_CMAKECACHE ${CMAKE_CURRENT_BINARY_DIR}/.simdjson-user-CMakeCache.txt) + add_custom_command( + OUTPUT ${SIMDJSON_USER_CMAKECACHE} + COMMAND bash -c "grep SIMDJSON_ ${PROJECT_BINARY_DIR}/CMakeCache.txt | grep -v SIMDJSON_LIB_ > ${SIMDJSON_USER_CMAKECACHE}" + VERBATIM # Makes it not do weird escaping with the command + ) + add_custom_target(simdjson-user-cmakecache ALL DEPENDS ${SIMDJSON_USER_CMAKECACHE}) +endif() From 92f3bb89c3bbbb458f7950c15c11e632bf4ad505 Mon Sep 17 00:00:00 2001 From: John Keiser Date: Thu, 23 Apr 2020 16:09:44 -0700 Subject: [PATCH 2/2] Remove Makefile and update CI not to use it --- .circleci/config.yml | 21 +- .drone.yml | 846 ++++++++++++++++--------------------------- HACKING.md | 48 +-- Makefile | 326 ----------------- 4 files changed, 324 insertions(+), 917 deletions(-) delete mode 100644 Makefile diff --git a/.circleci/config.yml b/.circleci/config.yml index 71d262730..660634bf7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -48,15 +48,6 @@ commands: - run: apt-get update -qq - run: apt-get install -y clang build-essential git - make_test: - steps: - - checkout - - run: make $BUILD_FLAGS - - run: ./json2json -h # Print out the implementation we're using on this hardware - - run: make amalgamate - - run: make test - - run: make checkperf - install_cmake: steps: - run: apt-get update -qq @@ -170,18 +161,18 @@ jobs: arch-haswell-gcc7: description: Build, run tests and check performance on GCC 7 with -march=haswell executor: gcc7 - environment: { ARCHFLAGS: -march=haswell } - steps: [ make_test ] + environment: { CXXFLAGS: -march=haswell } + steps: [ install_cmake, cmake_test ] arch-nehalem-gcc7: description: Build, run tests and check performance on GCC 7 with -march=nehalem executor: gcc7 - environment: { ARCHFLAGS: -march=nehalem } - steps: [ make_test ] + environment: { CXXFLAGS: -march=nehalem } + steps: [ install_cmake, cmake_test ] no-computed-goto-gcc7: description: Build, run tests and check performance on GCC 7 with -DSIMDJSON_NO_COMPUTED_GOTO=true executor: gcc7 - environment: { EXTRAFLAGS: -DSIMDJSON_NO_COMPUTED_GOTO=true } - steps: [ make_test ] + environment: { CXXFLAGS: -DSIMDJSON_NO_COMPUTED_GOTO=true } + steps: [ install_cmake, cmake_test ] workflows: version: 2.1 diff --git a/.drone.yml b/.drone.yml index 2861be77e..698b72d02 100644 --- a/.drone.yml +++ b/.drone.yml @@ -1,207 +1,321 @@ kind: pipeline -name: x64-quicktests-libc - -platform: - os: linux - arch: amd64 - +name: gcc8 +platform: { os: linux, arch: amd64 } steps: -- name: quicktests +- name: Build and Test + image: gcc:8 + environment: + CC: gcc + CXX: g++ + BUILD_FLAGS: -- -j + CTEST_FLAGS: -j4 --output-on-failure + commands: + - apt-get update -qq + - apt-get install -y cmake + - mkdir build + - cd build + - cmake $CMAKE_FLAGS .. + - cmake --build . $BUILD_FLAGS + - ctest $CTEST_FLAGS -L acceptance -LE per_implementation + - SIMDJSON_FORCE_IMPLEMENTATION=haswell ctest $CTEST_FLAGS -L per_implementation + - SIMDJSON_FORCE_IMPLEMENTATION=westmere ctest $CTEST_FLAGS -L per_implementation + - SIMDJSON_FORCE_IMPLEMENTATION=fallback ctest $CTEST_FLAGS -L per_implementation + - ctest $CTEST_FLAGS -LE "acceptance|per_implementation" # Everything we haven't run yet, run now. +--- +kind: pipeline +name: clang6 +platform: { os: linux, arch: amd64 } +steps: +- name: Build and Test + image: ubuntu:18.04 + environment: + CC: clang-6.0 + CXX: clang++-6.0 + BUILD_FLAGS: -- -j + CTEST_FLAGS: -j4 --output-on-failure + commands: + - apt-get update -qq + - apt-get install -y clang cmake git + - mkdir build + - cd build + - cmake $CMAKE_FLAGS .. + - cmake --build . $BUILD_FLAGS + - ctest $CTEST_FLAGS -L acceptance -LE per_implementation + - SIMDJSON_FORCE_IMPLEMENTATION=haswell ctest $CTEST_FLAGS -L per_implementation + - SIMDJSON_FORCE_IMPLEMENTATION=westmere ctest $CTEST_FLAGS -L per_implementation + - SIMDJSON_FORCE_IMPLEMENTATION=fallback ctest $CTEST_FLAGS -L per_implementation + - ctest $CTEST_FLAGS -LE "acceptance|per_implementation" # Everything we haven't run yet, run now. +--- +kind: pipeline +name: dynamic-gcc8 +platform: { os: linux, arch: amd64 } +steps: +- name: Build and Test + image: gcc:8 + environment: + CC: gcc + CXX: g++ + BUILD_FLAGS: -- -j + CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=OFF + CTEST_FLAGS: -j4 --output-on-failure + commands: + - apt-get update -qq + - apt-get install -y cmake + - mkdir build + - cd build + - cmake $CMAKE_FLAGS .. + - cmake --build . $BUILD_FLAGS + - ctest $CTEST_FLAGS +--- +kind: pipeline +name: dynamic-clang6 +platform: { os: linux, arch: amd64 } +steps: +- name: Build and Test + image: ubuntu:18.04 + environment: + CC: clang-6.0 + CXX: clang++-6.0 + CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=OFF + BUILD_FLAGS: -- -j + CTEST_FLAGS: -j4 --output-on-failure + commands: + - apt-get update -qq + - apt-get install -y clang cmake git + - mkdir build + - cd build + - cmake $CMAKE_FLAGS .. + - cmake --build . $BUILD_FLAGS + - ctest $CTEST_FLAGS +--- +kind: pipeline +name: sanitize-gcc8 +platform: { os: linux, arch: amd64 } +steps: +- name: Build and Test + image: gcc:8 + environment: + BUILD_FLAGS: -- -j + CTEST_FLAGS: -j4 --output-on-failure + CC: gcc + CXX: g++ + commands: + - apt-get update -qq + - apt-get install -y cmake + - mkdir build + - cd build + - cmake $CMAKE_FLAGS .. + - cmake --build . $BUILD_FLAGS + - ASAN_OPTIONS="detect_leaks=0" ctest $CTEST_FLAGS -L acceptance -LE per_implementation + - SIMDJSON_FORCE_IMPLEMENTATION=haswell ASAN_OPTIONS="detect_leaks=0" ctest $CTEST_FLAGS -L per_implementation + - SIMDJSON_FORCE_IMPLEMENTATION=westmere ASAN_OPTIONS="detect_leaks=0" ctest $CTEST_FLAGS -L per_implementation + - SIMDJSON_FORCE_IMPLEMENTATION=fallback ASAN_OPTIONS="detect_leaks=0" ctest $CTEST_FLAGS -L per_implementation + - ASAN_OPTIONS="detect_leaks=0" ctest $CTEST_FLAGS -LE "acceptance|per_implementation" # Everything we haven't run yet, run now. +--- +kind: pipeline +name: sanitize-clang6 +platform: { os: linux, arch: amd64 } +steps: +- name: Build and Test + image: ubuntu:18.04 + environment: + CC: clang-6.0 + CXX: clang++-6.0 + CMAKE_FLAGS: -DSIMDJSON_SANITIZE=ON + BUILD_FLAGS: -- -j + CTEST_FLAGS: -j4 --output-on-failure + commands: + - apt-get update -qq + - apt-get install -y clang cmake git + - mkdir build + - cd build + - cmake $CMAKE_FLAGS .. + - cmake --build . $BUILD_FLAGS + - ASAN_OPTIONS="detect_leaks=0" ctest $CTEST_FLAGS -L acceptance -LE per_implementation + - SIMDJSON_FORCE_IMPLEMENTATION=haswell ASAN_OPTIONS="detect_leaks=0" ctest $CTEST_FLAGS -L per_implementation + - SIMDJSON_FORCE_IMPLEMENTATION=westmere ASAN_OPTIONS="detect_leaks=0" ctest $CTEST_FLAGS -L per_implementation + - SIMDJSON_FORCE_IMPLEMENTATION=fallback ASAN_OPTIONS="detect_leaks=0" ctest $CTEST_FLAGS -L per_implementation + - ASAN_OPTIONS="detect_leaks=0" ctest $CTEST_FLAGS -LE "acceptance|per_implementation" # Everything we haven't run yet, run now. +--- +kind: pipeline +name: arm64-gcc8 +platform: { os: linux, arch: arm64 } +steps: +- name: Build and Test + image: gcc:8 + environment: + BUILD_FLAGS: -- -j + CTEST_FLAGS: -j4 --output-on-failure + CC: gcc + CXX: g++ + commands: + - apt-get update -qq + - apt-get install -y cmake + - mkdir build + - cd build + - cmake $CMAKE_FLAGS .. + - cmake --build . $BUILD_FLAGS + - ctest $CTEST_FLAGS -L acceptance -LE per_implementation + - SIMDJSON_FORCE_IMPLEMENTATION=arm64 ctest $CTEST_FLAGS -L per_implementation + - SIMDJSON_FORCE_IMPLEMENTATION=fallback ctest $CTEST_FLAGS -L per_implementation + - ctest $CTEST_FLAGS -LE "acceptance|per_implementation" # Everything we haven't run yet, run now. +--- +kind: pipeline +name: arm64-clang6 +platform: { os: linux, arch: arm64 } +steps: +- name: Build and Test + image: ubuntu:18.04 + environment: + CC: clang-6.0 + CXX: clang++-6.0 + CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=OFF + BUILD_FLAGS: -- -j + CTEST_FLAGS: -j4 --output-on-failure + commands: + - apt-get update -qq + - apt-get install -y clang cmake git + - mkdir build + - cd build + - cmake $CMAKE_FLAGS .. + - cmake --build . $BUILD_FLAGS + - ctest $CTEST_FLAGS +--- +kind: pipeline +name: arm64-dynamic-gcc8 +platform: { os: linux, arch: arm64 } +steps: +- name: Build and Test + image: gcc:8 + environment: + CC: gcc + CXX: g++ + BUILD_FLAGS: -- -j + CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=OFF + CTEST_FLAGS: -j4 --output-on-failure + commands: + - apt-get update -qq + - apt-get install -y cmake + - mkdir build + - cd build + - cmake $CMAKE_FLAGS .. + - cmake --build . $BUILD_FLAGS + - ctest $CTEST_FLAGS +--- +kind: pipeline +name: arm64-dynamic-clang6 +platform: { os: linux, arch: arm64 } +steps: +- name: Build and Test + image: ubuntu:18.04 + environment: + CC: clang-6.0 + CXX: clang++-6.0 + CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=OFF + BUILD_FLAGS: -- -j + CTEST_FLAGS: -j4 --output-on-failure + commands: + - apt-get update -qq + - apt-get install -y clang cmake git + - mkdir build + - cd build + - cmake $CMAKE_FLAGS .. + - cmake --build . $BUILD_FLAGS + - ctest $CTEST_FLAGS +--- +kind: pipeline +name: arm64-sanitize-gcc8 +platform: { os: linux, arch: arm64 } +steps: +- name: Build and Test + image: gcc:8 + environment: + BUILD_FLAGS: -- -j + CTEST_FLAGS: -j4 --output-on-failure -E checkperf + CC: gcc + CXX: g++ + commands: + - apt-get update -qq + - apt-get install -y cmake + - mkdir build + - cd build + - cmake $CMAKE_FLAGS .. + - cmake --build . $BUILD_FLAGS + - ASAN_OPTIONS="detect_leaks=0" ctest $CTEST_FLAGS -L acceptance -LE per_implementation + - SIMDJSON_FORCE_IMPLEMENTATION=arm64 ASAN_OPTIONS="detect_leaks=0" ctest $CTEST_FLAGS -L per_implementation + - SIMDJSON_FORCE_IMPLEMENTATION=fallback ASAN_OPTIONS="detect_leaks=0" ctest $CTEST_FLAGS -L per_implementation + - ASAN_OPTIONS="detect_leaks=0" ctest $CTEST_FLAGS -LE "acceptance|per_implementation" # Everything we haven't run yet, run now. +--- +kind: pipeline +name: arm64-sanitize-clang6 +platform: { os: linux, arch: arm64 } +steps: +- name: Build and Test + image: ubuntu:18.04 + environment: + CC: clang-6.0 + CXX: clang++-6.0 + CMAKE_FLAGS: -DSIMDJSON_SANITIZE=ON + BUILD_FLAGS: -- -j + CTEST_FLAGS: -j4 --output-on-failure -E checkperf + commands: + - apt-get update -qq + - apt-get install -y clang cmake git + - mkdir build + - cd build + - cmake $CMAKE_FLAGS .. + - cmake --build . $BUILD_FLAGS + - ASAN_OPTIONS="detect_leaks=0" ctest $CTEST_FLAGS -L acceptance -LE per_implementation + - SIMDJSON_FORCE_IMPLEMENTATION=arm64 ASAN_OPTIONS="detect_leaks=0" ctest $CTEST_FLAGS -L per_implementation + - SIMDJSON_FORCE_IMPLEMENTATION=fallback ASAN_OPTIONS="detect_leaks=0" ctest $CTEST_FLAGS -L per_implementation + - ASAN_OPTIONS="detect_leaks=0" ctest $CTEST_FLAGS -LE "acceptance|per_implementation" # Everything we haven't run yet, run now. +--- +kind: pipeline +name: libcpp-clang8 +platform: { os: linux, arch: amd64 } +steps: +- name: Build and Test image: conanio/clang8 user: root - commands: [ EXTRAFLAGS=-stdlib=libc++ make quicktests ] + environment: + CC: clang-8 + CXX: clang++-8 + BUILD_FLAGS: -- -j + CTEST_FLAGS: -j4 --output-on-failure -E checkperf + CXXFLAGS: -stdlib=libc++ + commands: + - apt-get update -qq + - apt-get install -y cmake + - mkdir build + - cd build + - cmake $CMAKE_FLAGS .. + - cmake --build . $BUILD_FLAGS + - ctest $CTEST_FLAGS --- kind: pipeline -name: x64-quicktests - -platform: - os: linux - arch: amd64 - +name: noexceptions-gcc8 +platform: { os: linux, arch: amd64 } steps: -- name: quicktests - image: gcc:8 - commands: [ make quicktests ] ---- -kind: pipeline -name: x64-checkperf - -platform: - os: linux - arch: amd64 - -steps: -- name: checkperf +- name: Build and Test image: gcc:8 environment: - CHECKPERF_REPOSITORY: https://github.com/lemire/simdjson - commands: [ sed '/^$/Q' /proc/cpuinfo, make checkperf ] ---- -kind: pipeline -name: x64-build - -platform: - os: linux - arch: amd64 - -steps: -- name: build - image: gcc:8 - commands: [ make, make amalgamate_test ] ---- -kind: pipeline -name: x64-slowtests - -platform: - os: linux - arch: amd64 - -steps: -- name: slowtests - image: gcc:8 - commands: [ make slowtests ] ---- -kind: pipeline -name: x64-noexceptions-quicktests - -platform: - os: linux - arch: amd64 - -steps: -- name: quicktests - image: gcc:8 - environment: - EXTRA_FLAGS: -fno-exceptions - commands: [ make quicktests ] ---- -kind: pipeline -name: x64-noexceptions-build - -platform: - os: linux - arch: amd64 - -steps: -- name: build - image: gcc:8 - environment: - EXTRA_FLAGS: -fno-exceptions - commands: [ make, make amalgamate_test ] ---- -kind: pipeline -name: x64-noexceptions-slowtests - -platform: - os: linux - arch: amd64 - -steps: -- name: slowtests - image: gcc:8 - environment: - EXTRA_FLAGS: -fno-exceptions - commands: [ make slowtests ] ---- -kind: pipeline -name: arm64-quicktests - -platform: - os: linux - arch: arm64 - -steps: -- name: quicktests - image: gcc:8 - commands: [ make quicktests ] ---- -kind: pipeline -name: arm64-checkperf - -platform: - os: linux - arch: arm64 - -steps: -- name: checkperf - image: gcc:8 - environment: - CHECKPERF_REPOSITORY: https://github.com/lemire/simdjson - commands: [ cat /proc/cpuinfo, make checkperf ] ---- -kind: pipeline -name: arm64-build - -platform: - os: linux - arch: arm64 - -steps: -- name: build - image: gcc:8 - commands: [ make, make amalgamate_test ] ---- -kind: pipeline -name: arm64-slowtests - -platform: - os: linux - arch: arm64 - -steps: -- name: slowtests - image: gcc:8 - commands: [ make slowtests ] ---- -kind: pipeline -name: x64-amalgamated-build - -platform: - os: linux - arch: amd64 - -steps: -- name: build - image: gcc:8 - environment: - SIMDJSON_TEST_AMALGAMATED_HEADERS: 1 - commands: [ make amalgamate, make ] ---- -kind: pipeline -name: x64-amalgamated-quicktests - -platform: - os: linux - arch: amd64 - -steps: -- name: quicktests - image: gcc:8 - environment: - SIMDJSON_TEST_AMALGAMATED_HEADERS: 1 - commands: [ make amalgamate, make quicktests ] ---- -kind: pipeline -name: x64-amalgamated-slowtests - -platform: - os: linux - arch: amd64 - -steps: -- name: slowtests - image: gcc:8 - environment: - SIMDJSON_TEST_AMALGAMATED_HEADERS: 1 - commands: [ make amalgamate, make slowtests ] + CC: gcc + CXX: g++ + BUILD_FLAGS: -- -j + CMAKE_FLAGS: -DSIMDJSON_EXCEPTIONS=OFF + CTEST_FLAGS: -j4 --output-on-failure -E checkperf + commands: + - apt-get update -qq + - apt-get install -y cmake + - mkdir build + - cd build + - cmake $CMAKE_FLAGS .. + - cmake --build . $BUILD_FLAGS + - ctest $CTEST_FLAGS --- kind: pipeline name: stylecheck - -platform: - os: linux - arch: amd64 - +platform: { os: linux, arch: amd64 } steps: - name: Build and Test image: ubuntu:18.04 @@ -209,351 +323,3 @@ steps: - apt-get update -y - apt-get install -y python clang-format - ./style/run-clang-format.py -r include/ benchmark/ src/ tests/ ---- -kind: pipeline -name: amd64_clang_cmake_dynamic - -platform: - os: linux - arch: amd64 - -steps: -- name: Build and Test - image: ubuntu:18.04 - environment: - CC: clang - CXX: clang++ - CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=OFF - commands: - - apt-get update -qq - - apt-get install -y clang make cmake git - - $CC --version - - mkdir build && cd build - - cmake $CMAKE_FLAGS .. - - make -j - - ctest -j4 --output-on-failure -E checkperf ---- -kind: pipeline -name: amd64_clang_cmake_no_exceptions - -platform: - os: linux - arch: amd64 - -steps: -- name: Build and Test - image: ubuntu:18.04 - environment: - CC: clang - CXX: clang++ - CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=OFF - commands: - - apt-get update -qq - - apt-get install -y clang make cmake git - - $CC --version - - mkdir build && cd build - - cmake $CMAKE_FLAGS .. - - make -j - - ctest -j4 --output-on-failure -E checkperf ---- -kind: pipeline -name: amd64_clang_cmake_static - -platform: - os: linux - arch: amd64 - -steps: -- name: Build and Test - image: ubuntu:18.04 - environment: - CC: clang - CXX: clang++ - CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=ON - commands: - - apt-get update -qq - - apt-get install -y clang make cmake git - - $CC --version - - mkdir build && cd build - - cmake $CMAKE_FLAGS .. - - make -j - - ctest -j4 --output-on-failure -E checkperf ---- -kind: pipeline -name: amd64_gcc_cmake_static - -platform: - os: linux - arch: amd64 - -steps: -- name: Build and Test - image: gcc:8 - environment: - CC: gcc - CXX: g++ - CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=ON - commands: - - apt-get update -qq - - apt-get install -y cmake - - $CC --version - - mkdir build && cd build - - cmake $CMAKE_FLAGS .. - - make -j - - ctest -j4 --output-on-failure -E checkperf ---- -kind: pipeline -name: amd64_gcc_cmake_dynamic - -platform: - os: linux - arch: amd64 - -steps: -- name: Build and Test - image: gcc:8 - environment: - CC: gcc - CXX: g++ - CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=OFF - commands: - - apt-get update -qq - - apt-get install -y cmake - - $CC --version - - mkdir build && cd build - - cmake $CMAKE_FLAGS .. - - make -j - - ctest -j4 --output-on-failure -E checkperf ---- -kind: pipeline -name: arm64_clang_cmake_dynamic - -platform: - os: linux - arch: arm64 - -steps: -- name: Build and Test - image: ubuntu:18.04 - environment: - CC: clang - CXX: clang++ - CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=OFF - commands: - - apt-get update -qq - - apt-get install -y clang make cmake git - - $CC --version - - mkdir build && cd build - - cmake $CMAKE_FLAGS .. - - make -j - - ctest -j4 --output-on-failure -E checkperf ---- -kind: pipeline -name: arm64_gcc_cmake_dynamic - -platform: - os: linux - arch: arm64 - -steps: -- name: Build and Test - image: gcc:8 - environment: - CC: gcc - CXX: g++ - CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=OFF - commands: - - apt-get update -qq - - apt-get install -y cmake - - $CC --version - - mkdir build && cd build - - cmake $CMAKE_FLAGS .. - - make -j - - ctest -j4 --output-on-failure -E checkperf ---- -kind: pipeline -name: arm64_clang_cmake_static - -platform: - os: linux - arch: arm64 - -steps: -- name: Build and Test - image: ubuntu:18.04 - environment: - CC: clang - CXX: clang++ - CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=ON - commands: - - apt-get update -qq - - apt-get install -y clang make cmake git - - $CC --version - - mkdir build && cd build - - cmake $CMAKE_FLAGS .. - - make -j - - ctest -j4 --output-on-failure -E checkperf ---- -kind: pipeline -name: arm64_gcc_cmake_static - -platform: - os: linux - arch: arm64 - -steps: -- name: Build and Test - image: gcc:8 - environment: - CC: gcc - CXX: g++ - CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=ON - commands: - - apt-get update -qq - - apt-get install -y cmake - - $CC --version - - mkdir build && cd build - - cmake $CMAKE_FLAGS .. - - make -j - - ctest -j4 --output-on-failure -E checkperf - ---- -kind: pipeline -name: arm64_clang_cmake_sanitize - -platform: - os: linux - arch: arm64 - -steps: -- name: Build and Test - image: ubuntu:18.04 - environment: - CC: clang - CXX: clang++ - CMAKE_FLAGS: -DSIMDJSON_SANITIZE=ON - commands: - - apt-get update -qq - - apt-get install -y clang make cmake git - - $CC --version - - mkdir build && cd build - - cmake $CMAKE_FLAGS .. - - make -j - - ASAN_OPTIONS="detect_leaks=0" ctest -j4 --output-on-failure -E checkperf ---- -kind: pipeline -name: arm64_gcc_cmake_sanitize - -platform: - os: linux - arch: arm64 - -steps: -- name: Build and Test - image: gcc:8 - environment: - CC: gcc - CXX: g++ - CMAKE_FLAGS: -DSIMDJSON_SANITIZE=ON - commands: - - apt-get update -qq - - apt-get install -y cmake - - $CC --version - - mkdir build && cd build - - cmake $CMAKE_FLAGS .. - - make -j - - ASAN_OPTIONS="detect_leaks=0" ctest -j4 --output-on-failure -E checkperf - ---- -kind: pipeline -name: amd64_clang_cmake_sanitize - -platform: - os: linux - arch: amd64 - -steps: -- name: Build and Test - image: ubuntu:18.04 - environment: - CC: clang - CXX: clang++ - CMAKE_FLAGS: -DSIMDJSON_SANITIZE=ON - commands: - - apt-get update -y - - apt-get install -y make $CC g++ cmake git - - $CC --version - - mkdir build && cd build - - cmake $CMAKE_FLAGS .. - - make -j - - ASAN_OPTIONS="detect_leaks=0" ctest -j4 --output-on-failure -E checkperf ---- -kind: pipeline -name: amd64_gcc_cmake_sanitize - -platform: - os: linux - arch: amd64 - -steps: -- name: Build and Test - image: gcc:8 - environment: - CC: gcc - CXX: g++ - CMAKE_FLAGS: -DSIMDJSON_SANITIZE=ON - commands: - - apt-get update -qq - - apt-get install -y cmake - - $CC --version - - mkdir build && cd build - - cmake $CMAKE_FLAGS .. - - make -j - - ASAN_OPTIONS="detect_leaks=0" ctest -j4 --output-on-failure -E checkperf - ---- -kind: pipeline -name: amd64_clang_cmake_sanitize_noavx - -platform: - os: linux - arch: amd64 - -steps: -- name: Build and Test - image: ubuntu:18.04 - environment: - CC: clang - CXX: clang++ - CMAKE_FLAGS: -DSIMDJSON_SANITIZE=ON -DSIMDJSON_IMPLEMENTATION_HASWELL=OFF - commands: - - apt-get update -qq - - apt-get install -y clang make cmake git - - $CC --version - - mkdir build && cd build - - cmake $CMAKE_FLAGS .. - - make -j - - ASAN_OPTIONS="detect_leaks=0" ctest -j4 --output-on-failure -E checkperf ---- -kind: pipeline -name: amd64_gcc_cmake_sanitize_noavx - -platform: - os: linux - arch: amd64 - -steps: -- name: Build and Test - image: gcc:8 - environment: - CC: gcc - CXX: g++ - CMAKE_FLAGS: -DSIMDJSON_SANITIZE=ON -DSIMDJSON_IMPLEMENTATION_HASWELL=OFF - commands: - - apt-get update -qq - - apt-get install -y cmake - - $CC --version - - mkdir build && cd build - - cmake $CMAKE_FLAGS .. - - make -j - - ASAN_OPTIONS="detect_leaks=0" ctest -j4 --output-on-failure -E checkperf diff --git a/HACKING.md b/HACKING.md index ee8a7f047..a8239aa8d 100644 --- a/HACKING.md +++ b/HACKING.md @@ -9,8 +9,7 @@ Directory Structure and Source simdjson's source structure, from the top level, looks like this: -* **Makefile:** The main Makefile for Linux. This is not the same as CMakeLists.txt. -* **CMakeLists.txt:** A Makefile generator for non-default cases and options. +* **CMakeLists.txt:** The main build system. * **include:** User-facing declarations and inline definitions (most user-facing functions are inlined). * simdjson.h: A "master include" that includes files from include/simdjson/. This is equivalent to the distributed simdjson.h. @@ -70,7 +69,10 @@ simdjson.h and simdjson.cpp are not always up to date in master. To ensure you h you can regenerate them by running this at the top level: ```bash -make amalgamate +mkdir build +cd build +cmake .. +cmake --build . --target amalgamate ``` The amalgamator is at `amalgamate.sh` at the top level. It generates singleheader/simdjson.h by @@ -79,32 +81,6 @@ point it gets included (but only once per header). singleheader/simdjson.cpp is src/simdjson.cpp the same way, except files under generic/ may be included and copy/pasted multiple times. -### Usage (old-school Makefile on platforms like Linux or macOS) - -Requirements: recent clang or gcc, and make. We recommend at least GNU GCC/G++ 7 or LLVM clang 6. A 64-bit system like Linux or macOS is expected. - -To test: - -``` -make -make test -``` - -To run benchmarks: - -``` -make parse -./parse jsonexamples/twitter.json -``` - -Under Linux, the `parse` command gives a detailed analysis of the performance counters. - -To run comparative benchmarks (with other parsers): - -``` -make benchmark -``` - ### Usage (CMake on 64-bit platforms like Linux or macOS) Requirements: We require a recent version of cmake. On macOS, the easiest way to install cmake might be to use [brew](https://brew.sh) and then type @@ -129,8 +105,8 @@ Building: While in the project repository, do the following: mkdir build cd build cmake .. -make -make test +cmake --build . +ctest ``` CMake will build a library. By default, it builds a shared library (e.g., libsimdjson.so on Linux). @@ -141,8 +117,8 @@ You can build a static library: mkdir buildstatic cd buildstatic cmake -DSIMDJSON_BUILD_STATIC=ON .. -make -make test +cmake --build . +ctest ``` In some cases, you may want to specify your compiler, especially if the default compiler on your system is too old. You may proceed as follows: @@ -153,8 +129,8 @@ mkdir build cd build export CXX=g++-8 CC=gcc-8 cmake .. -make -make test +cmake --build . +ctest ``` linux way: @@ -164,7 +140,7 @@ mkdir build # if necessary cd build cmake .. -DCMAKE_CXX_COMPILER=g++ # or cmake .. -DCMAKE_CXX_COMPILER=clang++ -make -j +cmake --build . ``` diff --git a/Makefile b/Makefile deleted file mode 100644 index b44478bee..000000000 --- a/Makefile +++ /dev/null @@ -1,326 +0,0 @@ -CHECKPERF_BRANCH = master - -.SUFFIXES: -# -.SUFFIXES: .cpp .o .c .h - - -.PHONY: clean cleandist -COREDEPSINCLUDE = -Idependencies/json/single_include -Idependencies/rapidjson/include -Idependencies/sajson/include -Idependencies/cJSON -Idependencies/jsmn -EXTRADEPSINCLUDE = -Idependencies/jsoncppdist -Idependencies/json11 -Idependencies/fastjson/src -Idependencies/fastjson/include -Idependencies/gason/src -Idependencies/ujson4c/3rdparty -Idependencies/ujson4c/src -# users can provide their own additional flags with make EXTRAFLAGS=something -architecture:=$(shell arch) - -#### -# If you want to specify your own target architecture, -# then define ARCHFLAGS. Otherwise, we set good default. -# E.g., type ' ARCHFLAGS="-march=westmere" make parse ' -### - -CXXFLAGS = $(ARCHFLAGS) -std=c++17 -pthread -Wall -Wextra -Wshadow -Ibenchmark/linux -CFLAGS = $(ARCHFLAGS) -Idependencies/ujson4c/3rdparty -Idependencies/ujson4c/src $(EXTRAFLAGS) - -# This is a convenience flag -ifdef SANITIZEGOLD - SANITIZE = 1 - LINKER = gold -endif - -ifdef LINKER - CXXFLAGS += -fuse-ld=$(LINKER) - CFLAGS += -fuse-ld=$(LINKER) -endif - - -# SANITIZE *implies* DEBUG -ifeq ($(MEMSANITIZE),1) - CXXFLAGS += -g3 -O0 -fsanitize=memory -fno-omit-frame-pointer -fsanitize=undefined - CFLAGS += -g3 -O0 -fsanitize=memory -fno-omit-frame-pointer -fsanitize=undefined -else -ifeq ($(SANITIZE),1) - CXXFLAGS += -g3 -O0 -fsanitize=address -fno-omit-frame-pointer -fsanitize=undefined - CFLAGS += -g3 -O0 -fsanitize=address -fno-omit-frame-pointer -fsanitize=undefined -else -ifeq ($(DEBUG),1) - CXXFLAGS += -g3 -O0 - CFLAGS += -g3 -O0 -else -# we opt for -O3 for regular builds - CXXFLAGS += -O3 - CFLAGS += -O3 -endif # ifeq ($(DEBUG),1) -endif # ifeq ($(SANITIZE),1) -endif # ifeq ($(MEMSANITIZE),1) - -# Headers and sources -SRCHEADERS_GENERIC=src/generic/atomparsing.h src/generic/numberparsing.h src/generic/json_scanner.h src/generic/json_string_scanner.h src/generic/json_structural_indexer.h src/generic/json_minifier.h src/generic/buf_block_reader.h src/generic/stage2_build_tape.h src/generic/stringparsing.h src/generic/stage2_streaming_build_tape.h src/generic/utf8_fastvalidate_algorithm.h src/generic/utf8_lookup_algorithm.h src/generic/utf8_lookup2_algorithm.h src/generic/utf8_range_algorithm.h src/generic/utf8_zwegner_algorithm.h -SRCHEADERS_ARM64= src/arm64/bitmanipulation.h src/arm64/bitmask.h src/arm64/intrinsics.h src/arm64/numberparsing.h src/arm64/simd.h src/arm64/stage1_find_marks.h src/arm64/stage2_build_tape.h src/arm64/stringparsing.h -SRCHEADERS_HASWELL= src/haswell/bitmanipulation.h src/haswell/bitmask.h src/haswell/intrinsics.h src/haswell/numberparsing.h src/haswell/simd.h src/haswell/stage1_find_marks.h src/haswell/stage2_build_tape.h src/haswell/stringparsing.h -SRCHEADERS_FALLBACK= src/fallback/bitmanipulation.h src/fallback/implementation.h src/fallback/numberparsing.h src/fallback/stage1_find_marks.h src/fallback/stage2_build_tape.h src/fallback/stringparsing.h -SRCHEADERS_WESTMERE=src/westmere/bitmanipulation.h src/westmere/bitmask.h src/westmere/intrinsics.h src/westmere/numberparsing.h src/westmere/simd.h src/westmere/stage1_find_marks.h src/westmere/stage2_build_tape.h src/westmere/stringparsing.h -SRCHEADERS_SRC=src/isadetection.h src/jsoncharutils.h src/simdprune_tables.h src/error.cpp src/implementation.cpp src/stage1_find_marks.cpp src/stage2_build_tape.cpp src/document_parser_callbacks.h -SRCHEADERS=$(SRCHEADERS_SRC) $(SRCHEADERS_GENERIC) $(SRCHEADERS_ARM64) $(SRCHEADERS_HASWELL) $(SRCHEADERS_WESTMERE) $(SRCHEADERS_FALLBACK) - -INCLUDEHEADERS=include/simdjson.h include/simdjson/common_defs.h include/simdjson/internal/jsonformatutils.h include/simdjson/jsonioutil.h include/simdjson/jsonparser.h include/simdjson/padded_string.h include/simdjson/inline/padded_string.h include/simdjson/document.h include/simdjson/inline/document.h include/simdjson/parsedjson_iterator.h include/simdjson/inline/parsedjson_iterator.h include/simdjson/document_stream.h include/simdjson/inline/document_stream.h include/simdjson/implementation.h include/simdjson/parsedjson.h include/simdjson/portability.h include/simdjson/error.h include/simdjson/inline/error.h include/simdjson/nonstd/string_view.hpp include/simdjson/simdjson_version.h - -ifeq ($(SIMDJSON_TEST_AMALGAMATED_HEADERS),1) - HEADERS=singleheader/simdjson.h - LIBFILES=singleheader/simdjson.cpp - CXXFLAGS += -Isingleheader -else - HEADERS=$(INCLUDEHEADERS) $(SRCHEADERS) - LIBFILES=src/simdjson.cpp - CXXFLAGS += -Isrc -Iinclude -endif - -# We put EXTRAFLAGS after all other CXXFLAGS so they can override if necessary -CXXFLAGS += $(EXTRAFLAGS) - -FEATURE_JSON_FILES=jsonexamples/generated/0-structurals-full.json jsonexamples/generated/0-structurals-miss.json jsonexamples/generated/0-structurals.json jsonexamples/generated/15-structurals-full.json jsonexamples/generated/15-structurals-miss.json jsonexamples/generated/15-structurals.json jsonexamples/generated/23-structurals-full.json jsonexamples/generated/23-structurals-miss.json jsonexamples/generated/23-structurals.json jsonexamples/generated/7-structurals-full.json jsonexamples/generated/7-structurals-miss.json jsonexamples/generated/7-structurals.json jsonexamples/generated/escape-full.json jsonexamples/generated/escape-miss.json jsonexamples/generated/escape.json jsonexamples/generated/utf-8-full.json jsonexamples/generated/utf-8-miss.json jsonexamples/generated/utf-8.json - -RAPIDJSON_INCLUDE:=dependencies/rapidjson/include -SAJSON_INCLUDE:=dependencies/sajson/include -JSON11_INCLUDE:=dependencies/json11/json11.hpp -FASTJSON_INCLUDE:=dependencies/include/fastjson/fastjson.h -GASON_INCLUDE:=dependencies/gason/src/gason.h -UJSON4C_INCLUDE:=dependencies/ujson4c/src/ujdecode.c -CJSON_INCLUDE:=dependencies/cJSON/cJSON.h -JSMN_INCLUDE:=dependencies/jsmn/jsmn.h -JSON_INCLUDE:=dependencies/json/single_include/nlohmann/json.hpp - -EXTRAOBJECTS=ujdecode.o - -MAINEXECUTABLES=parse minify json2json jsonstats statisticalmodel jsonpointer get_corpus_benchmark -TESTEXECUTABLES=jsoncheck jsoncheck_westmere jsoncheck_fallback integer_tests numberparsingcheck stringparsingcheck pointercheck parse_many_test basictests errortests readme_examples readme_examples11 readme_examples_noexceptions readme_examples_noexceptions11 -COMPARISONEXECUTABLES=minifiercompetition parsingcompetition parseandstatcompetition distinctuseridcompetition allparserscheckfile allparsingcompetition -SUPPLEMENTARYEXECUTABLES=parse_noutf8validation parse_nonumberparsing parse_nostringparsing - -all: $(MAINEXECUTABLES) - -competition: $(COMPARISONEXECUTABLES) - -.PHONY: benchmark test - -benchmark: - bash ./scripts/parser.sh - bash ./scripts/parseandstat.sh - -run_basictests: basictests - ./basictests - -run_errortests: errortests - ./errortests - -run_numberparsingcheck: numberparsingcheck - ./numberparsingcheck - -run_integer_tests: integer_tests - ./integer_tests - -run_staticchecks: staticchecks - ./staticchecks - -run_stringparsingcheck: stringparsingcheck - ./stringparsingcheck - -run_jsoncheck: jsoncheck - ./jsoncheck - -run_parse_many_test: parse_many_test - ./parse_many_test - -run_jsoncheck_westmere: jsoncheck_westmere - ./jsoncheck_westmere - -run_jsoncheck_fallback: jsoncheck_fallback - ./jsoncheck_fallback - -run_pointercheck: pointercheck - ./pointercheck - -run_issue150_sh: allparserscheckfile - ./tests/issue150.sh - -quickstart: amalgamate - cd examples/quickstart && make quickstart - -run_quickstart: amalgamate - cd examples/quickstart && make test - -quickstart11: amalgamate - cd examples/quickstart && make quickstart11 - -run_quickstart11: amalgamate - cd examples/quickstart && make test11 - -quickstart14: amalgamate - cd examples/quickstart && make quickstart14 - -run_quickstart14: amalgamate - cd examples/quickstart && make test14 - -run_testjson2json_sh: minify json2json - ./tests/testjson2json.sh - -$(FEATURE_JSON_FILES): benchmark/genfeaturejson.rb - ruby ./benchmark/genfeaturejson.rb - -run_benchfeatures: benchfeatures $(FEATURE_JSON_FILES) - ./benchfeatures -n 1000 - -test: quicktests slowtests - @echo "It looks like the code is good!" - -quiettest: quicktests slowtests - -quicktests: run_basictests run_quickstart readme_examples readme_examples_noexceptions run_jsoncheck run_numberparsingcheck run_integer_tests run_stringparsingcheck run_jsoncheck run_parse_many_test run_pointercheck run_jsoncheck_westmere run_jsoncheck_fallback run_quickstart14 run_quickstart11 - -slowtests: run_testjson2json_sh run_issue150_sh - -amalgamate: - singleheader/amalgamate.sh - -singleheader/simdjson.h singleheader/simdjson.cpp singleheader/amalgamate_demo.cpp: singleheader/amalgamate.sh src/simdjson.cpp $(SRCHEADERS) $(INCLUDEHEADERS) - singleheader/amalgamate.sh - -singleheader/demo: singleheader/simdjson.h singleheader/simdjson.cpp singleheader/amalgamate_demo.cpp - $(CXX) $(CXXFLAGS) -o singleheader/demo singleheader/amalgamate_demo.cpp -Isingleheader - -amalgamate_test: singleheader/demo jsonexamples/twitter.json jsonexamples/amazon_cellphones.ndjson - singleheader/demo jsonexamples/twitter.json jsonexamples/amazon_cellphones.ndjson - -submodules: - -git submodule update --init --recursive - -touch submodules - -$(JSON_INCLUDE) $(SAJSON_INCLUDE) $(RAPIDJSON_INCLUDE) $(JSON11_INCLUDE) $(FASTJSON_INCLUDE) $(GASON_INCLUDE) $(UJSON4C_INCLUDE) $(CJSON_INCLUDE) $(JSMN_INCLUDE) : submodules - -parse: benchmark/parse.cpp benchmark/event_counter.h benchmark/benchmarker.h $(HEADERS) $(LIBFILES) - $(CXX) $(CXXFLAGS) -o parse benchmark/parse.cpp $(LIBFILES) $(LIBFLAGS) - -get_corpus_benchmark: benchmark/get_corpus_benchmark.cpp $(HEADERS) $(LIBFILES) - $(CXX) $(CXXFLAGS) -o get_corpus_benchmark benchmark/get_corpus_benchmark.cpp $(LIBFILES) $(LIBFLAGS) - -parse_stream: benchmark/parse_stream.cpp benchmark/event_counter.h benchmark/benchmarker.h $(HEADERS) $(LIBFILES) - $(CXX) $(CXXFLAGS) -o parse_stream benchmark/parse_stream.cpp $(LIBFILES) $(LIBFLAGS) - -benchfeatures: benchmark/benchfeatures.cpp benchmark/event_counter.h benchmark/benchmarker.h $(HEADERS) $(LIBFILES) - $(CXX) $(CXXFLAGS) -o benchfeatures benchmark/benchfeatures.cpp $(LIBFILES) $(LIBFLAGS) - -perfdiff: benchmark/perfdiff.cpp - $(CXX) $(CXXFLAGS) -o perfdiff benchmark/perfdiff.cpp $(LIBFILES) $(LIBFLAGS) - -checkperf: parse perfdiff - CHECKPERF_BRANCH=$(CHECKPERF_BRANCH) bash ./benchmark/checkperf.sh jsonexamples/twitter.json - -statisticalmodel: benchmark/statisticalmodel.cpp $(HEADERS) $(LIBFILES) - $(CXX) $(CXXFLAGS) -o statisticalmodel benchmark/statisticalmodel.cpp $(LIBFILES) $(LIBFLAGS) - - -parse_noutf8validation: benchmark/parse.cpp $(HEADERS) $(LIBFILES) - $(CXX) $(CXXFLAGS) -o parse_noutf8validation -DSIMDJSON_SKIPUTF8VALIDATION benchmark/parse.cpp $(LIBFILES) $(LIBFLAGS) - -parse_nonumberparsing: benchmark/parse.cpp $(HEADERS) $(LIBFILES) - $(CXX) $(CXXFLAGS) -o parse_nonumberparsing -DSIMDJSON_SKIPNUMBERPARSING benchmark/parse.cpp $(LIBFILES) $(LIBFLAGS) - -parse_nostringparsing: benchmark/parse.cpp $(HEADERS) $(LIBFILES) - $(CXX) $(CXXFLAGS) -o parse_nostringparsing -DSIMDJSON_SKIPSTRINGPARSING benchmark/parse.cpp $(LIBFILES) $(LIBFLAGS) - - -jsoncheck: tests/jsoncheck.cpp $(HEADERS) $(LIBFILES) - $(CXX) $(CXXFLAGS) -o jsoncheck tests/jsoncheck.cpp -I. $(LIBFILES) $(LIBFLAGS) - -parse_many_test: tests/parse_many_test.cpp $(HEADERS) $(LIBFILES) - $(CXX) $(CXXFLAGS) -o parse_many_test tests/parse_many_test.cpp -I. $(LIBFILES) $(LIBFLAGS) - - -jsoncheck_westmere: tests/jsoncheck.cpp $(HEADERS) $(LIBFILES) - $(CXX) $(CXXFLAGS) -o jsoncheck_westmere tests/jsoncheck.cpp -I. $(LIBFILES) $(LIBFLAGS) -DSIMDJSON_IMPLEMENTATION_HASWELL=0 - -jsoncheck_fallback: tests/jsoncheck.cpp $(HEADERS) $(LIBFILES) - $(CXX) $(CXXFLAGS) -o jsoncheck_fallback tests/jsoncheck.cpp -I. $(LIBFILES) $(LIBFLAGS) -DSIMDJSON_IMPLEMENTATION_HASWELL=0 -DSIMDJSON_IMPLEMENTATION_WESTMERE=0 -DSIMDJSON_IMPLEMENTATION_ARM64=0 - -basictests: tests/basictests.cpp $(HEADERS) $(LIBFILES) - $(CXX) $(CXXFLAGS) -o basictests tests/basictests.cpp -I. $(LIBFILES) $(LIBFLAGS) - -errortests: tests/errortests.cpp $(HEADERS) $(LIBFILES) - $(CXX) $(CXXFLAGS) -o errortests tests/errortests.cpp -I. $(LIBFILES) $(LIBFLAGS) - -readme_examples: tests/readme_examples.cpp $(HEADERS) $(LIBFILES) - $(CXX) $(CXXFLAGS) -o readme_examples tests/readme_examples.cpp -I. $(LIBFILES) $(LIBFLAGS) - -readme_examples11: tests/readme_examples.cpp $(HEADERS) $(LIBFILES) - $(CXX) $(CXXFLAGS) -o readme_examples11 tests/readme_examples.cpp -I. $(LIBFILES) $(LIBFLAGS) -std=c++11 - -readme_examples_noexceptions: tests/readme_examples_noexceptions.cpp $(HEADERS) $(LIBFILES) - $(CXX) $(CXXFLAGS) -o readme_examples_noexceptions tests/readme_examples_noexceptions.cpp -I. $(LIBFILES) $(LIBFLAGS) -fno-exceptions - -readme_examples_noexceptions11: tests/readme_examples_noexceptions.cpp $(HEADERS) $(LIBFILES) - $(CXX) $(CXXFLAGS) -o readme_examples_noexceptions11 tests/readme_examples_noexceptions.cpp -I. $(LIBFILES) $(LIBFLAGS) -fno-exceptions -std=c++11 - -staticchecks: tests/staticchecks.cpp $(HEADERS) src/simdjson.cpp - $(CXX) $(CXXFLAGS) -o staticchecks tests/staticchecks.cpp -I. $(LIBFLAGS) - -numberparsingcheck: tests/numberparsingcheck.cpp $(HEADERS) src/simdjson.cpp - $(CXX) $(CXXFLAGS) -o numberparsingcheck tests/numberparsingcheck.cpp -I. $(LIBFLAGS) -DJSON_TEST_NUMBERS - -integer_tests:tests/integer_tests.cpp $(HEADERS) $(LIBFILES) - $(CXX) $(CXXFLAGS) -o integer_tests tests/integer_tests.cpp -I. $(LIBFILES) $(LIBFLAGS) - -stringparsingcheck: tests/stringparsingcheck.cpp $(HEADERS) src/simdjson.cpp - $(CXX) $(CXXFLAGS) -o stringparsingcheck tests/stringparsingcheck.cpp -I. $(LIBFLAGS) -DJSON_TEST_STRINGS - -pointercheck:tests/pointercheck.cpp $(HEADERS) $(LIBFILES) - $(CXX) $(CXXFLAGS) -o pointercheck tests/pointercheck.cpp -I. $(LIBFILES) $(LIBFLAGS) - -minifiercompetition: benchmark/minifiercompetition.cpp submodules $(HEADERS) $(LIBFILES) - $(CXX) $(CXXFLAGS) -o minifiercompetition benchmark/minifiercompetition.cpp -I. $(LIBFILES) $(LIBFLAGS) $(COREDEPSINCLUDE) - -minify: tools/minify.cpp $(HEADERS) $(LIBFILES) - $(CXX) $(CXXFLAGS) -o minify tools/minify.cpp -I. $(LIBFILES) $(LIBFLAGS) - -json2json: tools/json2json.cpp $(HEADERS) $(LIBFILES) - $(CXX) $(CXXFLAGS) -o json2json $ tools/json2json.cpp -I. $(LIBFILES) $(LIBFLAGS) - -jsonpointer: tools/jsonpointer.cpp $(HEADERS) $(LIBFILES) - $(CXX) $(CXXFLAGS) -o jsonpointer $ tools/jsonpointer.cpp -I. $(LIBFILES) $(LIBFLAGS) - -jsonstats: tools/jsonstats.cpp $(HEADERS) $(LIBFILES) - $(CXX) $(CXXFLAGS) -o jsonstats $ tools/jsonstats.cpp -I. $(LIBFILES) $(LIBFLAGS) - -ujdecode.o: $(UJSON4C_INCLUDE) - $(CC) $(CFLAGS) -c dependencies/ujson4c/src/ujdecode.c - -parseandstatcompetition: benchmark/parseandstatcompetition.cpp $(HEADERS) $(LIBFILES) submodules - $(CXX) $(CXXFLAGS) -o parseandstatcompetition benchmark/parseandstatcompetition.cpp -I. $(LIBFILES) $(LIBFLAGS) $(COREDEPSINCLUDE) - -distinctuseridcompetition: benchmark/distinctuseridcompetition.cpp $(HEADERS) $(LIBFILES) submodules - $(CXX) $(CXXFLAGS) -o distinctuseridcompetition benchmark/distinctuseridcompetition.cpp -I. $(LIBFILES) $(LIBFLAGS) $(COREDEPSINCLUDE) - -parsingcompetition: benchmark/parsingcompetition.cpp $(HEADERS) $(LIBFILES) submodules - @echo "In case of build error due to missing files, try 'make clean'" - $(CXX) $(CXXFLAGS) -o parsingcompetition benchmark/parsingcompetition.cpp -I. $(LIBFILES) $(LIBFLAGS) $(COREDEPSINCLUDE) - -allparsingcompetition: benchmark/parsingcompetition.cpp $(HEADERS) $(LIBFILES) $(EXTRAOBJECTS) submodules - $(CXX) $(CXXFLAGS) -o allparsingcompetition benchmark/parsingcompetition.cpp $(EXTRAOBJECTS) -I. $(LIBFILES) $(LIBFLAGS) $(COREDEPSINCLUDE) $(EXTRADEPSINCLUDE) -DALLPARSER - - -allparserscheckfile: tests/allparserscheckfile.cpp $(HEADERS) $(LIBFILES) $(EXTRAOBJECTS) submodules - $(CXX) $(CXXFLAGS) -o allparserscheckfile tests/allparserscheckfile.cpp $(EXTRAOBJECTS) -I. $(LIBFILES) $(LIBFLAGS) $(COREDEPSINCLUDE) $(EXTRADEPSINCLUDE) - -.PHONY: clean cppcheck cleandist - -cppcheck: - cppcheck --enable=all src/*.cpp benchmarks/*.cpp tests/*.cpp -Iinclude -I. -Ibenchmark/linux - -everything: $(MAINEXECUTABLES) $(EXTRA_EXECUTABLES) $(TESTEXECUTABLES) $(COMPARISONEXECUTABLES) $(SUPPLEMENTARYEXECUTABLES) quickstart - -clean: - rm -f submodules $(EXTRAOBJECTS) $(MAINEXECUTABLES) $(EXTRA_EXECUTABLES) $(TESTEXECUTABLES) $(COMPARISONEXECUTABLES) $(SUPPLEMENTARYEXECUTABLES) - cd examples/quickstart && make clean - -cleandist: - rm -f submodules $(EXTRAOBJECTS) $(MAINEXECUTABLES) $(EXTRA_EXECUTABLES) $(TESTEXECUTABLES) $(COMPARISONEXECUTABLES) $(SUPPLEMENTARYEXECUTABLES) - -doc/api: Doxyfile $(HEADERS) - doxygen