From 640283fec60b6ea70906252827c79fa1c8163a08 Mon Sep 17 00:00:00 2001 From: John Keiser Date: Wed, 14 Aug 2019 13:19:55 -0700 Subject: [PATCH 1/2] Reuse jobs in Circle CI --- .circleci/config.yml | 362 +++++++---------------------- singleheader/amalgamation_demo.cpp | 2 +- singleheader/simdjson.cpp | 2 +- singleheader/simdjson.h | 2 +- 4 files changed, 89 insertions(+), 279 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e8316d2e1..4e23257c4 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,291 +1,101 @@ -version: 2 +version: 2.1 + +executors: + gcc7: + docker: + - image: gcc:7 + environment: + CXX: g++ + clang6: + docker: + - image: ubuntu:18.04 + environment: + CXX: clang++-6.0 + +commands: + init_gcc7: + steps: + - run: echo true + init_clang6: + steps: + - run: apt-get install -y clang build-essential git + jobs: - "gcc": - docker: - - image: ubuntu:18.04 - environment: - CXX: g++-7 + + # Parameterized job to run all tests with "make" + make_test: + description: Build, run tests and check performance + parameters: + compiler: + description: Compiler + type: string + archflags: + description: Architecture flags (ARCHFLAGS) + default: "" + type: string + + executor: << parameters.compiler >> + + environment: + ARCHFLAGS: << parameters.archflags >> + steps: - checkout - - run: apt-get update -qq - - run: > - apt-get install -y - build-essential - cmake - g++-7 - git + - init_<< parameters.compiler >> + - run: make + - run: make amalgamate + - run: make test + - run: make checkperf - - run: - name: Building (gcc) - command: make + # Parameterized job to run all tests with "cmake" + cmake_test: + description: Build, run tests and check performance - - run: - name: Running tests (gcc) - command: make quiettest amalgamate + parameters: + compiler: + description: Compiler + type: string + flags: + description: flags to pass to cmake + default: "" + type: string - - run: - name: Comparing perf against reference (gcc) - command: make checkperf + executor: << parameters.compiler >> - - run: - name: Building (gcc, cmake, dynamic) - command: | - mkdir build - cd build - cmake -DSIMDJSON_BUILD_STATIC=OFF .. - make - - - run: - name: Running tests (gcc, cmake, dynamic) - command: | - cd build - make test - - - run: - name: Building (gcc, cmake, static) - command: | - mkdir buildstatic - cd buildstatic - cmake -DSIMDJSON_BUILD_STATIC=ON .. - make - - - run: - name: Running tests (gcc, cmake, static) - command: | - cd buildstatic - make test - - - run: - name: Building (gcc, cmake, sanitize) - command: | - mkdir buildsani - cd buildsani - cmake -DSIMDJSON_SANITIZE=ON -DSIMDJSON_BUILD_STATIC=OFF .. - make - - - run: - name: Running tests (gcc, cmake, sanitize) - command: | - cd buildsani - make test - - "gccnoavx": - docker: - - image: ubuntu:18.04 - environment: - CXX: g++-7 steps: - - checkout - - run: apt-get update -qq - - run: > - apt-get install -y - build-essential - cmake - g++-7 - git - - - run: - name: Building (gcc) - command: ARCHFLAGS="-march=nehalem" make - - - run: - name: Running tests (gcc) - command: ARCHFLAGS="-march=nehalem" make quiettest amalgamate - - - run: - name: Comparing perf against reference (gcc) - command: ARCHFLAGS="-march=nehalem" make checkperf - - - run: - name: Building (gcc, cmake, dynamic) - command: | - mkdir build - cd build - cmake -DSIMDJSON_DISABLE_AVX=ON -DSIMDJSON_BUILD_STATIC=OFF .. - make - - - run: - name: Running tests (gcc, cmake, dynamic) - command: | - cd build - make test - - - run: - name: Building (gcc, cmake, static) - command: | - mkdir buildstatic - cd buildstatic - cmake -DSIMDJSON_DISABLE_AVX=ON -DSIMDJSON_BUILD_STATIC=ON .. - make - - - run: - name: Running tests (gcc, cmake, static) - command: | - cd buildstatic - make test - - - run: - name: Building (gcc, cmake, sanitize) - command: | - mkdir buildsani - cd buildsani - cmake -DSIMDJSON_BUILD_STATIC=OFF -DSIMDJSON_DISABLE_AVX=ON -DSIMDJSON_SANITIZE=ON .. - make - - - run: - name: Running tests (gcc, cmake, sanitize) - command: | - cd buildsani - make test - "clang": - docker: - - image: ubuntu:18.04 - environment: - CXX: clang++-6.0 - steps: + - init_<< parameters.compiler >> + - run: apt-get install -y cmake - checkout - - - run: apt-get update -qq - - run: > - apt-get install -y - build-essential - cmake - clang-6.0 - git - - - run: - name: Building (clang) - command: make - - - run: - name: Running tests (clang) - command: make quiettest amalgamate - - - run: - name: Comparing perf against reference (clang) - command: make checkperf - - - run: - name: Building (clang, cmake, dynamic) - command: | - mkdir build - cd build - cmake -DSIMDJSON_BUILD_STATIC=OFF .. - make - - - run: - name: Running tests (clang, cmake, dynamic) - command: | - cd build - make test - - - run: - name: Building (clang, cmake, static) - command: | - mkdir buildstatic - cd buildstatic - cmake -DSIMDJSON_BUILD_STATIC=ON .. - make - - - run: - name: Running tests (clang, cmake, static) - command: | - cd buildstatic - make test - - - run: - name: Building (clang, cmake, sanitize) - command: | - mkdir buildsani - cd buildsani - cmake -DSIMDJSON_BUILD_STATIC=OFF -DSIMDJSON_SANITIZE=ON .. - make - - - run: - name: Running tests (clang, cmake, sanitize) - command: | - cd buildsani - make test - - "clangnoavx": - docker: - - image: ubuntu:18.04 - environment: - CXX: clang++-6.0 - steps: - - checkout - - - run: apt-get update -qq - - run: > - apt-get install -y - build-essential - cmake - clang-6.0 - git - - - run: - name: Building (clang) - command: ARCHFLAGS="-march=nehalem" make - - - run: - name: Running tests (clang) - command: ARCHFLAGS="-march=nehalem" make quiettest amalgamate - - - run: - name: Comparing perf against reference (clang) - command: ARCHFLAGS="-march=nehalem" make checkperf - - - run: - name: Building (clang, cmake, dynamic) - command: | - mkdir build - cd build - cmake -DSIMDJSON_BUILD_STATIC=OFF -DSIMDJSON_DISABLE_AVX=on .. - make - - - run: - name: Running tests (clang, cmake, dynamic) - command: | - cd build - make test - - - - run: - name: Building (clang, cmake, static) - command: | - mkdir buildstatic - cd buildstatic - cmake -DSIMDJSON_BUILD_STATIC=ON -DSIMDJSON_DISABLE_AVX=on .. - make - - - run: - name: Running tests (clang, cmake, static) - command: | - cd buildstatic - make test - - - - run: - name: Building (clang, cmake, sanitize) - command: | - mkdir buildsani - cd buildsani - cmake -DSIMDJSON_BUILD_STATIC=OFF -DSIMDJSON_DISABLE_AVX=ON -DSIMDJSON_SANITIZE=ON .. - make - - - run: - name: Running tests (clang, cmake, sanitize) - command: | - cd buildsani - make test + - run: cmake << parameters.flags >> + - run: make + - run: make test workflows: - version: 2 + version: 2.1 build_and_test: jobs: - - "clang" - - "gcc" - - "clangnoavx" - - "gccnoavx" + # gcc7 + - make_test: { compiler: gcc7 } + - cmake_test: { compiler: gcc7, flags: -DSIMDJSON_BUILD_STATIC=ON } + - cmake_test: { compiler: gcc7, flags: -DSIMDJSON_BUILD_STATIC=ON } + - cmake_test: { compiler: gcc7, flags: -DSIMDJSON_BUILD_STATIC=OFF -DSIMDJSON_SANITIZE=ON } + # gcc7 - no AVX + - make_test: { compiler: gcc7, archflags: -march=nehalem } + - cmake_test: { compiler: gcc7, flags: -DSIMDJSON_DISABLE_AVX=ON -DSIMDJSON_BUILD_STATIC=ON } + - cmake_test: { compiler: gcc7, flags: -DSIMDJSON_DISABLE_AVX=ON -DSIMDJSON_BUILD_STATIC=ON } + - cmake_test: { compiler: gcc7, flags: -DSIMDJSON_DISABLE_AVX=ON -DSIMDJSON_BUILD_STATIC=OFF -DSIMDJSON_SANITIZE=ON } + # clang6 + - make_test: { compiler: clang6 } + - cmake_test: { compiler: clang6, flags: -DSIMDJSON_BUILD_STATIC=ON } + - cmake_test: { compiler: clang6, flags: -DSIMDJSON_BUILD_STATIC=ON } + - cmake_test: { compiler: clang6, flags: -DSIMDJSON_BUILD_STATIC=OFF -DSIMDJSON_SANITIZE=ON } + # clang6 - no AVX + - make_test: { compiler: clang6, archflags: -march=nehalem } + - cmake_test: { compiler: clang6, flags: -DSIMDJSON_DISABLE_AVX=ON -DSIMDJSON_BUILD_STATIC=ON } + - cmake_test: { compiler: clang6, flags: -DSIMDJSON_DISABLE_AVX=ON -DSIMDJSON_BUILD_STATIC=ON } + - cmake_test: { compiler: clang6, flags: -DSIMDJSON_DISABLE_AVX=ON -DSIMDJSON_BUILD_STATIC=OFF -DSIMDJSON_SANITIZE=ON } + + # TODO add windows: https://circleci.com/docs/2.0/configuration-reference/#windows + diff --git a/singleheader/amalgamation_demo.cpp b/singleheader/amalgamation_demo.cpp index 71b1e9c99..d179918c6 100644 --- a/singleheader/amalgamation_demo.cpp +++ b/singleheader/amalgamation_demo.cpp @@ -1,4 +1,4 @@ -/* auto-generated on Wed Aug 14 10:31:26 DST 2019. Do not edit! */ +/* auto-generated on Wed Aug 14 13:56:54 DST 2019. Do not edit! */ #include #include "simdjson.h" diff --git a/singleheader/simdjson.cpp b/singleheader/simdjson.cpp index ed000f63f..4e88e5f86 100644 --- a/singleheader/simdjson.cpp +++ b/singleheader/simdjson.cpp @@ -1,4 +1,4 @@ -/* auto-generated on Wed Aug 14 10:31:26 DST 2019. Do not edit! */ +/* auto-generated on Wed Aug 14 13:56:54 DST 2019. Do not edit! */ #include "simdjson.h" /* used for http://dmalloc.com/ Dmalloc - Debug Malloc Library */ diff --git a/singleheader/simdjson.h b/singleheader/simdjson.h index a16fe67b8..a522ee50e 100644 --- a/singleheader/simdjson.h +++ b/singleheader/simdjson.h @@ -1,4 +1,4 @@ -/* auto-generated on Wed Aug 14 10:31:26 DST 2019. Do not edit! */ +/* auto-generated on Wed Aug 14 13:56:54 DST 2019. Do not edit! */ /* begin file include/simdjson/simdjson_version.h */ // /include/simdjson/simdjson_version.h automatically generated by release.py, // do not change by hand From b49eefbee63faa15602178d4f9846551fc3fda9d Mon Sep 17 00:00:00 2001 From: John Keiser Date: Wed, 14 Aug 2019 15:47:20 -0700 Subject: [PATCH 2/2] Give test jobs better names --- .circleci/config.yml | 165 +++++++++++++++++++++++++++---------------- 1 file changed, 104 insertions(+), 61 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4e23257c4..76dd921b2 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,5 +1,6 @@ version: 2.1 +# Reusable image / compiler definitions executors: gcc7: docker: @@ -12,90 +13,132 @@ executors: environment: CXX: clang++-6.0 +# Reusable test commands (and initializer for clang 6) commands: - init_gcc7: - steps: - - run: echo true init_clang6: steps: + - run: apt-get update -qq - run: apt-get install -y clang build-essential git - -jobs: - - # Parameterized job to run all tests with "make" make_test: - description: Build, run tests and check performance - parameters: - compiler: - description: Compiler - type: string - archflags: - description: Architecture flags (ARCHFLAGS) - default: "" - type: string - - executor: << parameters.compiler >> - - environment: - ARCHFLAGS: << parameters.archflags >> - steps: - checkout - - run: apt-get update -qq - - init_<< parameters.compiler >> - run: make - run: make amalgamate - run: make test - run: make checkperf - - # Parameterized job to run all tests with "cmake" cmake_test: - description: Build, run tests and check performance - - parameters: - compiler: - description: Compiler - type: string - flags: - description: flags to pass to cmake - default: "" - type: string - - executor: << parameters.compiler >> - steps: - run: apt-get update -qq - - init_<< parameters.compiler >> - run: apt-get install -y cmake - checkout - - run: cmake << parameters.flags >> + - run: cmake $CMAKE_TEST_FLAGS - run: make - run: make test +jobs: + + gcc-avx: + description: Build, run tests and check performance on GCC 7 and AVX 2 + executor: gcc7 + steps: [ make_test ] + gcc-avx-dynamic: + description: Build, run tests and check performance on GCC 7 and AVX 2 with a cmake dynamic build + executor: gcc7 + environment: { CMAKE_TEST_FLAGS: -DSIMDJSON_BUILD_STATIC=OFF } + steps: [ cmake_test ] + gcc-avx-static: + description: Build, run tests and check performance on GCC 7 and AVX 2 with a cmake static build + executor: gcc7 + environment: { CMAKE_TEST_FLAGS: -DSIMDJSON_BUILD_STATIC=ON } + steps: [ cmake_test ] + gcc-avx-sanitize: + description: Build, run tests and check performance on GCC 7 and AVX 2 with a cmake sanitize build + executor: gcc7 + environment: { CMAKE_TEST_FLAGS: -DSIMDJSON_BUILD_STATIC=OFF -DSIMDJSON_SANITIZE=ON } + steps: [ cmake_test ] + + gcc-sse: + description: Build, run tests and check performance on GCC 7 and SSE 4.2 + executor: gcc7 + environment: { ARCHFLAGS: -march=nehalem } + steps: [ make_test ] + gcc-sse-dynamic: + description: Build, run tests and check performance on GCC 7 and SSE 4.2 with a cmake dynamic build + executor: gcc7 + environment: { CMAKE_TEST_FLAGS: -DSIMDJSON_BUILD_STATIC=OFF } + steps: [ cmake_test ] + gcc-sse-static: + description: Build, run tests and check performance on GCC 7 and SSE 4.2 with a cmake static build + executor: gcc7 + environment: { CMAKE_TEST_FLAGS: -DSIMDJSON_BUILD_STATIC=ON } + steps: [ cmake_test ] + gcc-sse-sanitize: + description: Build, run tests and check performance on GCC 7 and SSE 4.2 with a cmake sanitize build + executor: gcc7 + environment: { CMAKE_TEST_FLAGS: -DSIMDJSON_BUILD_STATIC=OFF -DSIMDJSON_SANITIZE=ON } + steps: [ cmake_test ] + + clang-avx: + description: Build, run tests and check performance on clang 6 and AVX 2 + executor: clang6 + steps: [ init_clang6, make_test ] + clang-avx-dynamic: + description: Build, run tests and check performance on clang 6 and AVX 2 with a cmake dynamic build + executor: clang6 + environment: { CMAKE_TEST_FLAGS: -DSIMDJSON_BUILD_STATIC=OFF } + steps: [ init_clang6, cmake_test ] + clang-avx-static: + description: Build, run tests and check performance on clang 6 and AVX 2 with a cmake static build + executor: clang6 + environment: { CMAKE_TEST_FLAGS: -DSIMDJSON_BUILD_STATIC=ON } + steps: [ init_clang6, cmake_test ] + clang-avx-sanitize: + description: Build, run tests and check performance on clang 6 and AVX 2 with a cmake sanitize build + executor: clang6 + environment: { CMAKE_TEST_FLAGS: -DSIMDJSON_BUILD_STATIC=OFF -DSIMDJSON_SANITIZE=ON } + steps: [ init_clang6, cmake_test ] + + clang-sse: + description: Build, run tests and check performance on GCC 7 and SSE 4.2 + executor: clang6 + environment: { ARCHFLAGS: -march=nehalem } + steps: [ init_clang6, make_test ] + clang-sse-dynamic: + description: Build, run tests and check performance on GCC 7 and SSE 4.2 with a cmake dynamic build + executor: clang6 + environment: { CMAKE_TEST_FLAGS: -DSIMDJSON_BUILD_STATIC=OFF } + steps: [ init_clang6, cmake_test ] + clang-sse-static: + description: Build, run tests and check performance on GCC 7 and SSE 4.2 with a cmake static build + executor: clang6 + environment: { CMAKE_TEST_FLAGS: -DSIMDJSON_BUILD_STATIC=ON } + steps: [ init_clang6, cmake_test ] + clang-sse-sanitize: + description: Build, run tests and check performance on GCC 7 and SSE 4.2 with a cmake sanitize build + executor: clang6 + environment: { CMAKE_TEST_FLAGS: -DSIMDJSON_BUILD_STATIC=OFF -DSIMDJSON_SANITIZE=ON } + steps: [ init_clang6, cmake_test ] + workflows: version: 2.1 build_and_test: jobs: - # gcc7 - - make_test: { compiler: gcc7 } - - cmake_test: { compiler: gcc7, flags: -DSIMDJSON_BUILD_STATIC=ON } - - cmake_test: { compiler: gcc7, flags: -DSIMDJSON_BUILD_STATIC=ON } - - cmake_test: { compiler: gcc7, flags: -DSIMDJSON_BUILD_STATIC=OFF -DSIMDJSON_SANITIZE=ON } - # gcc7 - no AVX - - make_test: { compiler: gcc7, archflags: -march=nehalem } - - cmake_test: { compiler: gcc7, flags: -DSIMDJSON_DISABLE_AVX=ON -DSIMDJSON_BUILD_STATIC=ON } - - cmake_test: { compiler: gcc7, flags: -DSIMDJSON_DISABLE_AVX=ON -DSIMDJSON_BUILD_STATIC=ON } - - cmake_test: { compiler: gcc7, flags: -DSIMDJSON_DISABLE_AVX=ON -DSIMDJSON_BUILD_STATIC=OFF -DSIMDJSON_SANITIZE=ON } - # clang6 - - make_test: { compiler: clang6 } - - cmake_test: { compiler: clang6, flags: -DSIMDJSON_BUILD_STATIC=ON } - - cmake_test: { compiler: clang6, flags: -DSIMDJSON_BUILD_STATIC=ON } - - cmake_test: { compiler: clang6, flags: -DSIMDJSON_BUILD_STATIC=OFF -DSIMDJSON_SANITIZE=ON } - # clang6 - no AVX - - make_test: { compiler: clang6, archflags: -march=nehalem } - - cmake_test: { compiler: clang6, flags: -DSIMDJSON_DISABLE_AVX=ON -DSIMDJSON_BUILD_STATIC=ON } - - cmake_test: { compiler: clang6, flags: -DSIMDJSON_DISABLE_AVX=ON -DSIMDJSON_BUILD_STATIC=ON } - - cmake_test: { compiler: clang6, flags: -DSIMDJSON_DISABLE_AVX=ON -DSIMDJSON_BUILD_STATIC=OFF -DSIMDJSON_SANITIZE=ON } + - gcc-avx + - gcc-avx-dynamic + - gcc-avx-static + - gcc-avx-sanitize + - gcc-sse + - gcc-sse-dynamic + - gcc-sse-static + - gcc-sse-sanitize + - clang-avx + - clang-avx-dynamic + - clang-avx-static + - clang-avx-sanitize + - clang-sse + - clang-sse-dynamic + - clang-sse-static + - clang-sse-sanitize # TODO add windows: https://circleci.com/docs/2.0/configuration-reference/#windows