diff --git a/scripts/data/skylake/apache_builds.json.table b/scripts/data/skylake/apache_builds.json.table index 82857874a..afbb15ef8 100644 --- a/scripts/data/skylake/apache_builds.json.table +++ b/scripts/data/skylake/apache_builds.json.table @@ -1,4 +1,4 @@ name cycles_per_byte cycles_per_byte_err gb_per_s gb_per_s_err -"simdjson " 1.587 0.041 2.133 0.053 -"RapidJSON" 10.333 0.095 0.329 0.003 -"sajson" 3.058 0.197 1.110 0.067 +"simdjson " 1.596 0.035 2.123 0.046 +"RapidJSON" 4.893 0.056 0.695 0.008 +"sajson" 2.765 0.056 1.228 0.024 diff --git a/scripts/data/skylake/canada.json.table b/scripts/data/skylake/canada.json.table index 7c7f5a924..d4a61ee9f 100644 --- a/scripts/data/skylake/canada.json.table +++ b/scripts/data/skylake/canada.json.table @@ -1,4 +1,4 @@ name cycles_per_byte cycles_per_byte_err gb_per_s gb_per_s_err -"simdjson " 3.481 0.035 0.979 0.010 -"RapidJSON" 9.576 0.036 0.356 0.001 -"sajson" 6.638 0.027 0.513 0.002 +"simdjson " 3.288 0.030 1.036 0.010 +"RapidJSON" 6.675 0.014 0.510 0.001 +"sajson" 4.258 0.023 0.800 0.004 diff --git a/scripts/data/skylake/citm_catalog.json.table b/scripts/data/skylake/citm_catalog.json.table index 695b2027a..a9ac2daf1 100644 --- a/scripts/data/skylake/citm_catalog.json.table +++ b/scripts/data/skylake/citm_catalog.json.table @@ -1,4 +1,4 @@ name cycles_per_byte cycles_per_byte_err gb_per_s gb_per_s_err -"simdjson " 1.403 0.020 2.428 0.034 -"RapidJSON" 5.368 0.016 0.635 0.002 -"sajson" 3.011 0.023 1.132 0.008 +"simdjson " 1.354 0.016 2.516 0.030 +"RapidJSON" 3.734 0.013 0.913 0.003 +"sajson" 2.643 0.069 1.289 0.033 diff --git a/scripts/data/skylake/gbps.pdf b/scripts/data/skylake/gbps.pdf index 725ce1762..e69de29bb 100644 Binary files a/scripts/data/skylake/gbps.pdf and b/scripts/data/skylake/gbps.pdf differ diff --git a/scripts/data/skylake/github_events.json.table b/scripts/data/skylake/github_events.json.table index c650f26c3..8132f7194 100644 --- a/scripts/data/skylake/github_events.json.table +++ b/scripts/data/skylake/github_events.json.table @@ -1,4 +1,4 @@ name cycles_per_byte cycles_per_byte_err gb_per_s gb_per_s_err -"simdjson " 1.506 0.055 2.234 0.078 -"RapidJSON" 9.808 0.147 0.347 0.005 -"sajson" 3.397 0.152 0.997 0.043 +"simdjson " 1.376 0.048 2.442 0.081 +"RapidJSON" 4.564 0.153 0.743 0.024 +"sajson" 3.067 0.141 1.104 0.048 diff --git a/scripts/data/skylake/gsoc-2018.json.table b/scripts/data/skylake/gsoc-2018.json.table index e02675e8f..5a2a7a991 100644 --- a/scripts/data/skylake/gsoc-2018.json.table +++ b/scripts/data/skylake/gsoc-2018.json.table @@ -1,4 +1,4 @@ name cycles_per_byte cycles_per_byte_err gb_per_s gb_per_s_err -"simdjson " 1.167 0.025 2.918 0.062 -"RapidJSON" 8.045 0.013 0.424 0.001 -"sajson" 2.866 0.025 1.189 0.010 +"simdjson " 1.141 0.022 2.984 0.057 +"RapidJSON" 3.457 0.007 0.986 0.002 +"sajson" 2.646 0.024 1.288 0.011 diff --git a/scripts/data/skylake/instruments.json.table b/scripts/data/skylake/instruments.json.table index c90e8a480..7f1e724c4 100644 --- a/scripts/data/skylake/instruments.json.table +++ b/scripts/data/skylake/instruments.json.table @@ -1,4 +1,4 @@ name cycles_per_byte cycles_per_byte_err gb_per_s gb_per_s_err -"simdjson " 1.755 0.044 1.935 0.047 -"RapidJSON" 9.003 0.061 0.378 0.003 -"sajson" 3.605 0.052 0.944 0.013 +"simdjson " 1.708 0.033 1.989 0.038 +"RapidJSON" 4.929 0.061 0.691 0.008 +"sajson" 3.017 0.105 1.128 0.038 diff --git a/scripts/data/skylake/marine_ik.json.table b/scripts/data/skylake/marine_ik.json.table index dd200a2a7..765a6a6f2 100644 --- a/scripts/data/skylake/marine_ik.json.table +++ b/scripts/data/skylake/marine_ik.json.table @@ -1,4 +1,4 @@ name cycles_per_byte cycles_per_byte_err gb_per_s gb_per_s_err -"simdjson " 3.818 0.053 0.892 0.012 -"RapidJSON" 11.256 0.040 0.303 0.001 -"sajson" 5.802 0.067 0.587 0.007 +"simdjson " 3.854 0.049 0.884 0.011 +"RapidJSON" 7.525 0.029 0.453 0.002 +"sajson" 4.620 0.042 0.738 0.007 diff --git a/scripts/data/skylake/mesh.json.table b/scripts/data/skylake/mesh.json.table index 059aa7b31..73e47d01f 100644 --- a/scripts/data/skylake/mesh.json.table +++ b/scripts/data/skylake/mesh.json.table @@ -1,4 +1,4 @@ name cycles_per_byte cycles_per_byte_err gb_per_s gb_per_s_err -"simdjson " 3.736 0.093 0.912 0.022 -"RapidJSON" 11.713 0.084 0.291 0.002 -"sajson" 6.609 0.156 0.515 0.012 +"simdjson " 3.763 0.059 0.905 0.014 +"RapidJSON" 7.760 0.060 0.439 0.003 +"sajson" 4.941 0.045 0.689 0.006 diff --git a/scripts/data/skylake/mesh.pretty.json.table b/scripts/data/skylake/mesh.pretty.json.table index 4399675f3..dafc7c3f0 100644 --- a/scripts/data/skylake/mesh.pretty.json.table +++ b/scripts/data/skylake/mesh.pretty.json.table @@ -1,4 +1,4 @@ name cycles_per_byte cycles_per_byte_err gb_per_s gb_per_s_err -"simdjson " 2.512 0.039 1.356 0.021 -"RapidJSON" 6.206 0.038 0.549 0.003 -"sajson" 4.395 0.060 0.775 0.010 +"simdjson " 2.472 0.025 1.378 0.014 +"RapidJSON" 4.777 0.043 0.713 0.006 +"sajson" 3.306 0.050 1.031 0.015 diff --git a/scripts/data/skylake/numbers.json.table b/scripts/data/skylake/numbers.json.table index 56572aeee..10a39b48f 100644 --- a/scripts/data/skylake/numbers.json.table +++ b/scripts/data/skylake/numbers.json.table @@ -1,4 +1,4 @@ name cycles_per_byte cycles_per_byte_err gb_per_s gb_per_s_err -"simdjson " 3.205 0.045 1.061 0.015 -"RapidJSON" 9.598 0.092 0.355 0.003 -"sajson" 5.982 0.039 0.569 0.004 +"simdjson " 3.200 0.032 1.062 0.010 +"RapidJSON" 6.548 0.092 0.520 0.007 +"sajson" 4.123 0.038 0.825 0.008 diff --git a/scripts/data/skylake/random.json.table b/scripts/data/skylake/random.json.table index 959ca896b..2f36c5e55 100644 --- a/scripts/data/skylake/random.json.table +++ b/scripts/data/skylake/random.json.table @@ -1,4 +1,4 @@ name cycles_per_byte cycles_per_byte_err gb_per_s gb_per_s_err -"simdjson " 2.601 0.044 1.309 0.022 -"RapidJSON" 14.636 0.041 0.233 0.001 -"sajson" 4.775 0.066 0.713 0.010 +"simdjson " 2.412 0.041 1.411 0.023 +"RapidJSON" 6.819 0.026 0.500 0.002 +"sajson" 4.152 0.086 0.820 0.017 diff --git a/scripts/data/skylake/twitter.json.table b/scripts/data/skylake/twitter.json.table index a6e56aad5..137cc7b06 100644 --- a/scripts/data/skylake/twitter.json.table +++ b/scripts/data/skylake/twitter.json.table @@ -1,4 +1,4 @@ name cycles_per_byte cycles_per_byte_err gb_per_s gb_per_s_err -"simdjson " 1.716 0.033 1.983 0.038 -"RapidJSON" 9.673 0.038 0.352 0.001 -"sajson" 3.577 0.047 0.952 0.012 +"simdjson " 1.573 0.031 2.164 0.042 +"RapidJSON" 4.837 0.029 0.704 0.004 +"sajson" 3.277 0.051 1.039 0.016 diff --git a/scripts/data/skylake/twitterescaped.json.table b/scripts/data/skylake/twitterescaped.json.table index 12a2b3ed5..cd4277692 100644 --- a/scripts/data/skylake/twitterescaped.json.table +++ b/scripts/data/skylake/twitterescaped.json.table @@ -1,4 +1,4 @@ name cycles_per_byte cycles_per_byte_err gb_per_s gb_per_s_err -"simdjson " 3.065 0.042 1.111 0.015 -"RapidJSON" 11.803 0.035 0.289 0.001 -"sajson" 4.651 0.038 0.732 0.006 +"simdjson " 3.077 0.036 1.107 0.013 +"RapidJSON" 5.632 0.028 0.605 0.003 +"sajson" 4.066 0.037 0.838 0.007 diff --git a/scripts/data/skylake/update-center.json.table b/scripts/data/skylake/update-center.json.table index bbe5cf35c..e4868a083 100644 --- a/scripts/data/skylake/update-center.json.table +++ b/scripts/data/skylake/update-center.json.table @@ -1,4 +1,4 @@ name cycles_per_byte cycles_per_byte_err gb_per_s gb_per_s_err -"simdjson " 2.016 0.042 1.688 0.034 -"RapidJSON" 13.795 0.028 0.247 0.000 -"sajson" 4.281 0.050 0.796 0.009 +"simdjson " 1.934 0.037 1.760 0.033 +"RapidJSON" 6.090 0.027 0.559 0.002 +"sajson" 4.007 0.058 0.850 0.012 diff --git a/scripts/forpaper.sh b/scripts/forpaper.sh new file mode 100755 index 000000000..bb1d97ed9 --- /dev/null +++ b/scripts/forpaper.sh @@ -0,0 +1,15 @@ +#!/bin/bash +[[ "$(command -v gnuplot)" ]] || { echo "gnuplot is not installed" 1>&2 ; exit 1; } + +SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )" + +cd $SCRIPTPATH/..; make clean + +cd $SCRIPTPATH/setupfortesting/; sudo ./setupfortesting.sh + +$SCRIPTPATH/parsingcompdata.sh + +$SCRIPTPATH/plotparse.sh + +$SCRIPTPATH/statisticalmodel.sh + diff --git a/scripts/modeldata/skylake/analysis.txt b/scripts/modeldata/skylake/analysis.txt index 0c79618e3..e69de29bb 100644 --- a/scripts/modeldata/skylake/analysis.txt +++ b/scripts/modeldata/skylake/analysis.txt @@ -1,24 +0,0 @@ -loading modeltable.txt -chosenpredictors= ['integer_count', 'float_count', 'string_count', 'backslash_count', 'nonasciibyte_count', 'object_count', 'array_count', 'null_count', 'true_count', 'false_count', 'byte_count', 'structural_indexes_count'] - -target = stage1_cycle_count - 0.49 cycles per byte_count -R2 = 0.9927537176836411 - -target = stage2_cycle_count - 2 cycles per structural_indexes_count - 0.11 cycles per byte_count -R2 = 0.9944225230951305 - -target = stage3_cycle_count - 18 cycles per float_count - 9 cycles per structural_indexes_count - 0.36 cycles per byte_count -R2 = 0.9861501257004178 - -target = total_cycles - 18 cycles per float_count - 11 cycles per structural_indexes_count - 0.95 cycles per byte_count -R2 = 0.9922642870761073 - diff --git a/scripts/modeldata/skylake/modeltable.txt b/scripts/modeldata/skylake/modeltable.txt index f719f7c6b..ebc9dc7e6 100644 --- a/scripts/modeldata/skylake/modeltable.txt +++ b/scripts/modeldata/skylake/modeltable.txt @@ -1,43 +1,43 @@ #/home/dlemire/CVS/github/simdjson/scripts/../jsonexamples/apache_builds.json # integer_count float_count string_count backslash_count nonasciibyte_count object_count array_count null_count true_count false_count byte_count structural_indexes_count stage1_cycle_count stage1_instruction_count stage2_cycle_count stage2_instruction_count stage3_cycle_count stage3_instruction_count -2 0 5289 26 0 884 3 0 2 1 127275 12365 98211.034000 352170.007000 111572.105000 368181.043000 +2 0 5289 26 0 884 3 0 2 1 127275 12365 98860.705000 352170.006000 110092.182000 368181.042000 #/home/dlemire/CVS/github/simdjson/scripts/../jsonexamples/canada.json # integer_count float_count string_count backslash_count nonasciibyte_count object_count array_count null_count true_count false_count byte_count structural_indexes_count stage1_cycle_count stage1_instruction_count stage2_cycle_count stage2_instruction_count stage3_cycle_count stage3_instruction_count -46 111080 12 0 0 4 56045 0 0 0 2251051 334374 2095985.740000 7018646.660000 5945447.860000 21368228.500000 +46 111080 12 0 0 4 56045 0 0 0 2251051 334374 2117954.980000 7018646.660000 5982086.300000 21368228.540000 #/home/dlemire/CVS/github/simdjson/scripts/../jsonexamples/citm_catalog.json # integer_count float_count string_count backslash_count nonasciibyte_count object_count array_count null_count true_count false_count byte_count structural_indexes_count stage1_cycle_count stage1_instruction_count stage2_cycle_count stage2_instruction_count stage3_cycle_count stage3_instruction_count -14392 0 26604 2 348 10937 10451 1263 0 0 1727204 135991 1335598.660000 4503327.740000 1259503.960000 4555510.200000 +14392 0 26604 2 348 10937 10451 1263 0 0 1727204 135991 1256015.620000 4503327.740000 1260240.200000 4555510.160000 #/home/dlemire/CVS/github/simdjson/scripts/../jsonexamples/github_events.json # integer_count float_count string_count backslash_count nonasciibyte_count object_count array_count null_count true_count false_count byte_count structural_indexes_count stage1_cycle_count stage1_instruction_count stage2_cycle_count stage2_instruction_count stage3_cycle_count stage3_instruction_count -149 0 1891 155 4 180 19 24 57 7 65132 4657 48022.321000 167625.005000 50155.521000 154544.020000 +149 0 1891 155 4 180 19 24 57 7 65132 4657 48294.810000 167625.000000 51049.382000 154544.023000 #/home/dlemire/CVS/github/simdjson/scripts/../jsonexamples/gsoc-2018.json # integer_count float_count string_count backslash_count nonasciibyte_count object_count array_count null_count true_count false_count byte_count structural_indexes_count stage1_cycle_count stage1_instruction_count stage2_cycle_count stage2_instruction_count stage3_cycle_count stage3_instruction_count -0 0 34128 15023 0 3793 0 0 0 0 3327831 75842 2222767.860000 6756257.660000 1860890.320000 4189900.280000 +0 0 34128 15023 0 3793 0 0 0 0 3327831 75842 2222068.100000 6756257.580000 1857685.960000 4189900.300000 #/home/dlemire/CVS/github/simdjson/scripts/../jsonexamples/instruments.json # integer_count float_count string_count backslash_count nonasciibyte_count object_count array_count null_count true_count false_count byte_count structural_indexes_count stage1_cycle_count stage1_instruction_count stage2_cycle_count stage2_instruction_count stage3_cycle_count stage3_instruction_count -4935 0 6889 0 0 1012 194 431 17 109 220346 27174 177422.325000 626094.037000 217311.684000 781025.073000 +4935 0 6889 0 0 1012 194 431 17 109 220346 27174 179195.077000 626094.037000 219242.424000 781025.075000 #/home/dlemire/CVS/github/simdjson/scripts/../jsonexamples/marine_ik.json # integer_count float_count string_count backslash_count nonasciibyte_count object_count array_count null_count true_count false_count byte_count structural_indexes_count stage1_cycle_count stage1_instruction_count stage2_cycle_count stage2_instruction_count stage3_cycle_count stage3_instruction_count -130225 114950 38268 0 0 9680 28377 0 6 0 2983466 643013 3081692.340000 10536841.780000 8966332.340000 28934128.040000 +130225 114950 38268 0 0 9680 28377 0 6 0 2983466 643013 3082930.040000 10536841.740000 8971625.320000 28934128.020000 #/home/dlemire/CVS/github/simdjson/scripts/../jsonexamples/mesh.json # integer_count float_count string_count backslash_count nonasciibyte_count object_count array_count null_count true_count false_count byte_count structural_indexes_count stage1_cycle_count stage1_instruction_count stage2_cycle_count stage2_instruction_count stage3_cycle_count stage3_instruction_count -40613 32400 11 0 0 3 3610 0 0 0 723597 153275 688102.314000 2415449.199000 2274427.180000 7726735.472000 +40613 32400 11 0 0 3 3610 0 0 0 723597 153275 692445.484000 2415449.195000 2282166.283000 7726735.452000 #/home/dlemire/CVS/github/simdjson/scripts/../jsonexamples/mesh.pretty.json # integer_count float_count string_count backslash_count nonasciibyte_count object_count array_count null_count true_count false_count byte_count structural_indexes_count stage1_cycle_count stage1_instruction_count stage2_cycle_count stage2_instruction_count stage3_cycle_count stage3_instruction_count -40613 32400 11 0 0 3 3610 0 0 0 1577353 153275 1201811.560000 4321852.060000 2993210.540000 9632717.200000 +40613 32400 11 0 0 3 3610 0 0 0 1577353 153275 1199740.400000 4321852.120000 2992529.340000 9632717.140000 #/home/dlemire/CVS/github/simdjson/scripts/../jsonexamples/numbers.json # integer_count float_count string_count backslash_count nonasciibyte_count object_count array_count null_count true_count false_count byte_count structural_indexes_count stage1_cycle_count stage1_instruction_count stage2_cycle_count stage2_instruction_count stage3_cycle_count stage3_instruction_count -0 10001 0 0 0 0 1 0 0 0 150124 20004 120823.525000 429712.027000 389954.315000 1261093.068000 +0 10001 0 0 0 0 1 0 0 0 150124 20004 125122.630000 429712.029000 394222.012000 1261093.069000 #/home/dlemire/CVS/github/simdjson/scripts/../jsonexamples/random.json # integer_count float_count string_count backslash_count nonasciibyte_count object_count array_count null_count true_count false_count byte_count structural_indexes_count stage1_cycle_count stage1_instruction_count stage2_cycle_count stage2_instruction_count stage3_cycle_count stage3_instruction_count -5002 0 33005 0 103482 4001 1001 0 495 505 510476 88018 595197.899000 2066269.128000 723745.943000 2475732.246000 +5002 0 33005 0 103482 4001 1001 0 495 505 510476 88018 594131.429000 2066269.128000 723559.441000 2475732.242000 #/home/dlemire/CVS/github/simdjson/scripts/../jsonexamples/twitterescaped.json # integer_count float_count string_count backslash_count nonasciibyte_count object_count array_count null_count true_count false_count byte_count structural_indexes_count stage1_cycle_count stage1_instruction_count stage2_cycle_count stage2_instruction_count stage3_cycle_count stage3_instruction_count -2108 1 18099 33048 0 1264 1050 1946 345 2446 562408 55264 478621.140000 1487521.088000 1405957.935000 4071016.264000 +2108 1 18099 33048 0 1264 1050 1946 345 2446 562408 55264 482727.010000 1487521.087000 1423074.231000 4071016.251000 #/home/dlemire/CVS/github/simdjson/scripts/../jsonexamples/twitter.json # integer_count float_count string_count backslash_count nonasciibyte_count object_count array_count null_count true_count false_count byte_count structural_indexes_count stage1_cycle_count stage1_instruction_count stage2_cycle_count stage2_instruction_count stage3_cycle_count stage3_instruction_count -2108 1 18099 1230 95406 1264 1050 1946 345 2446 631514 55264 559718.237000 1843920.095000 506910.598000 1577362.195000 +2108 1 18099 1230 95406 1264 1050 1946 345 2446 631514 55264 550609.995000 1843920.091000 509533.657000 1577362.190000 #/home/dlemire/CVS/github/simdjson/scripts/../jsonexamples/update-center.json # integer_count float_count string_count backslash_count nonasciibyte_count object_count array_count null_count true_count false_count byte_count structural_indexes_count stage1_cycle_count stage1_instruction_count stage2_cycle_count stage2_instruction_count stage3_cycle_count stage3_instruction_count -0 0 27229 284 49 1896 1937 0 134 252 533178 63420 490138.933000 1505783.098000 648771.784000 1842005.231000 +0 0 27229 284 49 1896 1937 0 134 252 533178 63420 495267.828000 1505783.095000 654760.566000 1842005.228000 diff --git a/scripts/parsingcompdata.sh b/scripts/parsingcompdata.sh index d18c9fea5..c3a95d0ce 100755 --- a/scripts/parsingcompdata.sh +++ b/scripts/parsingcompdata.sh @@ -21,3 +21,5 @@ for i in $SCRIPTPATH/../jsonexamples/*.json; do done echo "see results in "$datadirectory + +cd $datadirectory && gnuplot bar.gnuplot diff --git a/scripts/plots/skylake/apache_builds.json.pdf b/scripts/plots/skylake/apache_builds.json.pdf index c706eb6f1..9c4bb13cd 100644 Binary files a/scripts/plots/skylake/apache_builds.json.pdf and b/scripts/plots/skylake/apache_builds.json.pdf differ diff --git a/scripts/plots/skylake/apache_builds.jsonparseandstat.pdf b/scripts/plots/skylake/apache_builds.jsonparseandstat.pdf index f193e498d..55c3d7335 100644 Binary files a/scripts/plots/skylake/apache_builds.jsonparseandstat.pdf and b/scripts/plots/skylake/apache_builds.jsonparseandstat.pdf differ diff --git a/scripts/plots/skylake/canada.json.pdf b/scripts/plots/skylake/canada.json.pdf index f6496fabf..b8bd8a268 100644 Binary files a/scripts/plots/skylake/canada.json.pdf and b/scripts/plots/skylake/canada.json.pdf differ diff --git a/scripts/plots/skylake/canada.jsonparseandstat.pdf b/scripts/plots/skylake/canada.jsonparseandstat.pdf index 1ea3f49ee..0532e49c5 100644 Binary files a/scripts/plots/skylake/canada.jsonparseandstat.pdf and b/scripts/plots/skylake/canada.jsonparseandstat.pdf differ diff --git a/scripts/plots/skylake/citm_catalog.json.pdf b/scripts/plots/skylake/citm_catalog.json.pdf index f51f90108..151fe610d 100644 Binary files a/scripts/plots/skylake/citm_catalog.json.pdf and b/scripts/plots/skylake/citm_catalog.json.pdf differ diff --git a/scripts/plots/skylake/citm_catalog.jsonparseandstat.pdf b/scripts/plots/skylake/citm_catalog.jsonparseandstat.pdf index a211eb81c..9a72ca749 100644 Binary files a/scripts/plots/skylake/citm_catalog.jsonparseandstat.pdf and b/scripts/plots/skylake/citm_catalog.jsonparseandstat.pdf differ diff --git a/scripts/plots/skylake/github_events.json.pdf b/scripts/plots/skylake/github_events.json.pdf index fe3b3f90e..4a85a2f2f 100644 Binary files a/scripts/plots/skylake/github_events.json.pdf and b/scripts/plots/skylake/github_events.json.pdf differ diff --git a/scripts/plots/skylake/github_events.jsonparseandstat.pdf b/scripts/plots/skylake/github_events.jsonparseandstat.pdf index 242121587..9eb8acc22 100644 Binary files a/scripts/plots/skylake/github_events.jsonparseandstat.pdf and b/scripts/plots/skylake/github_events.jsonparseandstat.pdf differ diff --git a/scripts/plots/skylake/gsoc-2018.json.pdf b/scripts/plots/skylake/gsoc-2018.json.pdf index 461314201..5f88949f3 100644 Binary files a/scripts/plots/skylake/gsoc-2018.json.pdf and b/scripts/plots/skylake/gsoc-2018.json.pdf differ diff --git a/scripts/plots/skylake/gsoc-2018.jsonparseandstat.pdf b/scripts/plots/skylake/gsoc-2018.jsonparseandstat.pdf index 11f09c2f7..a0b46f10b 100644 Binary files a/scripts/plots/skylake/gsoc-2018.jsonparseandstat.pdf and b/scripts/plots/skylake/gsoc-2018.jsonparseandstat.pdf differ diff --git a/scripts/plots/skylake/instruments.json.pdf b/scripts/plots/skylake/instruments.json.pdf index f960e7beb..c2e1361a8 100644 Binary files a/scripts/plots/skylake/instruments.json.pdf and b/scripts/plots/skylake/instruments.json.pdf differ diff --git a/scripts/plots/skylake/instruments.jsonparseandstat.pdf b/scripts/plots/skylake/instruments.jsonparseandstat.pdf index 492090b57..727b2c582 100644 Binary files a/scripts/plots/skylake/instruments.jsonparseandstat.pdf and b/scripts/plots/skylake/instruments.jsonparseandstat.pdf differ diff --git a/scripts/plots/skylake/marine_ik.json.pdf b/scripts/plots/skylake/marine_ik.json.pdf index ed0d9a8fe..bbc8fed4f 100644 Binary files a/scripts/plots/skylake/marine_ik.json.pdf and b/scripts/plots/skylake/marine_ik.json.pdf differ diff --git a/scripts/plots/skylake/marine_ik.jsonparseandstat.pdf b/scripts/plots/skylake/marine_ik.jsonparseandstat.pdf index c8563bb1a..08421e7b0 100644 Binary files a/scripts/plots/skylake/marine_ik.jsonparseandstat.pdf and b/scripts/plots/skylake/marine_ik.jsonparseandstat.pdf differ diff --git a/scripts/plots/skylake/mesh.json.pdf b/scripts/plots/skylake/mesh.json.pdf index 0b88754d0..f5f658ab7 100644 Binary files a/scripts/plots/skylake/mesh.json.pdf and b/scripts/plots/skylake/mesh.json.pdf differ diff --git a/scripts/plots/skylake/mesh.jsonparseandstat.pdf b/scripts/plots/skylake/mesh.jsonparseandstat.pdf index d8ed76b6c..74a91fd31 100644 Binary files a/scripts/plots/skylake/mesh.jsonparseandstat.pdf and b/scripts/plots/skylake/mesh.jsonparseandstat.pdf differ diff --git a/scripts/plots/skylake/mesh.pretty.json.pdf b/scripts/plots/skylake/mesh.pretty.json.pdf index ec54f822e..a989d2cf0 100644 Binary files a/scripts/plots/skylake/mesh.pretty.json.pdf and b/scripts/plots/skylake/mesh.pretty.json.pdf differ diff --git a/scripts/plots/skylake/mesh.pretty.jsonparseandstat.pdf b/scripts/plots/skylake/mesh.pretty.jsonparseandstat.pdf index 45a4fd118..da41fa59c 100644 Binary files a/scripts/plots/skylake/mesh.pretty.jsonparseandstat.pdf and b/scripts/plots/skylake/mesh.pretty.jsonparseandstat.pdf differ diff --git a/scripts/plots/skylake/numbers.json.pdf b/scripts/plots/skylake/numbers.json.pdf index b5845f0e2..1d3230d98 100644 Binary files a/scripts/plots/skylake/numbers.json.pdf and b/scripts/plots/skylake/numbers.json.pdf differ diff --git a/scripts/plots/skylake/numbers.jsonparseandstat.pdf b/scripts/plots/skylake/numbers.jsonparseandstat.pdf index 5f3d0c886..ae7dc2915 100644 Binary files a/scripts/plots/skylake/numbers.jsonparseandstat.pdf and b/scripts/plots/skylake/numbers.jsonparseandstat.pdf differ diff --git a/scripts/plots/skylake/parselinuxtable.txt b/scripts/plots/skylake/parselinuxtable.txt index 8401ee386..1992c28a8 100644 --- a/scripts/plots/skylake/parselinuxtable.txt +++ b/scripts/plots/skylake/parselinuxtable.txt @@ -1,15 +1,15 @@ -"apache_builds" 0.007300 0.775670 0.870808 1.653778 "apache_builds" 0.005891 0.784870 0.870572 1.661334 "apache_builds" 0.006980 0.777089 0.884390 1.668459 "apache_builds" 0.007456 0.828218 0.357577 1.193251 -"canada" 0.001003 0.950349 2.686590 3.637943 "canada" 0.000965 0.886935 2.681328 3.569229 "canada" 0.001034 0.953994 0.791095 1.746123 "canada" 0.000980 0.953998 2.678491 3.633470 -"citm_catalog" 0.001193 0.739766 0.753185 1.494144 "citm_catalog" 0.001202 0.699101 0.752771 1.453074 "citm_catalog" 0.001245 0.739553 0.538863 1.279661 "citm_catalog" 0.001212 0.739028 0.580146 1.320387 -"github_events" 0.013858 0.735330 0.774285 1.523472 "github_events" 0.011755 0.684062 0.777737 1.473554 "github_events" 0.012967 0.735080 0.725050 1.473097 "github_events" 0.011784 0.728434 0.334171 1.074389 -"gsoc-2018" 0.000699 0.666937 0.605854 1.273489 "gsoc-2018" 0.000680 0.565583 0.598737 1.164999 "gsoc-2018" 0.000820 0.667234 0.602705 1.270759 "gsoc-2018" 0.000622 0.665018 0.094223 0.759863 -"instruments" 0.003411 0.800034 0.995731 1.799176 "instruments" 0.003376 0.815329 0.992875 1.811580 "instruments" 0.003573 0.798700 0.779480 1.581753 "instruments" 0.003434 0.797445 0.634332 1.435210 -"marine_ik" 0.000980 1.060306 3.241315 4.302601 "marine_ik" 0.000939 0.997864 3.090407 4.089210 "marine_ik" 0.000840 1.063785 1.136740 2.201365 "marine_ik" 0.000839 1.059532 2.882983 3.943354 -"mesh" 0.001076 0.947018 3.154163 4.102257 "mesh" 0.001120 0.896646 3.142759 4.040525 "mesh" 0.001123 0.930328 0.942846 1.874297 "mesh" 0.001150 0.949446 3.065905 4.016500 -"mesh.pretty" 0.001354 0.773095 1.937286 2.711735 "mesh.pretty" 0.001354 0.738777 1.913346 2.653476 "mesh.pretty" 0.001336 0.773140 0.451032 1.225507 "mesh.pretty" 0.001305 0.774712 1.900699 2.676716 -"numbers" 0.004872 0.801120 2.603307 3.409299 "numbers" 0.004970 0.824353 2.589625 3.418947 "numbers" 0.005350 0.802728 0.586431 1.394509 "numbers" 0.004979 0.811848 2.575365 3.392192 -"random" 0.001486 1.164623 1.419778 2.585886 "random" 0.001457 0.889932 1.417795 2.309184 "random" 0.001559 1.195521 1.346186 2.543266 "random" 0.001463 1.162662 0.719427 1.883552 -"twitterescaped" 0.001337 0.833890 2.510091 3.345319 "twitterescaped" 0.001375 0.755588 2.507559 3.264522 "twitterescaped" 0.001462 0.845094 2.436507 3.283063 "twitterescaped" 0.001369 0.837222 0.435890 1.274481 -"twitter" 0.001216 0.870702 0.814728 1.686645 "twitter" 0.001207 0.736268 0.807286 1.544760 "twitter" 0.001279 0.867516 0.731576 1.600371 "twitter" 0.001169 0.852261 0.398906 1.252336 -"update-center" 0.001396 0.912050 1.221697 2.135144 "update-center" 0.001537 0.835728 1.216266 2.053531 "update-center" 0.001536 0.916837 1.188241 2.106614 "update-center" 0.001433 0.907894 0.511708 1.421035 +"apache_builds" 0.007076 0.780769 0.886382 1.674227 "apache_builds" 0.006586 0.783953 0.883290 1.673829 "apache_builds" 0.006822 0.782415 0.876104 1.665341 "apache_builds" 0.006943 0.773784 0.355739 1.136465 +"canada" 0.000982 0.957865 2.701995 3.660842 "canada" 0.001029 0.888265 2.682198 3.571492 "canada" 0.001074 0.964296 0.790136 1.755506 "canada" 0.000973 0.958123 2.670876 3.629972 +"citm_catalog" 0.001162 0.744692 0.761032 1.506885 "citm_catalog" 0.001191 0.698182 0.754231 1.453604 "citm_catalog" 0.001239 0.744005 0.542710 1.287954 "citm_catalog" 0.001145 0.738397 0.580374 1.319916 +"github_events" 0.013214 0.742430 0.797997 1.553642 "github_events" 0.011561 0.684537 0.793071 1.489169 "github_events" 0.013321 0.745714 0.743376 1.502411 "github_events" 0.011536 0.732541 0.335945 1.080022 +"gsoc-2018" 0.000743 0.680432 0.606119 1.287294 "gsoc-2018" 0.000706 0.565100 0.596288 1.162094 "gsoc-2018" 0.000687 0.681239 0.605474 1.287400 "gsoc-2018" 0.000620 0.664937 0.093844 0.759401 +"instruments" 0.003431 0.805346 1.009534 1.818311 "instruments" 0.003431 0.816164 0.984895 1.804490 "instruments" 0.003665 0.806406 0.788119 1.598191 "instruments" 0.003525 0.797085 0.633919 1.434529 +"marine_ik" 0.001011 1.071855 3.240234 4.313101 "marine_ik" 0.000973 0.997722 3.073006 4.071701 "marine_ik" 0.000949 1.112344 1.141822 2.255115 "marine_ik" 0.000938 1.065826 2.881580 3.948344 +"mesh" 0.001004 0.955469 3.164815 4.121288 "mesh" 0.001156 0.895980 3.141239 4.038375 "mesh" 0.001133 0.983269 0.943240 1.927642 "mesh" 0.001210 0.948312 3.067489 4.017011 +"mesh.pretty" 0.001312 0.779534 1.946704 2.727550 "mesh.pretty" 0.001327 0.738347 1.918519 2.658192 "mesh.pretty" 0.001373 0.780041 0.451492 1.232906 "mesh.pretty" 0.001365 0.777786 1.900366 2.679517 +"numbers" 0.005071 0.810051 2.635595 3.450717 "numbers" 0.004974 0.824387 2.592081 3.421442 "numbers" 0.005283 0.812415 0.585167 1.402865 "numbers" 0.005258 0.801143 2.575507 3.381908 +"random" 0.001566 1.183265 1.429283 2.614113 "random" 0.001492 0.890873 1.435312 2.327677 "random" 0.001667 1.184863 1.354216 2.540746 "random" 0.001430 1.162871 0.719016 1.883318 +"twitterescaped" 0.001357 0.843848 2.535156 3.380361 "twitterescaped" 0.001549 0.756316 2.507179 3.265044 "twitterescaped" 0.001410 0.877832 2.481331 3.360573 "twitterescaped" 0.001345 0.816216 0.435993 1.253554 +"twitter" 0.001238 0.879171 0.820740 1.701149 "twitter" 0.001288 0.735255 0.805525 1.542068 "twitter" 0.001290 0.873760 0.740384 1.615434 "twitter" 0.001332 0.851993 0.398123 1.251448 +"update-center" 0.001412 0.944206 1.240759 2.186378 "update-center" 0.001597 0.835074 1.212884 2.049556 "update-center" 0.001553 0.921042 1.214792 2.137386 "update-center" 0.001411 0.900992 0.511462 1.413865 diff --git a/scripts/plots/skylake/random.json.pdf b/scripts/plots/skylake/random.json.pdf index e2fb0fc75..3904755c0 100644 Binary files a/scripts/plots/skylake/random.json.pdf and b/scripts/plots/skylake/random.json.pdf differ diff --git a/scripts/plots/skylake/random.jsonparseandstat.pdf b/scripts/plots/skylake/random.jsonparseandstat.pdf index f09304116..95de34236 100644 Binary files a/scripts/plots/skylake/random.jsonparseandstat.pdf and b/scripts/plots/skylake/random.jsonparseandstat.pdf differ diff --git a/scripts/plots/skylake/twitter.json.pdf b/scripts/plots/skylake/twitter.json.pdf index 995ac2793..4183727d2 100644 Binary files a/scripts/plots/skylake/twitter.json.pdf and b/scripts/plots/skylake/twitter.json.pdf differ diff --git a/scripts/plots/skylake/twitter.jsondistinctuserid.pdf b/scripts/plots/skylake/twitter.jsondistinctuserid.pdf index 64f39c58a..453a06710 100644 Binary files a/scripts/plots/skylake/twitter.jsondistinctuserid.pdf and b/scripts/plots/skylake/twitter.jsondistinctuserid.pdf differ diff --git a/scripts/plots/skylake/twitter.jsonparseandstat.pdf b/scripts/plots/skylake/twitter.jsonparseandstat.pdf index 2ededf379..934353431 100644 Binary files a/scripts/plots/skylake/twitter.jsonparseandstat.pdf and b/scripts/plots/skylake/twitter.jsonparseandstat.pdf differ diff --git a/scripts/plots/skylake/twitterescaped.json.pdf b/scripts/plots/skylake/twitterescaped.json.pdf index 2503f396b..cf4993b79 100644 Binary files a/scripts/plots/skylake/twitterescaped.json.pdf and b/scripts/plots/skylake/twitterescaped.json.pdf differ diff --git a/scripts/plots/skylake/twitterescaped.jsonparseandstat.pdf b/scripts/plots/skylake/twitterescaped.jsonparseandstat.pdf index aef29d99c..65c01e626 100644 Binary files a/scripts/plots/skylake/twitterescaped.jsonparseandstat.pdf and b/scripts/plots/skylake/twitterescaped.jsonparseandstat.pdf differ diff --git a/scripts/plots/skylake/update-center.json.pdf b/scripts/plots/skylake/update-center.json.pdf index 4e07cf0c5..d14c08a03 100644 Binary files a/scripts/plots/skylake/update-center.json.pdf and b/scripts/plots/skylake/update-center.json.pdf differ diff --git a/scripts/plots/skylake/update-center.jsonparseandstat.pdf b/scripts/plots/skylake/update-center.jsonparseandstat.pdf index 2bf6d0c36..0991f9747 100644 Binary files a/scripts/plots/skylake/update-center.jsonparseandstat.pdf and b/scripts/plots/skylake/update-center.jsonparseandstat.pdf differ diff --git a/scripts/setupfortesting/disablehyperthreading.sh b/scripts/setupfortesting/disablehyperthreading.sh new file mode 100755 index 000000000..38a9c8a1a --- /dev/null +++ b/scripts/setupfortesting/disablehyperthreading.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +# Be careful to not skip the space at the beginning nor the end +CPUS_TO_SKIP=" $(cat /sys/devices/system/cpu/cpu*/topology/thread_siblings_list | sed 's/[^0-9].*//' | sort | uniq | tr "\r\n" " ") " + + +for CPU_PATH in /sys/devices/system/cpu/cpu[0-9]*; do + CPU="$(echo $CPU_PATH | tr -cd "0-9")" + echo "$CPUS_TO_SKIP" | grep " $CPU " > /dev/null + if [ $? -ne 0 ]; then + echo 0 > $CPU_PATH/online + fi +done + +egrep 'siblings|cpu cores' /proc/cpuinfo | head -2 diff --git a/scripts/setupfortesting/powerpolicy.sh b/scripts/setupfortesting/powerpolicy.sh new file mode 100755 index 000000000..c505d898b --- /dev/null +++ b/scripts/setupfortesting/powerpolicy.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +# taken from http://hbfs.wordpress.com/2013/06/18/fast-path-finding-part-ii/ +# might require sudo apt-get install cpufrequtils +# invoke with performance or ondemand +# type cpufreq-info to check results, you can also verify with cat /proc/cpuinfo +# enumerate found CPUs +cpus=$( grep processor /proc/cpuinfo | cut -d: -f 2 ) + + +if [ "$1" = "ondemand" ]; then + echo "setting up ondemand" + policy="ondemand" +elif [ "$1" = "performance" ]; then + echo "setting up for performance" + policy="performance" +elif [ "$1" = "list" ]; then + cpufreq-info + exit 0 +else + echo "usage: powerpolicy.sh ondemand | performance list" + exit -1 +fi + +echo "chosen policy " $1 +# set governor for each CPU +# +for cpu in ${cpus[@]} +do + cpufreq-set -c $cpu -g $1 +done diff --git a/scripts/setupfortesting/setupfortesting.sh b/scripts/setupfortesting/setupfortesting.sh new file mode 100755 index 000000000..75352334f --- /dev/null +++ b/scripts/setupfortesting/setupfortesting.sh @@ -0,0 +1,6 @@ +cd "${0%/*}" +export CXX=g++-7 +export CC=gcc-7 +./powerpolicy.sh performance +./disablehyperthreading.sh +./turboboost.sh on diff --git a/scripts/setupfortesting/turboboost.sh b/scripts/setupfortesting/turboboost.sh new file mode 100755 index 000000000..269e370c1 --- /dev/null +++ b/scripts/setupfortesting/turboboost.sh @@ -0,0 +1,51 @@ +#!/bin/sh +# stolen from https://github.com/DropD/fnc-simplex/blob/master/linux_turboboost.sh + +# you might need to run sudo apt-get install msr-tools +# Toggle Turbo Boost for Ivy Bridge CPUs (should work for all newer Core) +# Requires a fairly new Linux kernel (let's say 3.0+) +# Written by Donjan Rodic, released for free use + +# check current real frequency with sudo turbostat -s -i1 + +sudo modprobe msr + +# all_cores FOO +# perform FOO(i) for each core i +all_cores() { + NPROCS=`cat /proc/cpuinfo | grep "core id" | wc -l` + NPROCS=$(($NPROCS - 1)) + for i in `seq 0 1 $NPROCS`; do + $1 $i + done +} + + +# report Turbo Boost state on core $1 +read_tb() { + ret=`sudo rdmsr -p"$1" 0x1a0 -f 38:38` + [ $ret -eq 0 ] && echo "$1": on || echo "$1": off +} + +# enable Turbo Boost on core $1 +enable_tb() { + sudo wrmsr -p"$1" 0x1a0 0x850089 +} + +# disable Turbo Boost on core $1 +disable_tb() { + sudo wrmsr -p"$1" 0x1a0 0x4000850089 +} + + +if [ "$1" = "on" ]; then + all_cores enable_tb + all_cores read_tb +elif [ "$1" = "off" ]; then + all_cores disable_tb + all_cores read_tb +elif [ "$1" = "list" ]; then + all_cores read_tb +else + echo "usage: turboboost.sh on | off | list" +fi