admin/lifting-bits-mcsema
1
0
Fork 0
mirror of https://github.com/lifting-bits/mcsema synced 2026-06-08 15:31:09 +00:00
Code Issues Packages Projects Releases Wiki Activity

Test suite hotfix (#601)

Browse Source 
* Add explicit targets to unit tests from src

* run_tests.py no longer crashes when program does not generate expected file as output.

* Fix bunch of issues if file is not lifted/recompiled properly

* Gzip empty command line arguments test case
This commit is contained in:
Aiethel
2019-10-30 18:35:59 +01:00
committed by Peter Goodman
parent d42db800a4
commit a577f39745
45 changed files with 107 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/integration_tests/run_tests.py
+19 -9
View File
@@ -135,9 +135,14 @@ def check_arguments(args):
def exec_and_log_fail(args):
pipes = subprocess.Popen(args, stdout = subprocess.PIPE, stderr = subprocess.PIPE)
std_out, std_err = pipes.communicate()
ret_code = pipes.returncode
try:
pipes = subprocess.Popen(args, stdout = subprocess.PIPE, stderr = subprocess.PIPE)
std_out, std_err = pipes.communicate(timeout = 180)
ret_code = pipes.returncode
except subprocess.TimeoutExpired as e:
pipes.terminate()
print("Timeout!")
return False
if ret_code:
print("** stdout:")
@@ -262,7 +267,7 @@ def parallel_lift(*t_args):
res = config.lift(test_dir)
# TODO: More fine grained log
if res != Config.Result.SUCCESS:
results[self.id] = result_data.TCData(self.id, self.recompiled, self.binary)
results[config.id] = result_data.TCData(config.id,config.recompiled, config.binary)
def get_configs(directory, allowed_tags, batched):
result = []
@@ -400,11 +405,14 @@ class Runner:
base_name = os.path.basename(name)
actual = os.path.join(self.t_recompiled, base_name)
expected = os.path.join(self.t_bin, base_name)
if not filecmp.cmp(expected, actual):
counterexample += 'Files do not match: ' + basename + '\n'
try:
if not filecmp.cmp(expected, actual):
counterexample += 'Files do not match: ' + base_name + '\n'
correct = correct and False
except FileNotFoundError as e:
counterexample += 'File {} not found/produced! '.format(base_name)
correct = correct and False
result = result_data.RUN if correct else result_data.FAIL
return (result, counterexample)
@@ -443,8 +451,9 @@ class Runner:
actual = _exec(self, self.t_recompiled, [filename] + detail.cmd, stdin)
return self.compare(expected, actual, files)
except subprocess.TimeoutExpired as e:
return result_data.TIMEOUT
return result_data.TIMEOUT, "Timeout"
except FileNotFoundError as e:
return result_data.ERROR, "File not found"
def run(self, detail):
self.set_up()
@@ -511,6 +520,7 @@ g_complex_test = {
TestDetails(['-l', './dec_data.txt.gz']).set_files(['inputs/dec_data.txt.gz']),
TestDetails(['-df', './dec_data.txt.gz']).set_files(['inputs/dec_data.txt.gz']).\
set_check(['dec_data.txt']),
TestDetails(['']),
],
"cat":
tests/integration_tests/src/all_data_array.c
+2
View File
@@ -1,4 +1,6 @@
/* TAGS: min c */
/* LIFT_OPTS: explicit +--explicit_args +--explicit_args_count 8 */
/* LIFT_OPTS: default */
/*
* Copyright (c) 2018 Trail of Bits, Inc.
*
tests/integration_tests/src/all_globals.c
+2
View File
@@ -1,4 +1,6 @@
/* TAGS: min c */
/* LIFT_OPTS: explicit +--explicit_args +--explicit_args_count 8 */
/* LIFT_OPTS: default */
/*
* Copyright (c) 2018 Trail of Bits, Inc.
*
tests/integration_tests/src/all_stringpool.c
+2
View File
@@ -1,4 +1,6 @@
/* TAGS: min c */
/* LIFT_OPTS: explicit +--explicit_args +--explicit_args_count 8 */
/* LIFT_OPTS: default */
/*
* Copyright (c) 2018 Trail of Bits, Inc.
*
tests/integration_tests/src/all_switch.c
+2
View File
@@ -1,4 +1,6 @@
/* TAGS: min c */
/* LIFT_OPTS: explicit +--explicit_args +--explicit_args_count 8 */
/* LIFT_OPTS: default */
/* TEST: 12 */
/* TEST: 15 */
/*
tests/integration_tests/src/array.c
+2
View File
@@ -1,4 +1,6 @@
/* TAGS: min c */
/* LIFT_OPTS: explicit +--explicit_args +--explicit_args_count 8 */
/* LIFT_OPTS: default */
/*
* Copyright (c) 2018 Trail of Bits, Inc.
*
tests/integration_tests/src/bubblesort.c
+2
View File
@@ -1,4 +1,6 @@
/* TAGS: min c */
/* LIFT_OPTS: explicit +--explicit_args +--explicit_args_count 8 */
/* LIFT_OPTS: default */
/* TEST: 12 */
/* TEST: 14 */
/* TEST: 0 */
tests/integration_tests/src/calc.c
+2
View File
@@ -1,4 +1,6 @@
/* TAGS: min c */
/* LIFT_OPTS: explicit +--explicit_args +--explicit_args_count 8 */
/* LIFT_OPTS: default */
/* TEST: */
/* STDIN: Finputs/calc_input1.txt */
/* TEST: */
tests/integration_tests/src/complex_double.c
+2
View File
@@ -1,4 +1,6 @@
/* TAGS: min c */
/* LIFT_OPTS: explicit +--explicit_args +--explicit_args_count 8 */
/* LIFT_OPTS: default */
/* LD_OPTS: -lm */
/*
* Copyright (c) 2018 Trail of Bits, Inc.
tests/integration_tests/src/complex_float.c
+2
View File
@@ -1,4 +1,6 @@
/* TAGS: min c */
/* LIFT_OPTS: explicit +--explicit_args +--explicit_args_count 8 */
/* LIFT_OPTS: default */
/* LD_OPTS: -lm */
/*
* Copyright (c) 2018 Trail of Bits, Inc.
tests/integration_tests/src/complex_long_double.c
+2
View File
@@ -1,4 +1,6 @@
/* TAGS: min c */
/* LIFT_OPTS: explicit +--explicit_args +--explicit_args_count 8 */
/* LIFT_OPTS: default */
/* LD_OPTS: -lm */
/*
* Copyright (c) 2018 Trail of Bits, Inc.
tests/integration_tests/src/complex_numbers.c
+2
View File
@@ -1,4 +1,6 @@
/* TAGS: min c */
/* LIFT_OPTS: explicit +--explicit_args +--explicit_args_count 8 */
/* LIFT_OPTS: default */
/* LD_OPTS: -lm */
/*
* Copyright (c) 2018 Trail of Bits, Inc.
tests/integration_tests/src/dot_product.c
+2
View File
@@ -1,4 +1,6 @@
/* TAGS: min c */
/* LIFT_OPTS: explicit +--explicit_args +--explicit_args_count 8 */
/* LIFT_OPTS: default */
/*
* Copyright (c) 2018 Trail of Bits, Inc.
*
tests/integration_tests/src/fibonacci.c
+2
View File
@@ -1,4 +1,6 @@
/* TAGS: min c */
/* LIFT_OPTS: explicit +--explicit_args +--explicit_args_count 8 */
/* LIFT_OPTS: default */
/* TEST: 12 */
/* TEST: 26 */
/* TEST: 2 */
tests/integration_tests/src/fmodf.cpp
+2
View File
@@ -1,4 +1,6 @@
/* TAGS: min cpp */
/* LIFT_OPTS: explicit +--explicit_args +--explicit_args_count 8 */
/* LIFT_OPTS: default */
/*
* Copyright (c) 2018 Trail of Bits, Inc.
*
tests/integration_tests/src/global-ctors-dtors-return-type.c
+2
View File
@@ -1,4 +1,6 @@
/* TAGS: min c */
/* LIFT_OPTS: explicit +--explicit_args +--explicit_args_count 8 */
/* LIFT_OPTS: default */
#include <assert.h>
#include <stdio.h>
tests/integration_tests/src/global-ctors-dtors.c
+2
View File
@@ -1,4 +1,6 @@
/* TAGS: min c */
/* LIFT_OPTS: explicit +--explicit_args +--explicit_args_count 8 */
/* LIFT_OPTS: default */
#include <assert.h>
#include <stdio.h>
tests/integration_tests/src/global_array.c
+2
View File
@@ -1,4 +1,6 @@
/* TAGS: min c */
/* LIFT_OPTS: explicit +--explicit_args +--explicit_args_count 8 */
/* LIFT_OPTS: default */
/*
* Copyright (c) 2018 Trail of Bits, Inc.
*
tests/integration_tests/src/global_var.cpp
+2
View File
@@ -1,4 +1,6 @@
/* TAGS: min cpp */
/* LIFT_OPTS: explicit +--explicit_args +--explicit_args_count 8 */
/* LIFT_OPTS: default */
/*
* Copyright (c) 2017 Trail of Bits, Inc.
*
tests/integration_tests/src/globals_and_io.c
+2
View File
@@ -1,4 +1,6 @@
/* TAGS: min c */
/* LIFT_OPTS: explicit +--explicit_args +--explicit_args_count 8 */
/* LIFT_OPTS: default */
/*
* Copyright (c) 2018 Trail of Bits, Inc.
*
tests/integration_tests/src/hello_c.c
+2
View File
@@ -1,4 +1,6 @@
/* TAGS: min c */
/* LIFT_OPTS: explicit +--explicit_args +--explicit_args_count 8 */
/* LIFT_OPTS: default */
/*
* Copyright (c) 2019 Trail of Bits, Inc.
*
tests/integration_tests/src/hello_cpp.cpp
+2
View File
@@ -1,4 +1,6 @@
/* TAGS: min cpp */
/* LIFT_OPTS: explicit +--explicit_args +--explicit_args_count 8 */
/* LIFT_OPTS: default */
/*
* Copyright (c) 2019 Trail of Bits, Inc.
*
tests/integration_tests/src/helloworld.c
+2
View File
@@ -1,4 +1,6 @@
/* TAGS: min c */
/* LIFT_OPTS: explicit +--explicit_args +--explicit_args_count 8 */
/* LIFT_OPTS: default */
/*
* Copyright (c) 2018 Trail of Bits, Inc.
*
tests/integration_tests/src/iostream_basics.cpp
+2
View File
@@ -1,4 +1,6 @@
/* TAGS: min cpp */
/* LIFT_OPTS: explicit +--explicit_args +--explicit_args_count 8 */
/* LIFT_OPTS: default */
/*
* Copyright (c) 2018 Trail of Bits, Inc.
*
tests/integration_tests/src/local-array.c
+2
View File
@@ -1,4 +1,6 @@
/* TAGS: min c */
/* LIFT_OPTS: explicit +--explicit_args +--explicit_args_count 8 */
/* LIFT_OPTS: default */
/*
* Copyright (c) 2018 Trail of Bits, Inc.
*
tests/integration_tests/src/matrix_vector_mult.c
+2
View File
@@ -1,4 +1,6 @@
/* TAGS: min c */
/* LIFT_OPTS: explicit +--explicit_args +--explicit_args_count 8 */
/* LIFT_OPTS: default */
#include <stdio.h>
static const int A [] = { 1, 2, 3, 4, 5, 6, 7, 8, 9 };
tests/integration_tests/src/open_close_dir.c
+2
View File
@@ -1,4 +1,6 @@
/* TAGS: min c */
/* LIFT_OPTS: explicit +--explicit_args +--explicit_args_count 8 */
/* LIFT_OPTS: default */
/* TEST: /usr */
/* TEST: /eqeqeqwe */
/*
tests/integration_tests/src/operator_new.cpp
+2
View File
@@ -1,4 +1,6 @@
/* TAGS: min cpp */
/* LIFT_OPTS: explicit +--explicit_args +--explicit_args_count 8 */
/* LIFT_OPTS: default */
/*
* Copyright (c) 2018 Trail of Bits, Inc.
*
tests/integration_tests/src/pointers.c
+2
View File
@@ -1,4 +1,6 @@
/* TAGS: min c */
/* LIFT_OPTS: explicit +--explicit_args +--explicit_args_count 8 */
/* LIFT_OPTS: default */
/*
* Copyright (c) 2018 Trail of Bits, Inc.
*
tests/integration_tests/src/printf_floats.cpp
+2
View File
@@ -1,4 +1,6 @@
/* TAGS: min cpp */
/* LIFT_OPTS: explicit +--explicit_args +--explicit_args_count 8 */
/* LIFT_OPTS: default */
/*
* Copyright (c) 2018 Trail of Bits, Inc.
*
tests/integration_tests/src/pthread.cpp
+2
View File
@@ -1,4 +1,6 @@
/* TAGS: min cpp */
/* LIFT_OPTS: explicit +--explicit_args +--explicit_args_count 8 */
/* LIFT_OPTS: default */
/* LD_OPTS: -lpthread */
/*
* Copyright (c) 2017 Trail of Bits, Inc.
tests/integration_tests/src/qsort.c
+2
View File
@@ -1,4 +1,6 @@
/* TAGS: min c */
/* LIFT_OPTS: explicit +--explicit_args +--explicit_args_count 8 */
/* LIFT_OPTS: default */
/*
* Copyright (c) 2018 Trail of Bits, Inc.
*
tests/integration_tests/src/qsort_function_ptrs.cpp
+2
View File
@@ -1,4 +1,6 @@
/* TAGS: min cpp */
/* LIFT_OPTS: explicit +--explicit_args +--explicit_args_count 8 */
/* LIFT_OPTS: default */
/* TEST: 23 */
/* TEST: 43 */
/* TEST: 435 */
tests/integration_tests/src/rand_and_strtol.c
+2
View File
@@ -1,4 +1,6 @@
/* TAGS: min c */
/* LIFT_OPTS: explicit +--explicit_args +--explicit_args_count 8 */
/* LIFT_OPTS: default */
/*
* Copyright (c) 2018 Trail of Bits, Inc.
*
tests/integration_tests/src/readdir.c
+2
View File
@@ -1,4 +1,6 @@
/* TAGS: min c */
/* LIFT_OPTS: explicit +--explicit_args +--explicit_args_count 8 */
/* LIFT_OPTS: default */
/* TEST: readdir.c */
/* TEST: /tmp */
/* TEST: file-that-does-not-exist */
tests/integration_tests/src/simple_array.c
+2
View File
@@ -1,4 +1,6 @@
/* TAGS: min c */
/* LIFT_OPTS: explicit +--explicit_args +--explicit_args_count 8 */
/* LIFT_OPTS: default */
/*
* Copyright (c) 2018 Trail of Bits, Inc.
*
tests/integration_tests/src/simple_exit.c
+2
View File
@@ -1,4 +1,6 @@
/* TAGS: min c */
/* LIFT_OPTS: explicit +--explicit_args +--explicit_args_count 8 */
/* LIFT_OPTS: default */
/*
* Copyright (c) 2018 Trail of Bits, Inc.
*
tests/integration_tests/src/simple_for_loop.c
+2
View File
@@ -1,4 +1,6 @@
/* TAGS: min c */
/* LIFT_OPTS: explicit +--explicit_args +--explicit_args_count 8 */
/* LIFT_OPTS: default */
/*
* Copyright (c) 2018 Trail of Bits, Inc.
*
tests/integration_tests/src/simple_main.c
+2 -8
View File
@@ -1,12 +1,6 @@
/* TAGS: min c */
/* CC_OPTS: */
/* LD_OPTS: */
/* LIFT_OPTS: EXP +explicit_args +explicit_args_count 8 !abi_libraries */
/* TEST: --help */
/* STDIN: F/inputs.txt */
/* TEST: --version */
/* TEST: --extract nonsense.txt */
/* STDIN: Hello World\nThis Should Be Together */
/* LIFT_OPTS: explicit +--explicit_args +--explicit_args_count 8 */
/* LIFT_OPTS: default */
/*
* Copyright (c) 2018 Trail of Bits, Inc.
*
tests/integration_tests/src/struct.c
+2
View File
@@ -1,4 +1,6 @@
/* TAGS: min c */
/* LIFT_OPTS: explicit +--explicit_args +--explicit_args_count 8 */
/* LIFT_OPTS: default */
/*
* Copyright (c) 2018 Trail of Bits, Inc.
*
tests/integration_tests/src/struct_func_ptr.cpp
+2
View File
@@ -1,4 +1,6 @@
/* TAGS: min cpp */
/* LIFT_OPTS: explicit +--explicit_args +--explicit_args_count 8 */
/* LIFT_OPTS: default */
/* TEST: */
/* STDIN: 4\n4\n */
/* TEST: */
tests/integration_tests/src/template_function_ptrs.cpp
+2
View File
@@ -1,4 +1,6 @@
/* TAGS: min cpp */
/* LIFT_OPTS: explicit +--explicit_args +--explicit_args_count 8 */
/* LIFT_OPTS: default */
/* TEST: 42 */
/* TEST: -543 */
/* TEST: 21 */
tests/integration_tests/src/virtual.cpp
+2
View File
@@ -1,4 +1,6 @@
/* TAGS: min cpp */
/* LIFT_OPTS: explicit +--explicit_args +--explicit_args_count 8 */
/* LIFT_OPTS: default */
/*
* Copyright (c) 2018 Trail of Bits, Inc.
*
tests/integration_tests/src/virtual_simpler.cpp
+2
View File
@@ -1,4 +1,6 @@
/* TAGS: min cpp */
/* LIFT_OPTS: explicit +--explicit_args +--explicit_args_count 8 */
/* LIFT_OPTS: default */
/*
* Copyright (c) 2018 Trail of Bits, Inc.
*
tests/integration_tests/src/x86_bts.c
+2
View File
@@ -1,4 +1,6 @@
/* TAGS: min c */
/* LIFT_OPTS: explicit +--explicit_args +--explicit_args_count 8 */
/* LIFT_OPTS: default */
/*
* Copyright (c) 2018 Trail of Bits, Inc.
*