diff --git a/.gitignore b/.gitignore index ea1da220..ebcfd9ff 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ third_party/* build/* generated/* +gtest_build/* *.o *.ll diff --git a/.travis.yml b/.travis.yml index b18d6c1a..afda8192 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,11 +1,11 @@ sudo: required dist: trusty osx_image: xcode8 -language: cpp +language: generic os: - linux - osx -compiler: +compiler: - gcc - clang matrix: @@ -14,11 +14,14 @@ matrix: os: osx - compiler: clang os: linux -before_install: - - if [ "$TRAVIS_OS_NAME" = "linux" ]; then sudo apt-get update -qq && sudo apt-get install -y libunwind8 libunwind8-dev libc6-dev-i386 lib32gcc-4.8-dev lib32stdc++-4.8-dev && sudo pip install futures; fi + allow_failures: + - os: osx +install: + - ./scripts/travis/install.sh +before_script: + - ./scripts/x86/generate_tests.sh script: - - "./scripts/bootstrap.sh" - - "./scripts/run_tests_x86.sh" + - ./scripts/x86/run_tests.sh notifications: slack: secure: eW0eTnlEz1Pa68ZJOps7L0UscoxoFa+JLuOIOy4PPrLs1DfIiGdhHMfWs2U8mF8YaBYmrXgav/1IxNVtdPxiwfyLdYbiNSoEfo1ofPpizTH02flXthnUVIzlQD1jT6OSk840vskT2WWufBEkoaEIxFDXVI8PFtAtIYyfveVEnRdkTPUvuLV0pTX92c+ZCH3FIWtDey+4aVUwsttJSVloDSExy27LuoBLDPSNX8goiZFFrIcBRpspumysEJXdj5FSNiED9TQA1NCGWhsUqUlTPMYp1B8M/lKciugqhAlGijSL3lOisJZWsf/YZcto/N7wNMU54YNbfmMUZu6K9UkveVOD8qtlAs/8MDPyH0O+s/Hf2y6KGb7PSO9sxYF/WWRLNqxp8SgmY3IKu8niglXsHBNd4cXibvM3a2KJoJo453PI48XAfaygPYcYcuNedsQT9Fng384vpUjV7YLLU1452qTjx5T5SarjeF4ExtzJVYAQWhFT7RkNpdtDnKNLczwicM8wRSCHfEmGLR1Dpw5zQlS3uFJydTWTgAeLSHKKkfgqLYiR7/Ongv1M8Q+JUgYtF2dV9PLEpb0QGzw6B/RnW2vGNWIE+JJCjpZTizQj0f8UIh+k//v9jL4simVHRDPArnvJaFP0T4BrUl4Z4XN/dYFOogp9rCHgQCvqVOCPrFI= diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 00000000..9997f2d4 --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,205 @@ +# Copyright 2015 Peter Goodman (peter@trailofbits.com), all rights reserved. + +set(CMAKE_ASM_COMPILER ${CMAKE_CXX_COMPILER}) + +project(remill) +cmake_minimum_required (VERSION 3.2) + +enable_language(ASM) + +LIST(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/cmake") + +set(LLVM_DIR "/usr/lib/llvm-3.9/lib/cmake/llvm/") +find_package(LLVM 3.9 REQUIRED) +find_package(XED REQUIRED) +find_package(GLOG REQUIRED) +find_package(GFLAGS REQUIRED) +find_package(Protobuf REQUIRED) + +include_directories("${LLVM_INCLUDE_DIRS}") +include_directories("${XED_INCLUDE_DIRS}") +include_directories("${GLOG_INCLUDE_DIRS}") +include_directories("${GFLAGS_INCLUDE_DIRS}") +include_directories("${PROTOBUF_INCLUDE_DIRS}") +include_directories("${CMAKE_CURRENT_LIST_DIR}") + +if(NOT DEFINED CMAKE_C_COMPILER) + set(CMAKE_C_COMPILER "${LLVM_INSTALL_PREFIX}/bin/clang-3.9") +endif() + +if(NOT DEFINED CMAKE_CXX_COMPILER) + set(CMAKE_CXX_COMPILER ${CMAKE_C_COMPILER}) +endif() + +if(NOT DEFINED CMAKE_ASM_COMPILER) + set(CMAKE_ASM_COMPILER ${CMAKE_C_COMPILER}) +endif() + +if(NOT DEFINED CMAKE_LLVM_LINK) + set(CMAKE_LLVM_LINK "${LLVM_INSTALL_PREFIX}/bin/llvm-link") +endif() + +add_definitions(${LLVM_DEFINITIONS}) + +add_executable(remill-lift + remill/Lift.cpp + remill/Arch/X86/Arch.cpp + remill/Arch/Arch.cpp + remill/Arch/Instruction.cpp + remill/CFG/CFG.cpp + remill/BC/Translator.cpp + remill/BC/Util.cpp + remill/BC/IntrinsicTable.cpp + remill/OS/OS.cpp +) + +target_link_libraries(remill-lift + ${GFLAGS_LIBRARIES} + ${GLOG_LIBRARIES} + ${XED_LIBRARIES} + ${PROTOBUF_LIBRARIES} + + LLVMIRReader LLVMBitReader LLVMBitWriter + LLVMTransformUtils + LLVMScalarOpts + LLVMSupport +) + +target_compile_options(remill-lift PRIVATE + -x c++ + + -Wall + -Wextra + -Werror + -pedantic + + -g3 + -O0 + + # LLVM headers break these :-( + #-Wshadow + #-Wshorten-64-to-32 + + -Wno-unused-parameter + -Wno-c++98-compat + -Wno-unreachable-code-return + -Wno-nested-anon-types + -Wno-extended-offsetof + -Wno-gnu-anonymous-struct + -Wno-variadic-macros + -Wno-gnu-zero-variadic-macro-arguments + -Wno-gnu-statement-expression + -Wno-error=unused-command-line-argument + -Wno-override-module + -Wno-return-type-c-linkage + -Wno-c99-extensions + -Wno-ignored-attributes + + -fno-omit-frame-pointer + -fvisibility-inlines-hidden + + -fno-rtti + -DGOOGLE_PROTOBUF_NO_RTTI + + -std=gnu++11 + + -fPIC + -fpie + -m64 + + -DINSTALL_SEMANTICS_DIR="${CMAKE_INSTALL_PREFIX}/share/remill/semantics/" + -DBUILD_SEMANTICS_DIR="${CMAKE_CURRENT_BINARY_DIR}/" +) + +add_custom_target(semantics) + +macro(COMPILE_X86_SEMANTICS name address_size has_avx has_avx512) + set(SEMANTICS_CXX_FLAGS + + -I${PROJECT_SOURCE_DIR} + -emit-llvm + -DADDRESS_SIZE_BITS=${address_size} + -DHAS_FEATURE_AVX=${has_avx} + -DHAS_FEATURE_AVX512=${has_avx512} + + -std=gnu++11 + + -m${address_size} + + -Wall + -Werror + -Wshadow + -Wconversion + -pedantic + + -Wshorten-64-to-32 + -Wno-gnu-anonymous-struct + -Wno-return-type-c-linkage + -Wno-gnu-zero-variadic-macro-arguments + -Wno-nested-anon-types + -Wno-extended-offsetof + -Wno-gnu-statement-expression + -Wno-c99-extensions + -Wno-ignored-attributes + + -ffreestanding + + -fno-vectorize + -fno-slp-vectorize + -fno-common + -fno-builtin + -fno-exceptions + -fno-rtti + -fno-asynchronous-unwind-tables + ) + + # Note: `-O0 -g3` is significant here. We need to make sure that we can + # recover variables names within `__remill_basic_block` that may have + # been lost when the code was compiled. + add_custom_target(BasicBlock.${name}.bc + COMMAND ${CMAKE_CXX_COMPILER} + ${SEMANTICS_CXX_FLAGS} + -O0 -g3 + -c ${PROJECT_SOURCE_DIR}/remill/Arch/X86/Runtime/BasicBlock.cpp + -o BasicBlock.${name}.bc + SOURCES remill/Arch/X86/Runtime/BasicBlock.cpp + ) + + add_custom_target(Instructions.${name}.bc + COMMAND ${CMAKE_CXX_COMPILER} + ${SEMANTICS_CXX_FLAGS} + -O3 -g0 + -c ${PROJECT_SOURCE_DIR}/remill/Arch/X86/Runtime/Instructions.cpp + -o Instructions.${name}.bc + SOURCES remill/Arch/X86/Runtime/Instructions.cpp + ) + + add_custom_target(${name}.bc + COMMAND ${CMAKE_LLVM_LINK} + BasicBlock.${name}.bc + Instructions.${name}.bc + -o ${name}.bc + DEPENDS BasicBlock.${name}.bc + Instructions.${name}.bc + ) + + add_dependencies(semantics ${name}.bc) + install( + FILES ${CMAKE_CURRENT_BINARY_DIR}/${name}.bc + DESTINATION share/remill/semantics + ) + +endmacro() + +COMPILE_X86_SEMANTICS(x86 32 0 0) +COMPILE_X86_SEMANTICS(x86_avx 32 1 0) +COMPILE_X86_SEMANTICS(x86_avx512 32 1 1) +COMPILE_X86_SEMANTICS(amd64 64 0 0) +COMPILE_X86_SEMANTICS(amd64_avx 64 1 0) +COMPILE_X86_SEMANTICS(amd64_avx512 64 1 1) + +install( + TARGETS remill-lift + RUNTIME DESTINATION bin + LIBRARY DESTINATION lib +) \ No newline at end of file diff --git a/README.md b/README.md index 76f14d5d..36dc765a 100644 --- a/README.md +++ b/README.md @@ -1,174 +1,193 @@ # Remill -[![Build Status](https://travis-ci.org/trailofbits/remill.svg?branch=master)](https://travis-ci.org/trailofbits/remill) + Remill is a static binary translator that translates machine code into [LLVM bitcode](http://llvm.org/docs/LangRef.html). It translates x86 and amd64 machine code (including AVX and AVX512) into LLVM bitcode. -## Documentation +## Build Status -Please consult the [documentation](docs/README.md), which describes the goals, design, and inner workings of Remill. +| | master | +| ----- | ------ | +| Linux | [![Build Status](https://travis-ci-job-status.herokuapp.com/badge/trailofbits/remill/master/linux)](https://travis-ci.org/trailofbits/remill) | +| macOS | [![Build Status](https://travis-ci-job-status.herokuapp.com/badge/trailofbits/remill/master/osx)](https://travis-ci.org/trailofbits/remill) | -## Purpose +## Additional Documentation + + - [How to contribute](docs/CONTRIBUTING.md) + - [How to implement the semantics of an instruction](docs/ADD_AN_INSTRUCTION.md) + - [How instructions are lifted](docs/LIFE_OF_AN_INSTRUCTION.md) + - [How binaries are represented](docs/CFG_FORMAT.md) + - [The design and architecture of Remill](docs/DESIGN.md) -Remill translates machine code, and *only* machine code, into LLVM bitcode. Remill's translation approach is inspired by [dynamic](https://software.intel.com/en-us/articles/pin-a-dynamic-binary-instrumentation-tool) -[binary](https://github.com/DynamoRIO/dynamorio) [translators](https://github.com/Granary/granary2). Remill translates one [basic block](https://en.wikipedia.org/wiki/Basic_block) of machine code into LLVM bitcode at a time. The translation process defers many decisions to downstream consumers on how the translated bitcode should be interpreted. +## Getting Help -## Goals +If you are experiencing undocumented problems with Remill then ask for help in the `#tool-remill` channel of the [Empire Hacking Slack](https://empireslacking.herokuapp.com/). -Remill was designed with the following goals in mind. +## Supported Platforms -- It should be easy to add new instruction implementations. Instruction - semantics are implemented using C++. Instruction implementations should be - thoroughly tested. +Remill is supported on Linux platforms and has been tested on Ubuntu 14.04 and 16.04. -- Remill-produced bitcode should achieve the sometimes conflicting goals of - maintaining the semantics of the translated machine code and enabling - aggressive optimization of the produced bitcode. +We are actively working on porting Remill to macOS. -- Decisions affecting the use of the produced bitcode should be deferred via - intrinsics. Remill-produced bitcode should not commit a consumer of that - bitcode to one use case. +## Dependencies -## Design +| Name | Version | +| ---- | ------- | +| [Git](https://git-scm.com/) | Latest | +| [CMake](https://cmake.org/) | 3.2+ | +| [Google Log](https://github.com/google/glog) | 0.3.3 | +| [Google Test](https://github.com/google/googletest) | 1.6.0 | +| [Google Protobuf](https://github.com/google/protobuf) | 2.4.1 | +| [LLVM](http://llvm.org/) | 3.9 | +| [Clang](http://clang.llvm.org/) | 3.9 | +| [Intel XED](https://software.intel.com/en-us/articles/xed-x86-encoder-decoder-software-library) | 2016-02-02 | +| [Python](https://www.python.org/) | 2.7 | +| [Python Package Index](https://pypi.python.org/pypi) | Latest | +| [python-magic](https://pypi.python.org/pypi/python-magic) | Latest | +| Unzip | Latest | +| [python-protobuf](https://pypi.python.org/pypi/protobuf) | 2.4.1 | +| [Binary Ninja](https://binary.ninja) | Latest | +| [IDA Pro](https://www.hex-rays.com/products/ida) | 6.7+ | -### Intrinsics +## Getting and Building the Code -Remill defers the "implementation" of memory accesses and certain types of -control flows to the consumers of the produced bitcode. Deferral in this takes -the form of Remill [intrinsics](remill/Arch/Runtime/Intrinsics.h). +### Step 1: Install dependencies -For example, the `__remill_read_memory_8` intrinsic function represents the -action of reading 8 bits of memory. Via this and similar intrinsics, downstream -tools can distinguish between LLVM `load` and `store` instructions from accesses -to the modelled program's memory. Downstream tools can, of course, implement -memory intrinsics using LLVM's own memory access instructions. +#### On Linux -### Instruction Semantics +##### Install Dependencies -Instruction semantics are implemented using C++, and tested against their -native counterparts. Often, the high-level semantics of an instruction are -implemented using a C++ function template. This template is then instantiated -for each possible encoding of the modelled instruction. +```shell +sudo apt-get update +sudo apt-get upgrade -### Machine State +sudo apt-get install \ + git \ + cmake \ + libgoogle-glog-dev \ + libgtest-dev \ + libprotoc-dev libprotobuf-dev libprotobuf-dev protobuf-compiler \ + python2.7 python-pip \ + g++-multilib \ + unzip \ + software-properties-common -The register state of a machine is represented by a single `State` structure. -For example, the x86/amd64 state structure is defined in -[State.h](remill/Arch/X86/Runtime/State.h). State structures are carefully -designed to maintain the following properties. +sudo pip install --upgrade pip - - They should actively prevent certain compiler optimizations that obscure the - semantics of the translated machine code. For example, special - [tear fields](https://github.com/trailofbits/remill/blob/master/remill/Arch/X86/Runtime/State.h#L211) - are introduced so as to prevent load and store coalescing, and preserve the - semantics that writes to logical units of data remain as such. - - They should have a uniform size across all architecture revisions and - generations. This permits things such as: - - Mixing separately translated bitcode from two binaries, one with and one without AVX support. - - Mixing 32-bit ad 64-bit translated bitcode, or cross-compiling 32-bit and - 64-bit bitcode. - - They should accurately describe all register state maintained by the - emulated machine. - - It should be easy to convert to/from Remill's state structures and actual - machine-derived state. - -### Memory Model and the Remill Runtime - -Remill-produced bitcode has a memory model that includes memory barriers and atomic region. -It also explicitly distinguishes loads/stores to the modelled program's memory from -loads and stores to "runtime memory." - -Remill-produced bitcode can be thought of as an emulator for a program. -Through this lens, the memory used to store a `State` structure or any local -variables (`alloca`s in LLVM) needed to support the emulation must be treated -as distinct from the modelled program's memory itself. This separation enables -Remill to maintain [transparency](http://www.burningcutlery.com/derek/docs/transparency-VEE12.pdf) -with respect to memory accesses. - -# Setup - -## Linux-specific -```sh -sudo apt-get install libunwind8 libunwind8-dev -sudo pip install futures -sudo pip install python-magic +sudo pip install python-magic 'protobuf==2.4.1' ``` -## Generic -```sh -./scripts/bootstrap.sh +##### Upgrade CMake (Ubuntu 14.04) + +Users wishing to run Remill on Ubuntu 14.04 should upgrade their version of CMake. + +```shell +sudo add-apt-repository -y ppa:george-edison55/cmake-3.x +sudo apt-get update +sudo apt-get upgrade +sudo apt-get install cmake ``` -### Example +##### Install LLVM 3.9 -First, extract the control-flow graph information from your binary. +> **Note:** Installing LLVM on Ubuntu in such a way that it works for CMake can be tricky. We use LLVM 3.9. What I have found works is to start by removing all versions of all LLVM-related packages. Then, add in the official LLVM repositories (as shown below). Finally, install `llvm-3.9-dev`. If you also need older versions of LLVM-related tools, then re-install them after installing LLVM 3.9. -```sh -BIN=/path/to/binary -CFG=$(./scripts/ida_get_cfg.sh $BIN) +```shell +UBUNTU_RELEASE=`lsb_release -sc` + +wget -qO - http://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add - + +sudo add-apt-repository "deb http://apt.llvm.org/${UBUNTU_RELEASE}/ llvm-toolchain-${UBUNTU_RELEASE} main" +sudo add-apt-repository "deb http://apt.llvm.org/${UBUNTU_RELEASE}/ llvm-toolchain-${UBUNTU_RELEASE}-3.8 main" +sudo add-apt-repository "deb http://apt.llvm.org/${UBUNTU_RELEASE}/ llvm-toolchain-${UBUNTU_RELEASE}-3.9 main" + +sudo apt-get update +sudo apt-get install llvm-3.9-dev clang-3.9 ``` -This script will tell you where it puts the CFG. For example, it might output something like `/tmp/tmp.E3RWcczulG.cfg`. +#### On macOS (experimental) -Lets assume that `/path/to/binary` is a 32-bit ELF file. Now you can do the following: +Instructions for building on macOS are not yet available. -```sh -/path/to/remill/build/cfg_to_bc \ - --arch_in=x86 --arch_out=x86 --os_in=linux --os_out=linux \ - --bc_in=/path/to/remill/generated/sem_x86.bc --bc_out=$BIN.bc --cfg=$CFG +### Step 2: Clone and Enter the Repository + +#### Clone the repository +```shell +git clone git@github.com:trailofbits/remill.git ``` -For 64-bit x86 programs, specificy `--arch_in=amd64`. If you intend to run a 32-bit binary as a 64-bit program then specify `--arch_in=x86 --arch_out=amd64`. Similar switching can be done for the OS. The `--bc_in` flag also needs to be changes to point at the AMD64 semantics bitcode file. For example, `sem_amd64.bc`. If your lifted code uses AVX, then you can use `sem_amd64_avx.bc`. - -**Note:** This arch/OS switching only affects the ABI used by the bitcode, and how instructions are decoded. The translator itself has no other concept of arch/OS types. It is up to the next tool in the pipeline to implement the desired behavior. - -**Note:** Always use absolute paths when specifying files to Remill. I have no patience for handling paths in a 100% correct, generic way in C++. To that end, users of the tool should make it as easy as possible for Remill to do the right thing. - -#### Optimizing the bitcode - -There are a few ways to optimize the produced bitcode. The first is to tell remill to perform a data flow analysis and to try to kill things like dead registers. - -The data flow analyzer is enabled by specifying a maximum number of data flow analysis iterations to perform. By default, the maximum number is `0` (disabled). For a comprehensive analysis, specify a large number, e.g.: - -```sh -/path/to/remill/build/cfg_to_bc ... --max_dataflow_analysis_iterations=99999 ... +#### Enter the repository +```shell +cd remill ``` -In order to maintain correctness, the data-flow analysis is conservative. However, if all code is avaiable for analysis within the CFG file, then a more agressive analysis can be performed. This analysis will try to propagate data flow information across function returns, for instance. It is enabled with the `--aggressive_dataflow_analysis` flag. +### Step 3: Install Intel XED -Once bitcode has been produced, it can be optimized using the remill-specific LLVM optimization plugin. The following will produce optimized bitcode in a file named by `$OPT`. +#### On Linux and macOS -```sh -OPT=$(./scripts/optimize_bitcode.sh $BIN.bc) +This script will unpack and install Intel XED. It will require `sudo`er permissions. The XED library will be installed into `/usr/local/lib`, and the headers will be installed into `/usr/local/include/intel`. + +```shell +./scripts/unix/install_xed.sh ``` -### Miscellaneous +### Step 4: Compile Protocol Buffers -To recompile the code, run `./scripts/build.py`. Ideally, you should install the `concurrent.futures` package to make the build faster, though it is not required. If you want to see what the build script is doing to compile the code, then run the following: +Remill represents disassembled binaries using a [protocol buffer format](docs/CFG_FORMAT.md). This step compiles that format into files that can be used by `remill-lift`. -``` -./scripts/build.py --debug --num_workers 1 --stop_on_error +#### On Linux and macOS + +```shell +./scripts/unix/compile_protobufs.sh ``` -To recompile the semantics (if you add an instruction) then run `./scripts/compile_semantics.sh`. If you want to test your new semantics, then also recompile the code using the above command. +### Step 5: Run a Basic Build -If you make any changes to the register machine `State` structure, then before recompiling, run the script `./scripts/print_x86_save_state_asm.sh`. This produces an assembly source code file used by the unit tests for marshaling the machine state to/from the `State` structure. +#### Create a build location -## Third-Party Dependencies +```shell +mkdir build +cd build +``` -### Intel XED +#### Compile the code -Remill depends on and redistributes [Intel XED](https://software.intel.com/en-us/articles/xed-x86-encoder-decoder-software-library), the highest-quality x86 instruction -encoder and decoder. XED is licensed under the What If pre-release license. A copy of this license can be found [here](blob/xed/LICENSE.md). +Now the code must be compiled. It is best to manually specify the paths to the LLVM 3.9 and Clang 3.9 binaries. By default, Remill installs its files into `/usr/local`. An alternative installation directory prefix can be specified with `-DCMAKE_INSTALL_PREFIX=/path/to/prefix`. -### LLVM +```shell +cmake \ +-DCMAKE_C_COMPILER=/path/to/clang-3.9 \ +-DCMAKE_CXX_COMPILER=/path/to/clang++-3.9 \ +-DCMAKE_LLVM_LINK=/path/to/llvm-link-3.9 \ +.. -Remill depends on the [LLVM Compiler Infrastructure](http://llvm.org). A copy of this license can be found [here](http://llvm.org/releases/3.8.0/LICENSE.TXT). +make semantics +make all +sudo make install +``` -### IDA Pro + > **Note 1:** The `semantics` target must be built before `install`ing. -Remill depends on [IDA Pro](https://www.hex-rays.com/products/ida) to -accurately disassemble program binaries. + > **Note 2:** If you are implementing new instruction semantics, then the `semantics` target can be rebuilt and should take effect, even without re`install`ing. -### Binary Ninja +## Building and Running the Test Suite -An alternative to IDA Pro is [Binary Ninja](https://binary.ninja). Remill can use Binary Ninja to accurately disassemble program binaries. +### Build Google Test + +#### On Linux + +This script will build and install the Google Test framework. It will request administrator permissions. + +```shell +./scripts/linux/install_gtest.sh +``` + +### Generate and Run the Test Cases + +```shell +./scripts/x86/generate_tests.sh +./scripts/x86/run_tests.sh +``` + +## Try it Out + +**TODO(pag):** Make `remill-lift`. diff --git a/cmake/FindGFLAGS.cmake b/cmake/FindGFLAGS.cmake new file mode 100644 index 00000000..d2f34d71 --- /dev/null +++ b/cmake/FindGFLAGS.cmake @@ -0,0 +1,52 @@ +# https://github.com/yahoo/caffe/blob/master/cmake/Modules/FindGFlags.cmake + +# - Try to find GFLAGS +# +# The following variables are optionally searched for defaults +# GFLAGS_ROOT_DIR: Base directory where all GFLAGS components are found +# +# The following are set after configuration is done: +# GFLAGS_FOUND +# GFLAGS_INCLUDE_DIRS +# GFLAGS_LIBRARIES +# GFLAGS_LIBRARYRARY_DIRS + +include(FindPackageHandleStandardArgs) + +set(GFLAGS_ROOT_DIR "" CACHE PATH "Folder contains Gflags") + +# We are testing only a couple of files in the include directories +if(WIN32) + find_path(GFLAGS_INCLUDE_DIR gflags/gflags.h + PATHS ${GFLAGS_ROOT_DIR}/src/windows) +else() + find_path(GFLAGS_INCLUDE_DIR gflags/gflags.h + PATHS ${GFLAGS_ROOT_DIR}) +endif() + +if(MSVC) + find_library(GFLAGS_LIBRARY_RELEASE + NAMES libgflags + PATHS ${GFLAGS_ROOT_DIR} + PATH_SUFFIXES Release) + + find_library(GFLAGS_LIBRARY_DEBUG + NAMES libgflags-debug + PATHS ${GFLAGS_ROOT_DIR} + PATH_SUFFIXES Debug) + + set(GFLAGS_LIBRARY optimized ${GFLAGS_LIBRARY_RELEASE} debug ${GFLAGS_LIBRARY_DEBUG}) +else() + find_library(GFLAGS_LIBRARY gflags) +endif() + +find_package_handle_standard_args(GFlags DEFAULT_MSG GFLAGS_INCLUDE_DIR GFLAGS_LIBRARY) + + +if(GFLAGS_FOUND) + set(GFLAGS_INCLUDE_DIRS ${GFLAGS_INCLUDE_DIR}) + set(GFLAGS_LIBRARIES ${GFLAGS_LIBRARY}) + message(STATUS "Found gflags (include: ${GFLAGS_INCLUDE_DIR}, library: ${GFLAGS_LIBRARY})") + mark_as_advanced(GFLAGS_LIBRARY_DEBUG GFLAGS_LIBRARY_RELEASE + GFLAGS_LIBRARY GFLAGS_INCLUDE_DIR GFLAGS_ROOT_DIR) +endif() \ No newline at end of file diff --git a/cmake/FindGLOG.cmake b/cmake/FindGLOG.cmake new file mode 100644 index 00000000..82549b6a --- /dev/null +++ b/cmake/FindGLOG.cmake @@ -0,0 +1,81 @@ +# https://github.com/yahoo/caffe/blob/master/cmake/Modules/FindGlog.cmake + +# - Try to find the Google Glog library +# +# This module defines the following variables +# +# GLOG_FOUND - Was Glog found +# GLOG_INCLUDE_DIRS - the Glog include directories +# GLOG_LIBRARIES - Link to this +# +# This module accepts the following variables +# +# GLOG_ROOT - Can be set to Glog install path or Windows build path +# + +if (NOT DEFINED GLOG_ROOT) + set (GLOG_ROOT /usr /usr/local) +endif (NOT DEFINED GLOG_ROOT) + +if(MSVC) + set(LIB_PATHS ${GLOG_ROOT} ${GLOG_ROOT}/Release) +else(MSVC) + set (LIB_PATHS ${GLOG_ROOT} ${GLOG_ROOT}/lib) +endif(MSVC) + +macro(_FIND_GLOG_LIBRARIES _var) + find_library(${_var} + NAMES ${ARGN} + PATHS ${LIB_PATHS} /opt/local/lib + /usr/lib/x86_64-linux-gnu + /usr/local/lib + /usr/lib + PATH_SUFFIXES lib + ) + mark_as_advanced(${_var}) +endmacro() + +macro(_GLOG_APPEND_LIBRARIES _list _release) +set(_debug ${_release}_DEBUG) +if(${_debug}) + set(${_list} ${${_list}} optimized ${${_release}} debug ${${_debug}}) +else() + set(${_list} ${${_list}} ${${_release}}) +endif() +endmacro() + + +# Linux/OS X builds +find_path(GLOG_INCLUDE_DIR NAMES raw_logging.h + PATHS ${GLOG_ROOT}/include/glog + /usr/include/glog + /opt/local/include/glog # default location in Macports +) + +# Find the libraries +# Linux/OS X builds +if(UNIX) + _FIND_GLOG_LIBRARIES(GLOG_LIBRARIES libglog.so) +endif(UNIX) +if(APPLE) + _FIND_GLOG_LIBRARIES(GLOG_LIBRARIES libglog.dylib) +endif(APPLE) + +if(GLOG_FOUND) + message(STATUS "glog library found at ${GLOG_LIBRARIES}") +endif() + +# handle the QUIETLY and REQUIRED arguments and set GLOG_FOUND to TRUE if +# all listed variables are TRUE +include("${CMAKE_ROOT}/Modules/FindPackageHandleStandardArgs.cmake") +FIND_PACKAGE_HANDLE_STANDARD_ARGS(Glog DEFAULT_MSG + GLOG_LIBRARIES) + +# Linux/OS X builds +set(GLOG_INCLUDE_DIRS ${GLOG_INCLUDE_DIR}) +string(REGEX REPLACE "/libglog.so" "" GLOG_LIBRARIES_DIR ${GLOG_LIBRARIES}) + +if(GLOG_FOUND) + message(STATUS "Found glog (include: ${GLOG_INCLUDE_DIRS}, library: ${GLOG_LIBRARIES})") + # _GLOG_APPEND_LIBRARIES(GLOG GLOG_LIBRARIES) +endif() \ No newline at end of file diff --git a/cmake/FindXED.cmake b/cmake/FindXED.cmake new file mode 100644 index 00000000..eaf9ed89 --- /dev/null +++ b/cmake/FindXED.cmake @@ -0,0 +1,75 @@ +# - Try to find the Intel XED library +# +# This module defines the following variables +# +# XED_FOUND - Was XED found +# XED_INCLUDE_DIRS - the XED include directories +# XED_LIBRARIES - Link to this +# +# This module accepts the following variables +# +# XED_ROOT - Can be set to XED install path or Windows build path +# + +if (NOT DEFINED XED_ROOT) + set (XED_ROOT /usr /usr/local) +endif (NOT DEFINED XED_ROOT) + +set (LIB_PATHS ${XED_ROOT} ${XED_ROOT}/lib) + +macro(_FIND_XED_LIBRARIES _var) + find_library(${_var} + NAMES ${ARGN} + PATHS ${LIB_PATHS} /opt/local/lib + /usr/lib/x86_64-linux-gnu + /usr/local/lib + /usr/lib + PATH_SUFFIXES lib + ) + mark_as_advanced(${_var}) +endmacro() + +macro(_XED_APPEND_LIBRARIES _list _release) +set(_debug ${_release}_DEBUG) +if(${_debug}) + set(${_list} ${${_list}} optimized ${${_release}} debug ${${_debug}}) +else() + set(${_list} ${${_list}} ${${_release}}) +endif() +endmacro() + + +# Linux/OS X builds +find_path(XED_INCLUDE_DIR NAMES intel/xed-interface.h + PATHS ${XED_ROOT}/include + /usr/include + /opt/local/include +) + +# Find the libraries +# Linux/OS X builds +if(UNIX) + _FIND_XED_LIBRARIES(XED_LIBRARIES libxed.so) +endif(UNIX) +if(APPLE) + _FIND_XED_LIBRARIES(XED_LIBRARIES libxed.a) +endif(APPLE) + +if(XED_FOUND) + message(STATUS "xed library found at ${XED_LIBRARIES}") +endif() + +# handle the QUIETLY and REQUIRED arguments and set XED_FOUND to TRUE if +# all listed variables are TRUE +include("${CMAKE_ROOT}/Modules/FindPackageHandleStandardArgs.cmake") +FIND_PACKAGE_HANDLE_STANDARD_ARGS(XED DEFAULT_MSG + XED_LIBRARIES) + +# Linux/OS X builds +set(XED_INCLUDE_DIRS ${XED_INCLUDE_DIR}) +string(REGEX REPLACE "/libxed.so" "" XED_LIBRARIES_DIR ${XED_LIBRARIES}) + +if(XED_FOUND) + message(STATUS "Found xed (include: ${XED_INCLUDE_DIRS}, library: ${XED_LIBRARIES})") + # _XED_APPEND_LIBRARIES(XED XED_LIBRARIES) +endif() \ No newline at end of file diff --git a/docs/DESIGN.md b/docs/DESIGN.md new file mode 100644 index 00000000..5003d3e7 --- /dev/null +++ b/docs/DESIGN.md @@ -0,0 +1,61 @@ + +# Design and architecture of Remill + +Remill translates machine code, and *only* machine code, into LLVM bitcode. Remill's translation approach is inspired by [dynamic](https://software.intel.com/en-us/articles/pin-a-dynamic-binary-instrumentation-tool) +[binary](https://github.com/DynamoRIO/dynamorio) [translators](https://github.com/Granary/granary2). Remill translates one [basic block](https://en.wikipedia.org/wiki/Basic_block) of machine code into LLVM bitcode at a time. The translation process defers many decisions to downstream consumers on how the translated bitcode should be interpreted. + +## Intrinsics + +Remill defers the "implementation" of memory accesses and certain types of +control flows to the consumers of the produced bitcode. Deferral in this takes +the form of Remill [intrinsics](remill/Arch/Runtime/Intrinsics.h). + +For example, the `__remill_read_memory_8` intrinsic function represents the +action of reading 8 bits of memory. Via this and similar intrinsics, downstream +tools can distinguish between LLVM `load` and `store` instructions from accesses +to the modelled program's memory. Downstream tools can, of course, implement +memory intrinsics using LLVM's own memory access instructions. + +## Instruction semantics + +Instruction semantics are implemented using C++, and tested against their +native counterparts. Often, the high-level semantics of an instruction are +implemented using a C++ function template. This template is then instantiated +for each possible encoding of the modelled instruction. + +## Machine state + +The register state of a machine is represented by a single `State` structure. +For example, the x86/amd64 state structure is defined in +[State.h](remill/Arch/X86/Runtime/State.h). State structures are carefully +designed to maintain the following properties. + + - They should actively prevent certain compiler optimizations that obscure the + semantics of the translated machine code. For example, special + [tear fields](https://github.com/trailofbits/remill/blob/master/remill/Arch/X86/Runtime/State.h#L211) + are introduced so as to prevent load and store coalescing, and preserve the + semantics that writes to logical units of data remain as such. + - They should have a uniform size across all architecture revisions and + generations. This permits things such as: + - Mixing separately translated bitcode from two binaries, one with and one without AVX support. + - Mixing 32-bit ad 64-bit translated bitcode, or cross-compiling 32-bit and + 64-bit bitcode. + - They should accurately describe all register state maintained by the + emulated machine. + - It should be easy to convert to/from Remill's state structures and actual + machine-derived state. + +## Memory model + +Remill-produced bitcode has a memory model that includes memory barriers and atomic region. +It also explicitly distinguishes loads/stores to the modelled program's memory from +loads and stores to "runtime memory." + +## Runtime + +Remill-produced bitcode can be thought of as an emulator for a program. +Through this lens, the memory used to store a `State` structure or any local +variables (`alloca`s in LLVM) needed to support the emulation must be treated +as distinct from the modelled program's memory itself. This separation enables +Remill to maintain [transparency](http://www.burningcutlery.com/derek/docs/transparency-VEE12.pdf) +with respect to memory accesses. \ No newline at end of file diff --git a/generated/Arch/X86/SaveState.S b/generated/Arch/X86/SaveState.S new file mode 100644 index 00000000..dbce8341 --- /dev/null +++ b/generated/Arch/X86/SaveState.S @@ -0,0 +1,239 @@ +/* Auto-generated file! Don't modify! */ + +#if 64 == ADDRESS_SIZE_BITS +fxsave64 [RIP + STATE_PTR + 0] +#else +fxsave [RIP + STATE_PTR + 0] +#endif +fstp QWORD PTR [RIP + STATE_PTR + 2888] +fstp QWORD PTR [RIP + STATE_PTR + 2904] +fstp QWORD PTR [RIP + STATE_PTR + 2920] +fstp QWORD PTR [RIP + STATE_PTR + 2936] +fstp QWORD PTR [RIP + STATE_PTR + 2952] +fstp QWORD PTR [RIP + STATE_PTR + 2968] +fstp QWORD PTR [RIP + STATE_PTR + 2984] +fstp QWORD PTR [RIP + STATE_PTR + 3000] +movq [RIP + STATE_PTR + 3016], MM0 +movq [RIP + STATE_PTR + 3032], MM1 +movq [RIP + STATE_PTR + 3048], MM2 +movq [RIP + STATE_PTR + 3064], MM3 +movq [RIP + STATE_PTR + 3080], MM4 +movq [RIP + STATE_PTR + 3096], MM5 +movq [RIP + STATE_PTR + 3112], MM6 +movq [RIP + STATE_PTR + 3128], MM7 +#if 64 == ADDRESS_SIZE_BITS +fxrstor64 [RIP + STATE_PTR + 0] +#else +fxrstor [RIP + STATE_PTR + 0] +#endif +lea RSP, [RSP - 8] +pop QWORD PTR [RIP + SYMBOL(gStackSaveSlot)] +#ifdef AFTER_TEST_CASE +pushfq +#else +push QWORD PTR [RIP + STATE_PTR + 2576] +#endif +bt QWORD PTR [RSP], 0 +adc BYTE PTR [RIP + STATE_PTR + 2561], 0 +bt QWORD PTR [RSP], 2 +adc BYTE PTR [RIP + STATE_PTR + 2563], 0 +bt QWORD PTR [RSP], 4 +adc BYTE PTR [RIP + STATE_PTR + 2565], 0 +bt QWORD PTR [RSP], 6 +adc BYTE PTR [RIP + STATE_PTR + 2567], 0 +bt QWORD PTR [RSP], 7 +adc BYTE PTR [RIP + STATE_PTR + 2569], 0 +bt QWORD PTR [RSP], 10 +adc BYTE PTR [RIP + STATE_PTR + 2571], 0 +bt QWORD PTR [RSP], 11 +adc BYTE PTR [RIP + STATE_PTR + 2573], 0 +#ifdef AFTER_TEST_CASE +pop QWORD PTR [RIP + STATE_PTR + 2576] +#else +popfq +#endif +push QWORD PTR [RIP + SYMBOL(gStackSaveSlot)] +lea RSP, [RSP + 8] +mov [RIP + STATE_PTR + 2617], AH +mov [RIP + STATE_PTR + 2633], BH +mov [RIP + STATE_PTR + 2649], CH +mov [RIP + STATE_PTR + 2665], DH +mov [RIP + STATE_PTR + 2616], AL +mov [RIP + STATE_PTR + 2632], BL +mov [RIP + STATE_PTR + 2648], CL +mov [RIP + STATE_PTR + 2664], DL +#if 64 == ADDRESS_SIZE_BITS +mov [RIP + STATE_PTR + 2680], SIL +mov [RIP + STATE_PTR + 2696], DIL +mov [RIP + STATE_PTR + 2712], SPL +mov [RIP + STATE_PTR + 2728], BPL +mov [RIP + STATE_PTR + 2744], R8B +mov [RIP + STATE_PTR + 2760], R9B +mov [RIP + STATE_PTR + 2776], R10B +mov [RIP + STATE_PTR + 2792], R11B +mov [RIP + STATE_PTR + 2808], R12B +mov [RIP + STATE_PTR + 2824], R13B +mov [RIP + STATE_PTR + 2840], R14B +mov [RIP + STATE_PTR + 2856], R15B +#endif /* 64 == ADDRESS_SIZE_BITS */ +mov [RIP + STATE_PTR + 2616], AX +mov [RIP + STATE_PTR + 2632], BX +mov [RIP + STATE_PTR + 2648], CX +mov [RIP + STATE_PTR + 2664], DX +mov [RIP + STATE_PTR + 2680], SI +mov [RIP + STATE_PTR + 2696], DI +mov [RIP + STATE_PTR + 2712], SP +mov [RIP + STATE_PTR + 2728], BP +#if 64 == ADDRESS_SIZE_BITS +mov [RIP + STATE_PTR + 2744], R8W +mov [RIP + STATE_PTR + 2760], R9W +mov [RIP + STATE_PTR + 2776], R10W +mov [RIP + STATE_PTR + 2792], R11W +mov [RIP + STATE_PTR + 2808], R12W +mov [RIP + STATE_PTR + 2824], R13W +mov [RIP + STATE_PTR + 2840], R14W +mov [RIP + STATE_PTR + 2856], R15W +#endif /* 64 == ADDRESS_SIZE_BITS */ +mov WORD PTR [RIP + STATE_PTR + 2872], 0 +mov [RIP + STATE_PTR + 2616], EAX +mov [RIP + STATE_PTR + 2632], EBX +mov [RIP + STATE_PTR + 2648], ECX +mov [RIP + STATE_PTR + 2664], EDX +mov [RIP + STATE_PTR + 2680], ESI +mov [RIP + STATE_PTR + 2696], EDI +mov [RIP + STATE_PTR + 2712], ESP +mov [RIP + STATE_PTR + 2728], EBP +#if 64 == ADDRESS_SIZE_BITS +mov [RIP + STATE_PTR + 2744], R8D +mov [RIP + STATE_PTR + 2760], R9D +mov [RIP + STATE_PTR + 2776], R10D +mov [RIP + STATE_PTR + 2792], R11D +mov [RIP + STATE_PTR + 2808], R12D +mov [RIP + STATE_PTR + 2824], R13D +mov [RIP + STATE_PTR + 2840], R14D +mov [RIP + STATE_PTR + 2856], R15D +mov [RIP + STATE_PTR + 2616], RAX +mov [RIP + STATE_PTR + 2632], RBX +mov [RIP + STATE_PTR + 2648], RCX +mov [RIP + STATE_PTR + 2664], RDX +mov [RIP + STATE_PTR + 2680], RSI +mov [RIP + STATE_PTR + 2696], RDI +mov [RIP + STATE_PTR + 2712], RSP +mov [RIP + STATE_PTR + 2728], RBP +mov [RIP + STATE_PTR + 2744], R8 +mov [RIP + STATE_PTR + 2760], R9 +mov [RIP + STATE_PTR + 2776], R10 +mov [RIP + STATE_PTR + 2792], R11 +mov [RIP + STATE_PTR + 2808], R12 +mov [RIP + STATE_PTR + 2824], R13 +mov [RIP + STATE_PTR + 2840], R14 +mov [RIP + STATE_PTR + 2856], R15 +#endif /* 64 == ADDRESS_SIZE_BITS */ +#if HAS_FEATURE_AVX +#if HAS_FEATURE_AVX512 +vmovdqu [RIP + STATE_PTR + 512], ZMM0 +vmovdqu [RIP + STATE_PTR + 576], ZMM1 +vmovdqu [RIP + STATE_PTR + 640], ZMM2 +vmovdqu [RIP + STATE_PTR + 704], ZMM3 +vmovdqu [RIP + STATE_PTR + 768], ZMM4 +vmovdqu [RIP + STATE_PTR + 832], ZMM5 +vmovdqu [RIP + STATE_PTR + 896], ZMM6 +vmovdqu [RIP + STATE_PTR + 960], ZMM7 +vmovdqu [RIP + STATE_PTR + 1024], ZMM8 +vmovdqu [RIP + STATE_PTR + 1088], ZMM9 +vmovdqu [RIP + STATE_PTR + 1152], ZMM10 +vmovdqu [RIP + STATE_PTR + 1216], ZMM11 +vmovdqu [RIP + STATE_PTR + 1280], ZMM12 +vmovdqu [RIP + STATE_PTR + 1344], ZMM13 +vmovdqu [RIP + STATE_PTR + 1408], ZMM14 +vmovdqu [RIP + STATE_PTR + 1472], ZMM15 +vmovdqu [RIP + STATE_PTR + 1536], ZMM16 +vmovdqu [RIP + STATE_PTR + 1600], ZMM17 +vmovdqu [RIP + STATE_PTR + 1664], ZMM18 +vmovdqu [RIP + STATE_PTR + 1728], ZMM19 +vmovdqu [RIP + STATE_PTR + 1792], ZMM20 +vmovdqu [RIP + STATE_PTR + 1856], ZMM21 +vmovdqu [RIP + STATE_PTR + 1920], ZMM22 +vmovdqu [RIP + STATE_PTR + 1984], ZMM23 +vmovdqu [RIP + STATE_PTR + 2048], ZMM24 +vmovdqu [RIP + STATE_PTR + 2112], ZMM25 +vmovdqu [RIP + STATE_PTR + 2176], ZMM26 +vmovdqu [RIP + STATE_PTR + 2240], ZMM27 +vmovdqu [RIP + STATE_PTR + 2304], ZMM28 +vmovdqu [RIP + STATE_PTR + 2368], ZMM29 +vmovdqu [RIP + STATE_PTR + 2432], ZMM30 +vmovdqu [RIP + STATE_PTR + 2496], ZMM31 +#endif /* HAS_FEATURE_AVX512 */ +vmovdqu [RIP + STATE_PTR + 512], YMM0 +vmovdqu [RIP + STATE_PTR + 576], YMM1 +vmovdqu [RIP + STATE_PTR + 640], YMM2 +vmovdqu [RIP + STATE_PTR + 704], YMM3 +vmovdqu [RIP + STATE_PTR + 768], YMM4 +vmovdqu [RIP + STATE_PTR + 832], YMM5 +vmovdqu [RIP + STATE_PTR + 896], YMM6 +vmovdqu [RIP + STATE_PTR + 960], YMM7 +#if HAS_FEATURE_AVX || 64 == ADDRESS_SIZE_BITS +vmovdqu [RIP + STATE_PTR + 1024], YMM8 +vmovdqu [RIP + STATE_PTR + 1088], YMM9 +vmovdqu [RIP + STATE_PTR + 1152], YMM10 +vmovdqu [RIP + STATE_PTR + 1216], YMM11 +vmovdqu [RIP + STATE_PTR + 1280], YMM12 +vmovdqu [RIP + STATE_PTR + 1344], YMM13 +vmovdqu [RIP + STATE_PTR + 1408], YMM14 +vmovdqu [RIP + STATE_PTR + 1472], YMM15 +#endif /* HAS_FEATURE_AVX || 64 == ADDRESS_SIZE_BITS */ +#if HAS_FEATURE_AVX512 +vmovdqu [RIP + STATE_PTR + 1536], YMM16 +vmovdqu [RIP + STATE_PTR + 1600], YMM17 +vmovdqu [RIP + STATE_PTR + 1664], YMM18 +vmovdqu [RIP + STATE_PTR + 1728], YMM19 +vmovdqu [RIP + STATE_PTR + 1792], YMM20 +vmovdqu [RIP + STATE_PTR + 1856], YMM21 +vmovdqu [RIP + STATE_PTR + 1920], YMM22 +vmovdqu [RIP + STATE_PTR + 1984], YMM23 +vmovdqu [RIP + STATE_PTR + 2048], YMM24 +vmovdqu [RIP + STATE_PTR + 2112], YMM25 +vmovdqu [RIP + STATE_PTR + 2176], YMM26 +vmovdqu [RIP + STATE_PTR + 2240], YMM27 +vmovdqu [RIP + STATE_PTR + 2304], YMM28 +vmovdqu [RIP + STATE_PTR + 2368], YMM29 +vmovdqu [RIP + STATE_PTR + 2432], YMM30 +vmovdqu [RIP + STATE_PTR + 2496], YMM31 +#endif /* HAS_FEATURE_AVX512 */ +#endif /* HAS_FEATURE_AVX */ +movdqu [RIP + STATE_PTR + 512], XMM0 +movdqu [RIP + STATE_PTR + 576], XMM1 +movdqu [RIP + STATE_PTR + 640], XMM2 +movdqu [RIP + STATE_PTR + 704], XMM3 +movdqu [RIP + STATE_PTR + 768], XMM4 +movdqu [RIP + STATE_PTR + 832], XMM5 +movdqu [RIP + STATE_PTR + 896], XMM6 +movdqu [RIP + STATE_PTR + 960], XMM7 +#if HAS_FEATURE_AVX || 64 == ADDRESS_SIZE_BITS +movdqu [RIP + STATE_PTR + 1024], XMM8 +movdqu [RIP + STATE_PTR + 1088], XMM9 +movdqu [RIP + STATE_PTR + 1152], XMM10 +movdqu [RIP + STATE_PTR + 1216], XMM11 +movdqu [RIP + STATE_PTR + 1280], XMM12 +movdqu [RIP + STATE_PTR + 1344], XMM13 +movdqu [RIP + STATE_PTR + 1408], XMM14 +movdqu [RIP + STATE_PTR + 1472], XMM15 +#endif /* HAS_FEATURE_AVX || 64 == ADDRESS_SIZE_BITS */ +#if HAS_FEATURE_AVX512 +movdqu [RIP + STATE_PTR + 1536], XMM16 +movdqu [RIP + STATE_PTR + 1600], XMM17 +movdqu [RIP + STATE_PTR + 1664], XMM18 +movdqu [RIP + STATE_PTR + 1728], XMM19 +movdqu [RIP + STATE_PTR + 1792], XMM20 +movdqu [RIP + STATE_PTR + 1856], XMM21 +movdqu [RIP + STATE_PTR + 1920], XMM22 +movdqu [RIP + STATE_PTR + 1984], XMM23 +movdqu [RIP + STATE_PTR + 2048], XMM24 +movdqu [RIP + STATE_PTR + 2112], XMM25 +movdqu [RIP + STATE_PTR + 2176], XMM26 +movdqu [RIP + STATE_PTR + 2240], XMM27 +movdqu [RIP + STATE_PTR + 2304], XMM28 +movdqu [RIP + STATE_PTR + 2368], XMM29 +movdqu [RIP + STATE_PTR + 2432], XMM30 +movdqu [RIP + STATE_PTR + 2496], XMM31 +#endif // HAS_FEATURE_AVX512 diff --git a/generated/CFG/__init__.py b/generated/CFG/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/generated/__init__.py b/generated/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/remill/Arch/Arch.cpp b/remill/Arch/Arch.cpp index b131bd39..a385eda7 100644 --- a/remill/Arch/Arch.cpp +++ b/remill/Arch/Arch.cpp @@ -23,27 +23,50 @@ const Arch *Arch::Create(OSName os_name_, ArchName arch_name_) { DLOG(INFO) << "Using architecture: X86"; return CreateX86(os_name_, arch_name_, 32); + case kArchX86_AVX: + DLOG(INFO) << "Using architecture: X86, feature set: AVX"; + return CreateX86(os_name_, arch_name_, 32); + + case kArchX86_AVX512: + DLOG(INFO) << "Using architecture: X86, feature set: AVX512"; + return CreateX86(os_name_, arch_name_, 32); + case kArchAMD64: DLOG(INFO) << "Using architecture: AMD64"; return CreateX86(os_name_, arch_name_, 64); + + case kArchAMD64_AVX: + DLOG(INFO) << "Using architecture: AMD64, feature set: AVX"; + return CreateX86(os_name_, arch_name_, 64); + + case kArchAMD64_AVX512: + DLOG(INFO) << "Using architecture: AMD64, feature set: AVX512"; + return CreateX86(os_name_, arch_name_, 64); } return nullptr; } + ArchName Arch::GetName(const std::string &arch_name) { - if (arch_name == "x86" || - arch_name == "x86_32" || - arch_name == "x86-32") { + + if (arch_name == "x86") { return kArchX86; - } else if (arch_name == "amd64" || - arch_name == "x86_64" || - arch_name == "x86-64" || - arch_name == "x64" || - arch_name == "intel64" || - arch_name == "emt64") { + } else if (arch_name == "x86_avx") { + return kArchX86_AVX; + + } else if (arch_name == "x86_avx512") { + return kArchX86_AVX512; + + } else if (arch_name == "amd64") { return kArchAMD64; + } else if (arch_name == "amd64_avx") { + return kArchAMD64_AVX; + + } else if (arch_name == "amd64_avx512") { + return kArchAMD64_AVX512; + } else { LOG(ERROR) << "Unrecognized architecture: " << arch_name << "."; diff --git a/remill/Arch/Arch.h b/remill/Arch/Arch.h index abd22630..09d6f2c1 100644 --- a/remill/Arch/Arch.h +++ b/remill/Arch/Arch.h @@ -20,7 +20,11 @@ class Instruction; enum ArchName : unsigned { kArchInvalid, kArchX86, - kArchAMD64 + kArchX86_AVX, + kArchX86_AVX512, + kArchAMD64, + kArchAMD64_AVX, + kArchAMD64_AVX512 }; class Arch { diff --git a/remill/Arch/Instruction.cpp b/remill/Arch/Instruction.cpp index 2edb50cc..94f76e43 100644 --- a/remill/Arch/Instruction.cpp +++ b/remill/Arch/Instruction.cpp @@ -114,9 +114,13 @@ std::string Instruction::Debug(void) const { case kArchInvalid: break; case kArchAMD64: + case kArchAMD64_AVX: + case kArchAMD64_AVX512: ss << "AMD64_"; break; case kArchX86: + case kArchX86_AVX: + case kArchX86_AVX512: ss << "X86_"; break; } diff --git a/remill/Arch/X86/Arch.cpp b/remill/Arch/X86/Arch.cpp index 796c9006..2d7ceba9 100644 --- a/remill/Arch/X86/Arch.cpp +++ b/remill/Arch/X86/Arch.cpp @@ -606,24 +606,49 @@ void X86Arch::PrepareModule(llvm::Module *mod) const { std::string triple; switch (os_name) { case kOSInvalid: - LOG(FATAL) << "Cannot convert module for an unrecognized operating system."; + LOG(FATAL) << "Cannot convert module for an unrecognized OS."; break; case kOSLinux: - if (kArchAMD64 == arch_name) { - dl = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"; - triple = "x86_64-unknown-linux-gnu"; - } else { - dl = "e-m:e-p:32:32-f64:32:64-f80:32-n8:16:32-S128"; - triple = "i386-unknown-linux-gnu"; + switch (arch_name) { + case kArchInvalid: + LOG(FATAL) + << "Cannot convert module for an unrecognized architecture."; + break; + + case kArchAMD64: + case kArchAMD64_AVX: + case kArchAMD64_AVX512: + dl = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"; + triple = "x86_64-pc-linux-gnu"; + break; + case kArchX86: + case kArchX86_AVX: + case kArchX86_AVX512: + dl = "e-m:e-p:32:32-f64:32:64-f80:32-n8:16:32-S128"; + triple = "i386-pc-linux-gnu"; + break; } break; + case kOSmacOS: - if (kArchAMD64 == arch_name) { - dl = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"; - triple = "x86_64-apple-macosx10.10.0"; - } else { - dl = "e-m:o-p:32:32-f64:32:64-f80:128-n8:16:32-S128"; - triple = "i386-apple-macosx10.10.0"; + switch (arch_name) { + case kArchInvalid: + LOG(FATAL) + << "Cannot convert module for an unrecognized architecture."; + break; + + case kArchAMD64: + case kArchAMD64_AVX: + case kArchAMD64_AVX512: + dl = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"; + triple = "x86_64-apple-macosx10.10.0"; + break; + case kArchX86: + case kArchX86_AVX: + case kArchX86_AVX512: + dl = "e-m:o-p:32:32-f64:32:64-f80:128-n8:16:32-S128"; + triple = "i386-apple-macosx10.10.0"; + break; } break; } @@ -637,7 +662,7 @@ Instruction *X86Arch::DecodeInstruction( const std::string &instr_bytes) const { xed_decoded_inst_t xedd_; xed_decoded_inst_t *xedd = &xedd_; - auto mode = kArchX86 == arch_name ? &kXEDState32 : &kXEDState64; + auto mode = 32 == address_size ? &kXEDState32 : &kXEDState64; DecodeXED(xedd, mode, instr_bytes, address); diff --git a/remill/BC/Translator.cpp b/remill/BC/Translator.cpp index 184175db..d7d01288 100644 --- a/remill/BC/Translator.cpp +++ b/remill/BC/Translator.cpp @@ -14,8 +14,10 @@ #include #include #include +#include #include #include +#include #include #include #include @@ -48,6 +50,41 @@ class ReturnInst; } // namespace namespace remill { +namespace { + +// These variables must always be defined within `__remill_basic_block`. +static bool BlockHasSpecialVars(llvm::Function *basic_block) { + return FindVarInFunction(basic_block, "STATE", true) && + FindVarInFunction(basic_block, "MEMORY", true) && + FindVarInFunction(basic_block, "PC", true) && + FindVarInFunction(basic_block, "NEXT_PC", true) && + FindVarInFunction(basic_block, "BRANCH_TAKEN", true); +} + +// Clang isn't guaranteed to play nice and name the LLVM values within the +// `__remill_basic_block` instrinsic with the same names as we find in the +// C++ definition of that function. However, we compile that function with +// debug information, and so we will try to recover the variables names for +// later lookup. +static void FixupBasicBlockVariables(llvm::Function *basic_block) { + if (BlockHasSpecialVars(basic_block)) { + return; + } + + for (auto &block : *basic_block) { + for (auto &inst : block) { + if (auto decl_inst = llvm::dyn_cast(&inst)) { + auto addr = decl_inst->getAddress(); + addr->setName(decl_inst->getVariable()->getName()); + } + } + } + + CHECK(BlockHasSpecialVars(basic_block)) + << "Unable to locate required variables in `__remill_basic_block`."; +} + +} // namespace Translator::Translator(const Arch *arch_, llvm::Module *module_) : arch(arch_), @@ -63,6 +100,7 @@ Translator::Translator(const Arch *arch_, llvm::Module *module_) CHECK(nullptr != basic_block) << "Unable to find __remill_basic_block."; + FixupBasicBlockVariables(basic_block); EnableDeferredInlining(); } @@ -143,7 +181,7 @@ static llvm::Function *CreateExternalFunction(llvm::Module *module, // Clone the block method template `TF` into a specific method `BF` that // will contain lifted code. static void AddBlockInitializationCode(llvm::Function *block_func, - const llvm::Function *template_func) { + llvm::Function *template_func) { llvm::ValueToValueMapTy var_map; var_map[template_func] = block_func; @@ -504,6 +542,17 @@ llvm::Function *Translator::GetOrCreateBlock(uint64_t addr) { return block_func; } +// Create a function for a single block. This block appears as the target +// of some control-flow instruction. Make sure that it has a default detach- +// based implementation. +llvm::Function *Translator::GetOrCreateTargetBlock(uint64_t address) { + auto block_func = GetOrCreateBlock(address); + if (block_func->isDeclaration()) { + AddTerminatingTailCall(block_func, intrinsics->detach); + } + return block_func; +} + // Create functions for every block in the CFG. We do this before lifting so // that we can easily reference those blocks. void Translator::CreateBlocks(const cfg::Module *cfg_module) { @@ -533,7 +582,7 @@ void Translator::CreateBlocks(const cfg::Module *cfg_module) { } for (const auto cfg_ref_block_addr : cfg_module->referenced_blocks()) { - auto block_func = GetOrCreateBlock(cfg_ref_block_addr); + auto block_func = GetOrCreateTargetBlock(cfg_ref_block_addr); if (block_func->isDeclaration()) { block_func->setLinkage(llvm::GlobalValue::ExternalLinkage); } @@ -581,9 +630,10 @@ void Translator::LiftBlocks(const cfg::Module *cfg_module) { func_pass_manager.doInitialization(); for (const auto &block : cfg_module->blocks()) { auto func = LiftBlock(&block); - if (func->isDeclaration()) { - continue; - } + + CHECK(!func->isDeclaration()) + << "Lifted block function " << func->getName().str() + << " should have an implementation."; // Make sure the translation is good before optimizing. std::string error; @@ -597,14 +647,49 @@ void Translator::LiftBlocks(const cfg::Module *cfg_module) { func_pass_manager.doFinalization(); } +namespace { + +// Returns `true` if a basic block function looks empty, i.e. it tail-calls to +// the `__remill_detach` intrinsics. This means that a previous call to +// `remill-lift` didn't have the implementation of this block available, so +// defaulted it to detaching instead. +static bool IsBlockEmpty(const llvm::Function *func, + const llvm::Function *detach_func) { + const auto &entry_block = func->front(); + if (auto term = entry_block.getTerminatingMustTailCall()) { + return term->getCalledFunction() == detach_func && + entry_block.size() == 2; + } else { + return false; + } +} + +// Delete a basic block. +static void DeleteBlock(llvm::BasicBlock *block) { + while (block->size()) { + block->getInstList().pop_back(); + } + block->eraseFromParent(); +} + +} // namespace + // Lift code contained within a single block. llvm::Function *Translator::LiftBlock(const cfg::Block *cfg_block) { auto block_func = GetOrCreateBlock(cfg_block->address()); if (!block_func->isDeclaration()) { - DLOG(WARNING) - << "Not going to lift duplicate block at " - << std::hex << cfg_block->address() << "."; - return block_func; + if (IsBlockEmpty(block_func, intrinsics->detach)) { + DeleteBlock(&block_func->front()); + CHECK(block_func->isDeclaration()) + << "Unable to delete blocks from previously detaching block function" + << block_func->getName().str() << "."; + + } else { + DLOG(WARNING) + << "Not going to lift duplicate block at " + << std::hex << cfg_block->address() << "."; + return block_func; + } } AddBlockInitializationCode(block_func, basic_block); @@ -712,7 +797,7 @@ void Translator::LiftTerminator(llvm::BasicBlock *block, case Instruction::kCategoryNoOp: AddTerminatingTailCall( block, - GetOrCreateBlock(arch_instr->next_pc)); + GetOrCreateTargetBlock(arch_instr->next_pc)); break; case Instruction::kCategoryError: @@ -724,7 +809,7 @@ void Translator::LiftTerminator(llvm::BasicBlock *block, case Instruction::kCategoryDirectJump: AddTerminatingTailCall( block, - GetOrCreateBlock(arch_instr->branch_taken_pc)); + GetOrCreateTargetBlock(arch_instr->branch_taken_pc)); break; case Instruction::kCategoryIndirectJump: @@ -736,7 +821,7 @@ void Translator::LiftTerminator(llvm::BasicBlock *block, case Instruction::kCategoryDirectFunctionCall: AddTerminatingTailCall( block, - GetOrCreateBlock(arch_instr->branch_taken_pc)); + GetOrCreateTargetBlock(arch_instr->branch_taken_pc)); if (FLAGS_lift_calls_as_calls) { auto term_call = block->getTerminatingMustTailCall(); @@ -744,7 +829,7 @@ void Translator::LiftTerminator(llvm::BasicBlock *block, block->getTerminator()->eraseFromParent(); AddTerminatingTailCall( block, - GetOrCreateBlock(arch_instr->next_pc)); + GetOrCreateTargetBlock(arch_instr->next_pc)); } break; @@ -760,7 +845,7 @@ void Translator::LiftTerminator(llvm::BasicBlock *block, AddTerminatingTailCall( block, - GetOrCreateBlock(arch_instr->next_pc)); + GetOrCreateTargetBlock(arch_instr->next_pc)); } break; @@ -773,8 +858,8 @@ void Translator::LiftTerminator(llvm::BasicBlock *block, case Instruction::kCategoryConditionalBranch: LiftConditionalBranch( block, - GetOrCreateBlock(arch_instr->branch_taken_pc), - GetOrCreateBlock(arch_instr->branch_not_taken_pc)); + GetOrCreateTargetBlock(arch_instr->branch_taken_pc), + GetOrCreateTargetBlock(arch_instr->branch_not_taken_pc)); break; case Instruction::kCategorySystemCall: diff --git a/remill/BC/Translator.h b/remill/BC/Translator.h index bff8903f..4a6f3dd1 100644 --- a/remill/BC/Translator.h +++ b/remill/BC/Translator.h @@ -69,6 +69,11 @@ class Translator { // Create a function for a single block. llvm::Function *GetOrCreateBlock(uint64_t address); + // Create a function for a single block. This block appears as the target + // of some control-flow instruction. Make sure that it has a default detach- + // based implementation. + llvm::Function *GetOrCreateTargetBlock(uint64_t address); + // Create functions for every imported function in the code. llvm::Function *CreateImportedFunction( const std::string &name, uintptr_t addr); diff --git a/remill/BC/Util.cpp b/remill/BC/Util.cpp index 7d608663..68a84969 100644 --- a/remill/BC/Util.cpp +++ b/remill/BC/Util.cpp @@ -175,9 +175,10 @@ llvm::GlobalVariable *FindGlobaVariable(const llvm::Module *module, } // Reads an LLVM module from a file. -llvm::Module *LoadModuleFromFile(std::string file_name) { +llvm::Module *LoadModuleFromFile(llvm::LLVMContext *context, + std::string file_name) { llvm::SMDiagnostic err; - auto mod_ptr = llvm::parseIRFile(file_name, err, llvm::getGlobalContext()); + auto mod_ptr = llvm::parseIRFile(file_name, err, *context); auto module = mod_ptr.get(); mod_ptr.release(); diff --git a/remill/BC/Util.h b/remill/BC/Util.h index 45e41d3d..31b6ebbf 100644 --- a/remill/BC/Util.h +++ b/remill/BC/Util.h @@ -11,6 +11,7 @@ class Function; class GlobalVariable; class Module; class Value; +class LLVMContext; } // namespace llvm namespace remill { @@ -56,7 +57,8 @@ llvm::GlobalVariable *FindGlobaVariable(const llvm::Module *M, std::string name); // Parses and loads a bitcode file into memory. -llvm::Module *LoadModuleFromFile(std::string file_name); +llvm::Module *LoadModuleFromFile(llvm::LLVMContext *context, + std::string file_name); // Store an LLVM module into a file. void StoreModuleToFile(llvm::Module *module, std::string file_name); diff --git a/remill/Lift.cpp b/remill/Lift.cpp new file mode 100644 index 00000000..16a5522f --- /dev/null +++ b/remill/Lift.cpp @@ -0,0 +1,206 @@ +/* Copyright 2015 Peter Goodman (peter@trailofbits.com), all rights reserved. */ + +#include +#include + +#include +#include +#include +#include + +#include +#include +#include + +#include "remill/BC/Translator.h" +#include "remill/BC/Util.h" +#include "remill/Arch/Arch.h" +#include "remill/CFG/CFG.h" + +#ifndef REMILL_OS +# if defined(__APPLE__) +# define REMILL_OS "mac" +# elif defined(__linux__) +# define REMILL_OS "linux" +# endif +#endif + +#ifndef BUILD_SEMANTICS_DIR +# error "Macro `BUILD_SEMANTICS_DIR` must be defined." +# define BUILD_SEMANTICS_DIR +#endif // BUILD_SEMANTICS_DIR + +#ifndef INSTALL_SEMANTICS_DIR +# error "Macro `INSTALL_SEMANTICS_DIR` must be defined." +# define INSTALL_SEMANTICS_DIR +#endif // INSTALL_SEMANTICS_DIR + + +// TODO(pag): Support separate source and target architectures? +DEFINE_string(arch_in, "", "Architecture of the code being translated. " + "Valid architectures: x86, amd64 (with or without " + "`_avx` or `_avx512` appended)."); + +DEFINE_string(arch_out, "", "Architecture of the target architecture on " + "which the translated code will run. " + "Valid architectures: x86, amd64 (with or without " + "`_avx` or `_avx512` appended)."); + +DEFINE_string(os_in, REMILL_OS, "Source OS. Valid OSes: linux, mac."); +DEFINE_string(os_out, REMILL_OS, "Target OS. Valid OSes: linux, mac."); + +DEFINE_string(cfg, "", "Path to the CFG file containing code to lift."); + +DEFINE_string(bc_in, "", "Input bitcode file into which code will " + "be lifted. This should either be a semantics file " + "associated with `--arch_in`, or it should be " + "a bitcode file produced by `remill-lift`. Chaining " + "bitcode files produces by `remill-lift` can be " + "used to iteratively link in libraries to lifted " + "code."); + +DEFINE_string(bc_out, "", "Output bitcode file name."); + +namespace { + +static const char *gSearchPaths[] = { + // Derived from the build. + BUILD_SEMANTICS_DIR "\0", + INSTALL_SEMANTICS_DIR "\0", + + // Linux. + "/usr/local/share/remill/semantics/", + "/usr/share/remill/semantics/", + + // Other? + "/opt/local/share/remill/semantics/", + "/opt/share/remill/semantics/", + "/opt/remill/semantics/", + + // FreeBSD. + "/usr/share/compat/linux/remill/semantics", + "/usr/local/share/compat/linux/remill/semantics", + "/compat/linux/usr/share/remill/semantics", + "/compat/linux/usr/local/share/remill/semantics", +}; + +static bool CheckPath(const std::string &path) { + return !path.empty() && !access(path.c_str(), F_OK); +} + +static std::string InputBCPath(void) { + if (!FLAGS_bc_in.empty()) { + return FLAGS_bc_in; + } + + for (auto path : gSearchPaths) { + std::stringstream ss; + if ('/' != path[0]) { + ss << "./"; + } + ss << path; + if ('/' != path[strlen(path) - 1]) { + ss << "/"; + } + ss << FLAGS_arch_in << ".bc"; + auto sem_path = ss.str(); + if (CheckPath(sem_path)) { + return sem_path; + } + } + + LOG(FATAL) << "Cannot deduce path to " << FLAGS_arch_in + << " semantics bitcode file."; +} + +} // namespace + +int main(int argc, char *argv[]) { + google::ParseCommandLineFlags(&argc, &argv, true); + google::InitGoogleLogging(argv[0]); + + // GFlags will have removed everything that it recognized from argc/argv. + llvm::cl::ParseCommandLineOptions(argc, argv, "Remill: Lift CFG to LLVM"); + + if (FLAGS_os_in.empty()) { + std::cerr + << "Need to specify a source operating system with --os_in." + << std::endl; + return EXIT_FAILURE; + } + + if (FLAGS_os_out.empty()) { + std::cerr + << "Need to specify a target operating system with --os_out." + << std::endl; + return EXIT_FAILURE; + } + + if (FLAGS_arch_in.empty()) { + std::cerr + << "Need to specify a source architecture with --arch_in." + << std::endl; + return EXIT_FAILURE; + } + + if (FLAGS_arch_out.empty()) { + std::cerr + << "Need to specify a target architecture with --arch_out." + << std::endl; + return EXIT_FAILURE; + } + + if (FLAGS_cfg.empty()) { + std::cerr + << "Must specify CFG file with --cfg." + << std::endl; + return EXIT_FAILURE; + } + + if (FLAGS_bc_out.empty()) { + std::cerr + << "Please specify an output bitcode file with --bc_out." + << std::endl; + return EXIT_FAILURE; + } + + auto source_os = remill::GetOSName(FLAGS_os_in); + auto target_os = remill::GetOSName(FLAGS_os_out); + + auto source_arch = remill::Arch::Create(source_os, FLAGS_arch_in); + auto target_arch = remill::Arch::Create(target_os, FLAGS_arch_out); + + if (!CheckPath(FLAGS_cfg)) { + std::cerr + << "Must specify valid path for `--cfg`. CFG file " << FLAGS_cfg + << " cannot be opened." + << std::endl; + return EXIT_FAILURE; + } + + FLAGS_bc_in = InputBCPath(); + if (!CheckPath(FLAGS_bc_in)) { + std::cerr + << "Must specify valid path for `--bc_in`. Bitcode file " + << FLAGS_bc_in << " cannot be opened." + << std::endl; + return EXIT_FAILURE; + } + + auto context = new llvm::LLVMContext; + auto module = remill::LoadModuleFromFile(context, FLAGS_bc_in); + target_arch->PrepareModule(module); + + remill::Translator lifter(source_arch, module); + + auto cfg = remill::ReadCFG(FLAGS_cfg); + lifter.LiftCFG(cfg); + + remill::StoreModuleToFile(module, FLAGS_bc_out); + + delete cfg; + delete source_arch; + + google::ShutdownGoogleLogging(); + return EXIT_SUCCESS; +} diff --git a/remill/Translate.cpp b/remill/Translate.cpp deleted file mode 100644 index ac3384e4..00000000 --- a/remill/Translate.cpp +++ /dev/null @@ -1,94 +0,0 @@ -/* Copyright 2015 Peter Goodman (peter@trailofbits.com), all rights reserved. */ - -#include -#include - -#include -#include - -#include -#include - -#include "remill/BC/Translator.h" -#include "remill/BC/Util.h" -#include "remill/Arch/Arch.h" -#include "remill/CFG/CFG.h" - -#ifndef REMILL_OS -# if defined(__APPLE__) -# define REMILL_OS "mac" -# elif defined(__linux__) -# define REMILL_OS "linux" -# endif -#endif - -// TODO(pag): Support separate source and target architectures? -DEFINE_string(arch_in, "", "Architecture of the code being translated. " - "Valid architectures: x86, amd64."); - -DEFINE_string(arch_out, "", "Architecture of the target architecture on " - "which the translated code will run. Valid " - "architectures: x86, amd64."); - -DEFINE_string(os_in, REMILL_OS, "Source OS. Valid OSes: linux, mac."); -DEFINE_string(os_out, REMILL_OS, "Target OS. Valid OSes: linux, mac."); - -DEFINE_string(cfg, "", "Path to the CFG file containing code to lift."); - -DEFINE_string(bc_in, "", "Input bitcode file into which code will " - "be lifted. This should either be a semantics file " - "associated with `--arch_in`, or it should be " - "a bitcode file produced by `cfg_to_bc`. Chaining " - "bitcode files produces by `cfg_to_bc` can be " - "used to iteratively link in libraries to lifted " - "code."); - -DEFINE_string(bc_out, "", "Output bitcode file name."); - -extern "C" int main(int argc, char *argv[]) { - google::ParseCommandLineFlags(&argc, &argv, true); - google::InitGoogleLogging(argv[0]); - - // GFlags will have removed everything that it recognized from argc/argv. - llvm::cl::ParseCommandLineOptions(argc, argv, "Remill CFG to LLVM"); - - CHECK(!FLAGS_os_out.empty()) - << "Need to specify a target operating system with --os."; - - CHECK(!FLAGS_arch_in.empty()) - << "Need to specify a source architecture with --arch_in."; - - CHECK(!FLAGS_arch_out.empty()) - << "Need to specify a target architecture with --arch_out."; - - CHECK(!FLAGS_cfg.empty()) - << "Must specify CFG file with --cfg."; - - CHECK(!FLAGS_bc_out.empty()) - << "Please specify an input bitcode file with --bc_in."; - - CHECK(!FLAGS_bc_out.empty()) - << "Please specify an output bitcode file with --bc_out."; - - auto source_os = remill::GetOSName(FLAGS_os_in); - auto target_os = remill::GetOSName(FLAGS_os_out); - - auto source_arch = remill::Arch::Create(source_os, FLAGS_arch_in); - auto target_arch = remill::Arch::Create(target_os, FLAGS_arch_out); - - auto module = remill::LoadModuleFromFile(FLAGS_bc_in); - target_arch->PrepareModule(module); - - remill::Translator lifter(source_arch, module); - - auto cfg = remill::ReadCFG(FLAGS_cfg); - lifter.LiftCFG(cfg); - - remill::StoreModuleToFile(module, FLAGS_bc_out); - - delete cfg; - delete source_arch; - - google::ShutdownGoogleLogging(); - return EXIT_SUCCESS; -} diff --git a/scripts/bootstrap.sh b/scripts/bootstrap.sh deleted file mode 100755 index 68514400..00000000 --- a/scripts/bootstrap.sh +++ /dev/null @@ -1,345 +0,0 @@ -#!/usr/bin/env bash -# Copyright 2015 Peter Goodman (peter@trailofbits.com), all rights reserved. - -# Directory in which the script dir resides (i.e. Remill root dir). -DIR=$(dirname $( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )) - -RED=`tput setaf 1` -GREEN=`tput setaf 2` -YELLOW=`tput setaf 3` -BLUE=`tput setaf 4` -RESET=`tput sgr0` - - -GLOG_RELEASE=v0.3.4 -GFLAGS_RELEASE=v2.1.2 -PROTOBUF_VERSION=2.5.0 -PROTOBUF_RELEASE=protobuf-${PROTOBUF_VERSION} -GTEST_RELEASE=release-1.7.0 -XED_RELEASE=2016-02-02 - -LLVM_RELEASE=3.8.1 -LLVM_RELEASE_DIR=releases/${LLVM_RELEASE} - -if [[ "$OSTYPE" == "linux"* ]] ; then - FLAVOR=`grep -P '^NAME=' /etc/os-release` - XED_VERSION=xed-install-base-${XED_RELEASE}-lin-x86-64 - REMILL_OS_NAME="linux" - LIB_EXT=so - STDLIB="libstdc++" - LIB_LINK_FLAGS= - - if [[ "$FLAVOR" == *"Ubuntu"* ]] ; then - UBUNTU_VERSION=`lsb_release -r -s` - LLVM_VERSION=clang+llvm-${LLVM_RELEASE}-x86_64-linux-gnu-ubuntu-${UBUNTU_VERSION} - - elif [[ "$FLAVOR" == *"openSUSE"* ]] ; then - LLVM_VERSION=clang+llvm-3.8.1-x86_64-opensuse13.2 - - else - printf "${RED}Unsupported Linux flavor ${FLAVOR}${RESET}\n" - exit 1 - fi; - -elif [[ "$OSTYPE" == "darwin"* ]] ; then - XED_VERSION=xed-install-base-${XED_RELEASE}-mac-x86-64 - LLVM_RELEASE=3.8.0 - LLVM_VERSION=clang+llvm-${LLVM_RELEASE}-x86_64-apple-darwin - REMILL_OS_NAME="mac" - LIB_EXT=dylib - STDLIB="libc++" - LIB_LINK_FLAGS="-lc++ -lc++abi" - - if [ -z "$SDKROOT" ]; then - export SDKROOT=$(xcrun --sdk macosx --show-sdk-path) - fi - -else - printf "${RED}Unsupported platform: ${OSTYPE}${RESET}\n" - exit 1 -fi; - -XED_URL=https://software.intel.com/system/files/managed/58/cc/${XED_VERSION}.zip -LLVM_URL=http://llvm.org/${LLVM_RELEASE_DIR}/${LLVM_VERSION}.tar.xz -GLOG_URL=https://github.com/google/glog/archive/${GLOG_RELEASE}.tar.gz -GFLAGS_URL=https://github.com/gflags/gflags/archive/${GFLAGS_RELEASE}.tar.gz -PROTOBUF_URL=https://github.com/google/protobuf/releases/download/v${PROTOBUF_VERSION}/${PROTOBUF_RELEASE}.tar.gz -GTEST_URL=https://github.com/google/googletest/archive/${GTEST_RELEASE}.tar.gz - -function fix_library() -{ - if [[ "$OSTYPE" == "darwin"* ]] ; then - install_name_tool -id $DIR/third_party/lib/lib$1.dylib $DIR/third_party/lib/lib$1.dylib - fi; -} - -function category() -{ - printf "\n${GREEN}${1}${RESET}\n" -} - -function sub_category() -{ - printf "${YELLOW}${1}${RESET}\n" -} - -function notice() -{ - printf "${BLUE}${1}${RESET}\n" -} - -function error() -{ - printf "${RED}${1}${RESET}\n" - exit 1 -} - -function download_and_install_llvm() -{ - sub_category "Downloading LLVM and clang ${LLVM_VERSION}." - - # Create third_party directory if it doesn't exist - mkdir -p $DIR/third_party - pushd $DIR/third_party - - # Download a pre-built clang+llvm to it. OS X doesn't have wget by default. :( - curl -O ${LLVM_URL} - - # Extract it to the path expected by compile_semantics.sh - mkdir -p $DIR/third_party/src/llvm/ - tar xf ${LLVM_VERSION}.tar.xz -C $DIR/third_party --strip-components=1 - rm ${LLVM_VERSION}.tar.xz - popd -} - -function download_and_install_gflags() -{ - sub_category "Downloading and installing gflags." - - pushd $DIR/third_party - curl -L -O ${GFLAGS_URL} - mkdir -p $DIR/third_party/src/gflags - tar xf ${GFLAGS_RELEASE}.tar.gz -C src/gflags/ --strip-components=1 - rm ${GFLAGS_RELEASE}.tar.gz - popd - - pushd $DIR/third_party/src/gflags - mkdir build - cd build - cmake \ - -G "Unix Makefiles" \ - -DCMAKE_INSTALL_PREFIX:STRING=$DIR/third_party \ - -DGFLAGS_NAMESPACE:STRING=google \ - -DBUILD_SHARED_LIBS=ON \ - -DBUILD_STATIC_LIBS=ON \ - .. - make - make install - fix_library gflags - popd -} - -function download_and_install_glog() -{ - sub_category "Downloading and installing glog." - - pushd $DIR/third_party - - curl -L -O https://github.com/google/glog/archive/${GLOG_RELEASE}.tar.gz - mkdir -p $DIR/third_party/src/glog - tar xf ${GLOG_RELEASE}.tar.gz -C src/glog/ --strip-components=1 - rm ${GLOG_RELEASE}.tar.gz - popd - - pushd $DIR/third_party/src/glog - - ./configure \ - --prefix=$DIR/third_party \ - --libdir=$DIR/third_party/lib \ - --enable-static \ - --enable-shared \ - --disable-rtti \ - --disable-backtrace - - make - make install - fix_library glog - popd -} - -function download_and_install_protobuf() -{ - sub_category "Downloading and installing protobuf." - - pushd $DIR/third_party - curl -L -O ${PROTOBUF_URL} - mkdir -p $DIR/third_party/src/protobuf - tar xf ${PROTOBUF_RELEASE}.tar.gz -C src/protobuf/ --strip-components=1 - rm ${PROTOBUF_RELEASE}.tar.gz - popd - - pushd $DIR/third_party/src/protobuf - - CXXFLAGS="${CXXFLAGS} -DGOOGLE_PROTOBUF_NO_RTTI" \ - ./configure \ - --prefix=$DIR/third_party \ - --libdir=$DIR/third_party/lib \ - --enable-static \ - --enable-shared \ - --disable-rtti - make - make install - fix_library protobuf - popd -} - -function download_and_install_gtest() -{ - sub_category "Downloading and installing googletest." - pushd $DIR/third_party - curl -L -O ${GTEST_URL} - mkdir -p $DIR/third_party/src/googletest - tar xf ${GTEST_RELEASE}.tar.gz -C src/googletest/ --strip-components=1 - rm ${GTEST_RELEASE}.tar.gz - popd - - pushd $DIR/third_party/src/googletest - mkdir build - cd build - cmake \ - -G "Unix Makefiles" \ - -DCMAKE_INSTALL_PREFIX:STRING=$DIR/third_party \ - -DBUILD_SHARED_LIBS=ON \ - -DBUILD_STATIC_LIBS=OFF \ - .. - make - cp $DIR/third_party/src/googletest/build/libgtest.$LIB_EXT $DIR/third_party/lib - cp -r $DIR/third_party/src/googletest/include/gtest $DIR/third_party/include - fix_library gtest - popd -} - -function download_and_extract_xed() -{ - sub_category "Downloading and installing XED." - - if [[ ! -e $DIR/blob/xed/${XED_VERSION}.zip ]] ; then - error "Please download XED from ${XED_URL} and place it into ${DIR}/blob/xed/." - fi; - - mkdir -p $DIR/third_party/src/xed - unzip $DIR/blob/xed/${XED_VERSION}.zip -d $DIR/third_party/src/xed - - # 'install' XED. - mkdir -p $DIR/third_party/include/intel - cp -r $DIR/third_party/src/xed/kits/${XED_VERSION}/lib/* $DIR/third_party/lib - cp -r $DIR/third_party/src/xed/kits/${XED_VERSION}/include/* $DIR/third_party/include/intel -} - -function create_directory_tree() -{ - mkdir -p $DIR/third_party - mkdir -p $DIR/third_party/bin - mkdir -p $DIR/third_party/lib - mkdir -p $DIR/third_party/include - mkdir -p $DIR/third_party/src - mkdir -p $DIR/third_party/share - - mkdir -p $DIR/generated - mkdir -p $DIR/generated/Arch - mkdir -p $DIR/generated/CFG - - touch $DIR/generated/__init__.py - touch $DIR/generated/CFG/__init__.py -} - -function change_compiler_to_llvm() -{ - export CC="${DIR}/third_party/bin/clang" - export CXX="${DIR}/third_party/bin/clang++" - - export CFLAGS="-isystem ${DIR}/third_party/include -g3 -fno-omit-frame-pointer -fPIC" - export CXXFLAGS="-isystem ${DIR}/third_party/include -g3 -fno-omit-frame-pointer -fPIC -fno-rtti -fno-exceptions -stdlib=${STDLIB}" - export LDFLAGS="-g ${LIB_LINK_FLAGS}" -} - -function download_dependencies() -{ - category "Checking dependencies." - - if [[ -e $DIR/third_party/bin/clang ]] ; then - notice "LLVM and clang FOUND!" - else - download_and_install_llvm - fi; - - change_compiler_to_llvm - - if [[ -e $DIR/third_party/lib/libgflags.$LIB_EXT ]] ; then - notice "gflags FOUND!" - else - download_and_install_gflags - fi; - - if [[ -e $DIR/third_party/lib/libglog.$LIB_EXT ]] ; then - notice "${BLUE}glog FOUND!" - else - download_and_install_glog - fi; - - if [[ -e $DIR/third_party/bin/protoc ]] ; then - notice "${BLUE}protobuf FOUND!" - else - download_and_install_protobuf - fi; - - if [[ -e $DIR/third_party/lib/libgtest.$LIB_EXT ]] ; then - notice "${BLUE}googletest FOUND!" - else - download_and_install_gtest - fi; - - if [[ -e $DIR/third_party/lib/libxed.$LIB_EXT || -e $DIR/third_party/lib/libxed.a ]] ; then - notice "${BLUE}XED FOUND!" - else - download_and_extract_xed - fi; -} - -function generate_files() -{ - # Create the generated files directories. - category "Auto-generating files." - pushd $DIR - - # Generate 32- and 64-bit x86 machine state modules for importing by - # `cfg_to_bc`. - sub_category "Generating architecture-specific state files." - $DIR/scripts/compile_semantics.sh || { - error "Error compiling instruction semantics." - } - - # Generate the protocol buffer file for the CFG definition. The lifter will - # read in CFG protobuf files and output LLVM bitcode files. - sub_category "Generating protocol buffers." - cd $DIR/generated/CFG - cp $DIR/remill/CFG/CFG.proto $DIR/generated/CFG - $DIR/third_party/bin/protoc --cpp_out=. CFG.proto - $DIR/third_party/bin/protoc --python_out=. CFG.proto - - sub_category "Generating test save state code." - $DIR/scripts/print_x86_save_state_asm.sh - - popd -} - -function build_remill() -{ - category "Compiling Remill" - python $DIR/scripts/build.py --debug --num_workers 1 --stop_on_error -} - -create_directory_tree -download_dependencies -generate_files -build_remill diff --git a/scripts/compile_semantics.sh b/scripts/compile_semantics.sh deleted file mode 100755 index 931b24fe..00000000 --- a/scripts/compile_semantics.sh +++ /dev/null @@ -1,87 +0,0 @@ -#!/usr/bin/env bash - -DIR=$(dirname $( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )) - -RED=`tput setaf 1` -BLUE=`tput setaf 4` -RESET=`tput sgr0` - -CXXFLAGS= -CXXFLAGS+=" -isystem ${DIR}/third_party/include" -CXXFLAGS+=" -isystem ${DIR}/third_party/include/c++/v1" -CXXFLAGS+=" -I${DIR}" -CXXFLAGS+=" -std=gnu++11 -g0 -O0" -CXXFLAGS+=" -fno-exceptions -fno-rtti -fno-asynchronous-unwind-tables" -CXXFLAGS+=" -ffreestanding -fno-common -fno-builtin" -CXXFLAGS+=" -Wall -Werror -Wshadow -Wconversion -Wshorten-64-to-32 -pedantic" -CXXFLAGS+=" -Wno-gnu-anonymous-struct -Wno-return-type-c-linkage" -CXXFLAGS+=" -Wno-gnu-zero-variadic-macro-arguments -Wno-nested-anon-types" -CXXFLAGS+=" -Wno-extended-offsetof -Wno-gnu-statement-expression" -CXXFLAGS+=" -Wno-c99-extensions -fno-vectorize -fno-slp-vectorize" -CXXFLAGS+=" -Wno-zero-length-array -Wno-ignored-attributes" - -function compile_x86() -{ - MACROS="-DADDRESS_SIZE_BITS=$1 -DHAS_FEATURE_AVX=$2 -DHAS_FEATURE_AVX512=$3" - FILE_NAME=sem - if [[ $1 -eq 64 ]] ; then - FILE_NAME="${FILE_NAME}_amd64" - MESSAGE="Building for AMD64" - else - FILE_NAME="${FILE_NAME}_x86" - MESSAGE="Building for x86" - fi - - if [[ $3 -eq 1 ]] ; then - MESSAGE="${MESSAGE} with AVX512" - FILE_NAME="${FILE_NAME}_avx512" - elif [[ $2 -eq 1 ]] ; then - MESSAGE="${MESSAGE} with AVX" - FILE_NAME="${FILE_NAME}_avx" - fi - - printf "${BLUE}${MESSAGE}${RESET}\n" - - $DIR/third_party/bin/clang++ -x c++ \ - -emit-llvm -O0 -g0 -m$1 -mtune=generic $MACROS $CXXFLAGS \ - -ffunction-sections -fdata-sections \ - -c $DIR/remill/Arch/X86/Runtime/Instructions.cpp \ - -o $DIR/generated/Arch/X86/Runtime/${FILE_NAME}_instr.bc - - $DIR/third_party/bin/clang++ -x c++ \ - -emit-llvm -O0 -g0 -m$1 -mtune=generic $MACROS $CXXFLAGS \ - -ffunction-sections -fdata-sections \ - -c $DIR/remill/Arch/X86/Runtime/BasicBlock.cpp \ - -o $DIR/generated/Arch/X86/Runtime/${FILE_NAME}_block.bc - - $DIR/third_party/bin/clang \ - -emit-llvm -O3 -g0 -m$1 -mtune=generic \ - -ffunction-sections -fdata-sections \ - -c $DIR/generated/Arch/X86/Runtime/${FILE_NAME}_instr.bc \ - -o $DIR/generated/Arch/X86/Runtime/${FILE_NAME}_instr.opt.bc - - $DIR/third_party/bin/llvm-link \ - -o=$DIR/generated/${FILE_NAME}.bc \ - $DIR/generated/Arch/X86/Runtime/${FILE_NAME}_block.bc \ - $DIR/generated/Arch/X86/Runtime/${FILE_NAME}_instr.opt.bc - - if [[ ! -e $DIR/generated/${FILE_NAME}.bc ]] ; then - printf "${RED}Error: ${MESSAGE}${RESET}\n" - exit 1 - fi; -} - -mkdir -p $DIR/generated/ -mkdir -p $DIR/generated/Arch/ -mkdir -p $DIR/generated/Arch/X86/ -mkdir -p $DIR/generated/Arch/X86/Runtime - -compile_x86 32 0 0 -compile_x86 32 1 0 -compile_x86 32 1 1 - -compile_x86 64 0 0 -compile_x86 64 1 0 -compile_x86 64 1 1 - -exit 0 diff --git a/scripts/install.sh b/scripts/install.sh deleted file mode 100755 index ca243ba4..00000000 --- a/scripts/install.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/env bash -# Copyright 2016 Peter Goodman (peter@trailofbits.com), all rights reserved. - -DIR=$(dirname $( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )) - -RED=`tput setaf 1` -RESET=`tput sgr0` - -PREFIX=$1 -if [[ -z "${PREFIX}" ]] ; then - printf "${RED}ERROR! Specify install prefix with first argument.${RESET}\n" - exit 1 -fi - -if [[ ! -e $DIR/build/cfg_to_bc ]] ; then - $DIR/scripts/build.py -fi - -mkdir -p $PREFIX/bin -mkdir -p $PREFIX/include -mkdir -p $PREFIX/lib/remill -mkdir -p $PREFIX/share/remill - -cp $DIR/build/cfg_to_bc $PREFIX/bin -cp $DIR/build/libOptimize.* $PREFIX/lib/remill -cp $DIR/generated/sem_* $PREFIX/share/remill - diff --git a/scripts/travis/install.sh b/scripts/travis/install.sh new file mode 100755 index 00000000..b1739b7e --- /dev/null +++ b/scripts/travis/install.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# Copyright 2016 Peter Goodman (peter@trailofbits.com), all rights reserved. + +DIR=$(dirname $(dirname $( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ))) + +if [[ "${TRAVIS_OS_NAME}" = "linux" ]] ; then + + # Print out useful information to see what Linux environment is being used + # by Travis. + uname -a + lsb_release -a + + sudo apt-get update -qq + sudo apt-get --force-yes purge isc-dhcp-client + sudo apt-get install -y isc-dhcp-client + sudo apt-get upgrade -yqq + + $DIR/scripts/travis/install_linux.sh +fi + +if [[ "${TRAVIS_OS_NAME}" = "osx" ]] ; then + $DIR/travis/install_macos.sh +fi diff --git a/scripts/travis/install_linux.sh b/scripts/travis/install_linux.sh new file mode 100755 index 00000000..ce737d14 --- /dev/null +++ b/scripts/travis/install_linux.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash +# Copyright 2016 Peter Goodman (peter@trailofbits.com), all rights reserved. + +DIR=$(dirname $(dirname $( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ))) +UBUNTU_RELEASE=`lsb_release -sc` + +# Make sure we have `add-apt-repository`. +sudo apt-get update -qq +sudo apt-get upgrade -yqq +sudo apt-get install -y software-properties-common +sudo apt-get install -y build-essential + +# Add the LLVM repositories and keys. +wget -qO - http://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add - +sudo add-apt-repository -y "deb http://apt.llvm.org/${UBUNTU_RELEASE}/ llvm-toolchain-${UBUNTU_RELEASE} main" +sudo add-apt-repository -y "deb http://apt.llvm.org/${UBUNTU_RELEASE}/ llvm-toolchain-${UBUNTU_RELEASE}-3.8 main" +sudo add-apt-repository -y "deb http://apt.llvm.org/${UBUNTU_RELEASE}/ llvm-toolchain-${UBUNTU_RELEASE}-3.9 main" + +# Add the CMake repository. +sudo add-apt-repository -y ppa:george-edison55/cmake-3.x + +# Update sources list, and then install needed packages. +sudo apt-get update -qq +sudo apt-get install -y git +sudo apt-get install -y libgoogle-glog-dev +sudo apt-get install -y libgtest-dev +sudo apt-get install -y libprotoc-dev +sudo apt-get install -y libprotobuf-dev +sudo apt-get install -y libprotobuf-dev +sudo apt-get install -y protobuf-compiler +sudo apt-get install -y python2.7 +sudo apt-get install -y python-pip +sudo apt-get install -y llvm-3.9-dev +sudo apt-get install -y clang-3.9 +sudo apt-get install -y g++-multilib +sudo apt-get install -y unzip +sudo apt-get install -y cmake + +# Upgrade PIP and install the python bindings for protocol buffers. +sudo pip install --upgrade pip +sudo pip install python-magic +sudo pip install 'protobuf==2.4.1' + +# Unpack and install Intel XED. +$DIR/scripts/unix/install_xed.sh + +# Compile and install Google Test. +$DIR/scripts/unix/install_gtest.sh + +# Compile .proto files into C++ and python files. +$DIR/scripts/unix/compile_protobufs.sh + +# Build and install Remill. +mkdir remill_build +pushd remill_build +cmake \ +-DCMAKE_C_COMPILER=clang-3.9 \ +-DCMAKE_CXX_COMPILER=clang++-3.9 \ +-DCMAKE_LLVM_LINK=llvm-link-3.9 \ +.. + +make semantics VERBOSE=1 +make all VERBOSE=1 +sudo make install +popd + diff --git a/scripts/travis/install_macos.sh b/scripts/travis/install_macos.sh new file mode 100755 index 00000000..54e20c8a --- /dev/null +++ b/scripts/travis/install_macos.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +# Copyright 2016 Peter Goodman (peter@trailofbits.com), all rights reserved. + +exit 1 diff --git a/scripts/unix/compile_protobufs.sh b/scripts/unix/compile_protobufs.sh new file mode 100755 index 00000000..bfad7473 --- /dev/null +++ b/scripts/unix/compile_protobufs.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +# Copyright 2015 Peter Goodman (peter@trailofbits.com), all rights reserved. + +DIR=$(dirname $(dirname $( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ))) + +mkdir -p $DIR/remill/CFG/CFG +cd $DIR/generated/CFG +cp $DIR/remill/CFG/CFG.proto $DIR/generated/CFG + +protoc --cpp_out=. CFG.proto +protoc --python_out=. CFG.proto diff --git a/scripts/unix/install_gtest.sh b/scripts/unix/install_gtest.sh new file mode 100755 index 00000000..aa3de32f --- /dev/null +++ b/scripts/unix/install_gtest.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +# Copyright 2015 Peter Goodman (peter@trailofbits.com), all rights reserved. + +# Build and install Google Test. + +if [[ "$OSTYPE" == "linux-gnu" ]] ; then + sudo apt-get install -y libgtest-dev + mkdir gtest_build + pushd gtest_build + cmake /usr/src/gtest + make + sudo cp *.a /usr/local/lib + sudo ldconfig + popd + +# elif [[ "$OSTYPE" == "darwin"* ]] ; then +# OS_NAME=macos + +else + printf "Unsupported platform: ${OSTYPE}${RESET}\n" > /dev/stderr + exit 1 +fi \ No newline at end of file diff --git a/scripts/unix/install_xed.sh b/scripts/unix/install_xed.sh new file mode 100755 index 00000000..92c2b111 --- /dev/null +++ b/scripts/unix/install_xed.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +# Copyright 2015 Peter Goodman (peter@trailofbits.com), all rights reserved. + +DIR=$(dirname $(dirname $( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ))) + + +INSTALL_DIR=/usr/local +INSTALL_INCLUDE_DIR=$INSTALL_DIR/include/intel +INSTALL_LIB_DIR=$INSTALL_DIR/lib +XED_RELEASE=2016-02-02 + +function error() +{ + printf "${1}\n" + exit 1 +} + +# Figure out what version of XED to install. +if [[ "$OSTYPE" == "linux"* ]] ; then + XED_VERSION=xed-install-base-${XED_RELEASE}-lin-x86-64 +elif [[ "$OSTYPE" == "darwin"* ]] ; then + XED_VERSION=xed-install-base-${XED_RELEASE}-mac-x86-64 +else + error "Unsupported platform: ${OSTYPE}" +fi ; + +if [[ ! -e $DIR/blob/xed/${XED_VERSION}.zip ]] ; then + error "Please download XED from ${XED_URL} and place it into ${DIR}/blob/xed/." +fi; + +mkdir -p $DIR/third_party +rm -rf $DIR/third_party/xed +unzip $DIR/blob/xed/${XED_VERSION}.zip -d $DIR/third_party/xed + +sudo mkdir -p $INSTALL_INCLUDE_DIR +sudo install -t $INSTALL_LIB_DIR $DIR/third_party/xed/kits/${XED_VERSION}/lib/* +sudo install -t $INSTALL_INCLUDE_DIR $DIR/third_party/xed/kits/${XED_VERSION}/include/* + +sudo ldconfig + +exit 0 diff --git a/scripts/x86/generate_tests.sh b/scripts/x86/generate_tests.sh new file mode 100755 index 00000000..0d180f47 --- /dev/null +++ b/scripts/x86/generate_tests.sh @@ -0,0 +1,112 @@ +#!/usr/bin/env bash +# Copyright 2015 Peter Goodman (peter@trailofbits.com), all rights reserved. + +DIR=$(dirname $(dirname $( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ))) + +if [[ "$OSTYPE" == "linux-gnu" ]] ; then + OS_NAME=linux + +elif [[ "$OSTYPE" == "darwin"* ]] ; then + OS_NAME=macos + +else + printf "Unsupported platform: ${OSTYPE}${RESET}\n" > /dev/stderr + exit 1 +fi + + +printf "Found OS ${OS_NAME}\n" + +CXX=$(which clang++-3.9) +if [[ $? -ne 0 ]] ; then + CXX=$(which clang++) +fi + +printf "Found C++ compiler ${CXX}\n" + +function generate_tests() +{ + printf "Generating tests for ${4}\n" + ${CXX} \ + -I${DIR} \ + -std=gnu++11 \ + -Wno-nested-anon-types \ + -Wno-variadic-macros \ + -Wno-extended-offsetof \ + -Wno-return-type-c-linkage \ + -Wno-expansion-to-defined \ + -m64 \ + -I${DIR} \ + -DIN_TEST_GENERATOR \ + -DADDRESS_SIZE_BITS=${1} \ + -DHAS_FEATURE_AVX=${2} \ + -DHAS_FEATURE_AVX512=${3} \ + -DGOOGLE_PROTOBUF_NO_RTTI \ + $DIR/tests/X86/Tests.S \ + $DIR/tests/X86/Generate.cpp \ + $DIR/remill/CFG/CFG.cpp \ + -lxed \ + -lgflags \ + -lglog \ + -lprotobuf + + ./a.out > $DIR/generated/Arch/X86/Tests/${4}.cfg + rm ./a.out +} + +function lift_tests() +{ + printf "Lifting tests for ${1}\n" + remill-lift --cfg $DIR/generated/Arch/X86/Tests/${1}.cfg \ + --os_in ${OS_NAME} --os_out ${OS_NAME} \ + --arch_in ${1} --arch_out amd64 \ + --bc_out $DIR/generated/Arch/X86/Tests/${1}.cfg.bc +} + +function compile_tests() +{ + printf "Compiling tests for ${4}\n" + ${CXX} \ + -I${DIR} \ + -std=gnu++11 \ + -Wno-nested-anon-types \ + -Wno-variadic-macros \ + -Wno-extended-offsetof \ + -Wno-return-type-c-linkage \ + -Wno-expansion-to-defined \ + -Wno-override-module \ + -m64 \ + -I${DIR} \ + -DADDRESS_SIZE_BITS=${1} \ + -DHAS_FEATURE_AVX=${2} \ + -DHAS_FEATURE_AVX512=${3} \ + -DGOOGLE_PROTOBUF_NO_RTTI \ + $DIR/tests/X86/Tests.S \ + $DIR/tests/X86/Run.cpp \ + $DIR/generated/Arch/X86/Tests/${4}.cfg.bc \ + -o $DIR/generated/Arch/X86/Tests/${4} \ + -lgflags \ + -lglog \ + -lgtest \ + -lpthread +} + +mkdir -p $DIR/generated/Arch/X86/Tests + +pushd /tmp + +generate_tests 32 0 0 x86 +generate_tests 32 1 0 x86_avx +generate_tests 64 0 0 amd64 +generate_tests 64 1 0 amd64_avx + +lift_tests x86 +lift_tests x86_avx +lift_tests amd64 +lift_tests amd64_avx + +compile_tests 32 0 0 x86 +compile_tests 32 1 0 x86_avx +compile_tests 64 0 0 amd64 +compile_tests 64 1 0 amd64_avx +popd \ No newline at end of file diff --git a/scripts/print_x86_save_state_asm.sh b/scripts/x86/print_save_state_asm.sh similarity index 66% rename from scripts/print_x86_save_state_asm.sh rename to scripts/x86/print_save_state_asm.sh index 73ef2143..42e32fa3 100755 --- a/scripts/print_x86_save_state_asm.sh +++ b/scripts/x86/print_save_state_asm.sh @@ -3,10 +3,15 @@ # This script is a convenience script for generating some assembly code that # is a template for saving the machine state to a `State` structure. -DIR=$(dirname $( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )) +DIR=$(dirname $(dirname $( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ))) + +CXX=$(which clang++-3.9) +if [[ $? -ne 0 ]] ; then + CXX=$(which clang++) +fi pushd /tmp -$DIR/third_party/bin/clang++ \ +${CXX} \ -std=gnu++11 \ -Wno-nested-anon-types -Wno-variadic-macros -Wno-extended-offsetof \ -Wno-return-type-c-linkage \ @@ -14,5 +19,6 @@ $DIR/third_party/bin/clang++ \ -DADDRESS_SIZE_BITS=64 -DHAS_FEATURE_AVX=1 -DHAS_FEATURE_AVX512=1 \ $DIR/tests/X86/PrintSaveState.cpp -./a.out > $DIR/tests/X86/SaveState.S +./a.out > $DIR/generated/Arch/X86/SaveState.S +rm ./a.out popd diff --git a/scripts/run_tests_x86.sh b/scripts/x86/run_tests.sh similarity index 57% rename from scripts/run_tests_x86.sh rename to scripts/x86/run_tests.sh index 2c5de514..26ea680d 100755 --- a/scripts/run_tests_x86.sh +++ b/scripts/x86/run_tests.sh @@ -1,8 +1,7 @@ #!/usr/bin/env bash # Copyright 2015 Peter Goodman (peter@trailofbits.com), all rights reserved. -# Directory in which the script dir resides (i.e. Remill root dir). -DIR=$(dirname $( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )) +DIR=$(dirname $(dirname $( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ))) if [[ "$OSTYPE" == "linux-gnu" ]]; then HAS_AVX=`cat /proc/cpuinfo | grep -o 'avx ' | wc -w` @@ -17,12 +16,12 @@ fi EXIT_CODE=0 -$DIR/build/run_tests_x86 || { EXIT_CODE=1 ; } -$DIR/build/run_tests_amd64 || { EXIT_CODE=1 ; } +$DIR/generated/Arch/X86/Tests/x86 || { EXIT_CODE=1 ; } +$DIR/generated/Arch/X86/Tests/amd64 || { EXIT_CODE=1 ; } if [[ 0 -lt $HAS_AVX ]] ; then - $DIR/build/run_tests_x86_avx || { EXIT_CODE=1 ; } - $DIR/build/run_tests_amd64_avx || { EXIT_CODE=1 ; } + $DIR/generated/Arch/X86/Tests/x86_avx || { EXIT_CODE=1 ; } + $DIR/generated/Arch/X86/Tests/amd64_avx || { EXIT_CODE=1 ; } fi exit $EXIT_CODE diff --git a/tests/X86/Generate.cpp b/tests/X86/Generate.cpp index f9af2cfc..e3a747b5 100644 --- a/tests/X86/Generate.cpp +++ b/tests/X86/Generate.cpp @@ -92,6 +92,8 @@ static void AddFunctionToModule(remill::cfg::Module *module, instr->set_address(addr); addr += ilen; } + + module->add_referenced_blocks(test.test_end); } } // namespace