From 592b6cbe8d9403de729d2ca12a8094fc07e2ff01 Mon Sep 17 00:00:00 2001 From: Peter Goodman Date: Wed, 14 Jun 2017 19:57:57 -0400 Subject: [PATCH] Aarch64 support and other stuff (#113) * This includes most of Alessandro's wonderful cmake changes, and starts into AArch64 support, though not using Capstone anymore. * Trying to make travis work * Minor additions, still not completely working on my ubuntu 14.04 vm or on travis * Making progress on getting travis to build * Some documentation improvements * I push the travis button --- .clang-format | 96 + .gitignore | 6 + .travis.yml | 18 +- CMakeLists.txt | 18 +- README.md | 82 +- cmake/BCCompiler/CMakeBCCompiler.cmake.in | 14 + cmake/BCCompiler/CMakeBCInformation.cmake | 19 +- .../BCCompiler/CMakeDetermineBCCompiler.cmake | 14 + cmake/BCCompiler/CMakeTestBCCompiler.cmake | 14 + distro_packages/ArchLinux/.gitignore | 14 + docs/LIFE_OF_AN_INSTRUCTION.md | 261 +- remill/Arch/AArch64/Arch.cpp | 672 + remill/Arch/AArch64/Decode.cpp | 61129 +++++++++++++ remill/Arch/AArch64/Decode.h | 7170 ++ remill/Arch/AArch64/Etc/GenOpMap.py | 1076 + remill/Arch/AArch64/Extract.cpp | 76131 ++++++++++++++++ remill/Arch/AArch64/Runtime/BasicBlock.cpp | 136 + remill/Arch/AArch64/Runtime/CMakeLists.txt | 44 + remill/Arch/AArch64/Runtime/Instructions.cpp | 108 + remill/Arch/AArch64/Runtime/Operators.h | 37 + remill/Arch/AArch64/Runtime/State.h | 218 + remill/Arch/AArch64/Runtime/Types.h | 58 + remill/Arch/AArch64/Semantics/BINARY.cpp | 74 + remill/Arch/AArch64/Semantics/BRANCH.cpp | 145 + remill/Arch/AArch64/Semantics/CALL_RET.cpp | 33 + remill/Arch/AArch64/Semantics/DATAXFER.cpp | 112 + remill/Arch/AArch64/Semantics/FLAGS.cpp | 166 + .../Arch.h => AArch64/Semantics/MISC.cpp} | 39 +- remill/Arch/Arch.cpp | 50 +- remill/Arch/Arch.h | 15 +- remill/Arch/Instruction.cpp | 77 +- remill/Arch/Instruction.h | 46 +- remill/Arch/Name.cpp | 16 +- remill/Arch/Name.h | 9 +- remill/Arch/Runtime/Definitions.h | 8 +- remill/Arch/Runtime/HyperCall.h | 6 + remill/Arch/Runtime/Operators.h | 41 + remill/Arch/X86/Arch.cpp | 299 +- remill/Arch/X86/Runtime/BasicBlock.cpp | 10 +- remill/Arch/X86/Runtime/CMakeLists.txt | 14 + remill/BC/Lifter.cpp | 221 +- remill/BC/Lifter.h | 25 +- remill/BC/Util.cpp | 110 +- scripts/build_ubuntu.sh | 97 + scripts/travis/build.sh | 94 - scripts/travis/install.sh | 35 - scripts/travis/install_linux.sh | 39 - tests/X86/Lift.cpp | 25 +- .../install_macos.sh => tools/CMakeLists.txt | 11 +- 49 files changed, 148394 insertions(+), 758 deletions(-) create mode 100644 .clang-format create mode 100644 remill/Arch/AArch64/Arch.cpp create mode 100644 remill/Arch/AArch64/Decode.cpp create mode 100644 remill/Arch/AArch64/Decode.h create mode 100644 remill/Arch/AArch64/Etc/GenOpMap.py create mode 100644 remill/Arch/AArch64/Extract.cpp create mode 100644 remill/Arch/AArch64/Runtime/BasicBlock.cpp create mode 100644 remill/Arch/AArch64/Runtime/CMakeLists.txt create mode 100644 remill/Arch/AArch64/Runtime/Instructions.cpp create mode 100644 remill/Arch/AArch64/Runtime/Operators.h create mode 100644 remill/Arch/AArch64/Runtime/State.h create mode 100644 remill/Arch/AArch64/Runtime/Types.h create mode 100644 remill/Arch/AArch64/Semantics/BINARY.cpp create mode 100644 remill/Arch/AArch64/Semantics/BRANCH.cpp create mode 100644 remill/Arch/AArch64/Semantics/CALL_RET.cpp create mode 100644 remill/Arch/AArch64/Semantics/DATAXFER.cpp create mode 100644 remill/Arch/AArch64/Semantics/FLAGS.cpp rename remill/Arch/{X86/Arch.h => AArch64/Semantics/MISC.cpp} (51%) create mode 100755 scripts/build_ubuntu.sh delete mode 100755 scripts/travis/build.sh delete mode 100755 scripts/travis/install.sh delete mode 100755 scripts/travis/install_linux.sh rename scripts/travis/install_macos.sh => tools/CMakeLists.txt (76%) mode change 100755 => 100644 diff --git a/.clang-format b/.clang-format new file mode 100644 index 00000000..981a0d28 --- /dev/null +++ b/.clang-format @@ -0,0 +1,96 @@ +--- +Language: Cpp +# BasedOnStyle: Google +AccessModifierOffset: -1 +AlignAfterOpenBracket: Align +AlignConsecutiveAssignments: false +AlignConsecutiveDeclarations: false +AlignEscapedNewlinesLeft: true +AlignOperands: true +AlignTrailingComments: true +AllowAllParametersOfDeclarationOnNextLine: true +AllowShortBlocksOnASingleLine: false +AllowShortCaseLabelsOnASingleLine: false +AllowShortFunctionsOnASingleLine: All +AllowShortIfStatementsOnASingleLine: true +AllowShortLoopsOnASingleLine: true +AlwaysBreakAfterDefinitionReturnType: None +AlwaysBreakAfterReturnType: None +AlwaysBreakBeforeMultilineStrings: true +AlwaysBreakTemplateDeclarations: true +BinPackArguments: true +BinPackParameters: true +BraceWrapping: + AfterClass: false + AfterControlStatement: false + AfterEnum: false + AfterFunction: false + AfterNamespace: false + AfterObjCDeclaration: false + AfterStruct: false + AfterUnion: false + BeforeCatch: false + BeforeElse: false + IndentBraces: false +BreakBeforeBinaryOperators: None +BreakBeforeBraces: Attach +BreakBeforeTernaryOperators: true +BreakConstructorInitializersBeforeComma: false +BreakAfterJavaFieldAnnotations: false +BreakStringLiterals: true +ColumnLimit: 80 +CommentPragmas: '^ IWYU pragma:' +ConstructorInitializerAllOnOneLineOrOnePerLine: true +ConstructorInitializerIndentWidth: 4 +ContinuationIndentWidth: 4 +Cpp11BracedListStyle: true +DerivePointerAlignment: true +DisableFormat: false +ExperimentalAutoDetectBinPacking: false +ForEachMacros: [ foreach, Q_FOREACH, BOOST_FOREACH ] +IncludeCategories: + - Regex: '^<.*\.h>' + Priority: 1 + - Regex: '^<.*' + Priority: 2 + - Regex: '.*' + Priority: 3 +IncludeIsMainRegex: '([-_](test|unittest))?$' +IndentCaseLabels: true +IndentWidth: 2 +IndentWrappedFunctionNames: false +JavaScriptQuotes: Leave +JavaScriptWrapImports: true +KeepEmptyLinesAtTheStartOfBlocks: false +MacroBlockBegin: '' +MacroBlockEnd: '' +MaxEmptyLinesToKeep: 1 +NamespaceIndentation: None +ObjCBlockIndentWidth: 2 +ObjCSpaceAfterProperty: false +ObjCSpaceBeforeProtocolList: false +PenaltyBreakBeforeFirstCallParameter: 1 +PenaltyBreakComment: 300 +PenaltyBreakFirstLessLess: 120 +PenaltyBreakString: 1000 +PenaltyExcessCharacter: 1000000 +PenaltyReturnTypeOnItsOwnLine: 200 +PointerAlignment: Left +ReflowComments: true +SortIncludes: true +SpaceAfterCStyleCast: false +SpaceAfterTemplateKeyword: true +SpaceBeforeAssignmentOperators: true +SpaceBeforeParens: ControlStatements +SpaceInEmptyParentheses: false +SpacesBeforeTrailingComments: 2 +SpacesInAngles: false +SpacesInContainerLiterals: true +SpacesInCStyleCastParentheses: false +SpacesInParentheses: false +SpacesInSquareBrackets: false +Standard: Auto +TabWidth: 2 +UseTab: Never +... + diff --git a/.gitignore b/.gitignore index 8d534531..6841bf5a 100644 --- a/.gitignore +++ b/.gitignore @@ -15,6 +15,12 @@ install_manifest.txt Makefile +CMakeLists.txt.user +.vscode +.idea +cmake-build-debug +cmake-build-release + bin/* lib/* diff --git a/.travis.yml b/.travis.yml index 7ceaf527..95c6440e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,27 +1,17 @@ sudo: required dist: trusty -osx_image: xcode8 language: generic os: - linux - - osx compiler: - gcc - - clang -matrix: - exclude: - - compiler: gcc - os: osx - - compiler: clang - os: linux - allow_failures: - - os: osx install: - - ./scripts/travis/install.sh + - ./scripts/build_ubuntu.sh before_script: - - cd build + - cd ../remill-build + - make build_x86_tests script: - - if [[ "${TRAVIS_OS_NAME}" == "linux" ]] ; then ctest ; fi + - make test notifications: slack: secure: eW0eTnlEz1Pa68ZJOps7L0UscoxoFa+JLuOIOy4PPrLs1DfIiGdhHMfWs2U8mF8YaBYmrXgav/1IxNVtdPxiwfyLdYbiNSoEfo1ofPpizTH02flXthnUVIzlQD1jT6OSk840vskT2WWufBEkoaEIxFDXVI8PFtAtIYyfveVEnRdkTPUvuLV0pTX92c+ZCH3FIWtDey+4aVUwsttJSVloDSExy27LuoBLDPSNX8goiZFFrIcBRpspumysEJXdj5FSNiED9TQA1NCGWhsUqUlTPMYp1B8M/lKciugqhAlGijSL3lOisJZWsf/YZcto/N7wNMU54YNbfmMUZu6K9UkveVOD8qtlAs/8MDPyH0O+s/Hf2y6KGb7PSO9sxYF/WWRLNqxp8SgmY3IKu8niglXsHBNd4cXibvM3a2KJoJo453PI48XAfaygPYcYcuNedsQT9Fng384vpUjV7YLLU1452qTjx5T5SarjeF4ExtzJVYAQWhFT7RkNpdtDnKNLczwicM8wRSCHfEmGLR1Dpw5zQlS3uFJydTWTgAeLSHKKkfgqLYiR7/Ongv1M8Q+JUgYtF2dV9PLEpb0QGzw6B/RnW2vGNWIE+JJCjpZTizQj0f8UIh+k//v9jL4simVHRDPArnvJaFP0T4BrUl4Z4XN/dYFOogp9rCHgQCvqVOCPrFI= diff --git a/CMakeLists.txt b/CMakeLists.txt index 9a39cd10..bf0fd65c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -98,7 +98,7 @@ if (NOT WIN32) endif () # warnings and compiler settings -set(PROJECT_CXXWARNINGS "-Wall -Wextra -Werror -pedantic -Wno-unused-parameter -Wno-c++98-compat -Wno-unreachable-code-return -Wno-nested-anon-types -Wno-extended-offsetof -Wno-gnu-anonymous-struct -Wno-gnu-designator -Wno-variadic-macros -Wno-gnu-zero-variadic-macro-arguments -Wno-gnu-statement-expression -Wno-return-type-c-linkage -Wno-c99-extensions -Wno-ignored-attributes -Wno-unused-local-typedef") +set(PROJECT_CXXWARNINGS "-Wall -Wextra -Wno-unused-parameter -Wno-c++98-compat -Wno-unreachable-code-return -Wno-nested-anon-types -Wno-extended-offsetof -Wno-gnu-anonymous-struct -Wno-gnu-designator -Wno-variadic-macros -Wno-gnu-zero-variadic-macro-arguments -Wno-gnu-statement-expression -Wno-return-type-c-linkage -Wno-c99-extensions -Wno-ignored-attributes -Wno-unused-local-typedef") set(PROJECT_CXXFLAGS "${PROJECT_CXXFLAGS} -Wno-unknown-warning-option ${PROJECT_CXXWARNINGS} -fPIC -fno-omit-frame-pointer -fvisibility-inlines-hidden -fno-exceptions -fno-asynchronous-unwind-tables -fno-rtti") # default build type @@ -150,9 +150,15 @@ list(APPEND PROJECT_LIBRARIES gflags) list(APPEND PROJECT_DEFINITIONS "INSTALL_SEMANTICS_DIR=\"${CMAKE_INSTALL_PREFIX}share/remill/semantics/\"") list(APPEND PROJECT_DEFINITIONS "BUILD_SEMANTICS_DIR=\"${CMAKE_CURRENT_BINARY_DIR}/remill/Arch/X86/Runtime/\"") +list(APPEND PROJECT_DEFINITIONS "BUILD_SEMANTICS_DIR_ARM=\"${CMAKE_CURRENT_BINARY_DIR}/remill/Arch/AArch64/Runtime/\"") add_library(${PROJECT_NAME} STATIC + + remill/Arch/AArch64/Arch.cpp + remill/Arch/AArch64/Decode.cpp + remill/Arch/AArch64/Extract.cpp remill/Arch/X86/Arch.cpp + remill/Arch/Arch.cpp remill/Arch/Instruction.cpp remill/Arch/Name.cpp @@ -186,13 +192,13 @@ install(DIRECTORY DESTINATION "share/remill") # add_custom_target(semantics) + +# runtimes add_subdirectory(remill/Arch/X86/Runtime) +add_subdirectory(remill/Arch/AArch64/Runtime) -# mcsema needs to be manually cloned into this repo. -if (EXISTS ${CMAKE_SOURCE_DIR}/tools/mcsema) - add_subdirectory(tools/mcsema) -endif () - +# tools +add_subdirectory(tools) # only enable tests when compiling under x64 if (CMAKE_SIZEOF_VOID_P EQUAL 8) diff --git a/README.md b/README.md index eb9a77d3..c03b4447 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Remill -Remill is a static binary translator that translates machine code into [LLVM bitcode](http://llvm.org/docs/LangRef.html). It translates x86 and amd64 machine code (including AVX and AVX512) into LLVM bitcode. +Remill is a static binary translator that translates machine code instructions into [LLVM bitcode](http://llvm.org/docs/LangRef.html). It translates x86 and amd64 machine code (including AVX and AVX512) into LLVM bitcode. AArch64 support is underway. Remill focuses on accurately lifting instructions. It is meant to be used as a library for other tools, e.g. [McSema](https://github.com/trailofbits/mcsema). @@ -9,7 +9,6 @@ Remill focuses on accurately lifting instructions. It is meant to be used as a l | | master | | ----- | ------ | | Linux | [![Build Status](https://travis-ci-job-status.herokuapp.com/badge/trailofbits/remill/master/linux/)](https://travis-ci.org/trailofbits/remill) | -| macOS | [![Build Status](https://travis-ci-job-status.herokuapp.com/badge/trailofbits/remill/master/osx/)](https://travis-ci.org/trailofbits/remill) | ## Additional Documentation @@ -30,6 +29,8 @@ We are actively working on porting Remill to macOS. ## Dependencies +Most of Remill's dependencies can be provided by the [cxx-common](https://github.com/trailofbits/cxx-common) repository. Trail of Bits hosts downloadable, pre-built versions of cxx-common, which makes it substantially easier to get up and running with Remill. Nonetheless, the following table represents most of Remill's dependencies. + | Name | Version | | ---- | ------- | | [Git](https://git-scm.com/) | Latest | @@ -45,11 +46,9 @@ We are actively working on porting Remill to macOS. ## Getting and Building the Code -### Step 1: Install dependencies +### On Linux -#### On Linux - -##### Install Dependencies +First, update aptitude and get install the baseline dependencies. ```shell sudo apt-get update @@ -57,70 +56,41 @@ sudo apt-get upgrade sudo apt-get install \ git \ - cmake \ - python2.7 python-pip \ + python-pip \ build-essential \ - unzip \ - software-properties-common \ realpath ``` -##### Upgrade CMake (Ubuntu 14.04) - -Users wishing to run Remill on Ubuntu 14.04 should upgrade their version of CMake. +Next, clone the repository. This will clone the code into the `remill` directory. ```shell -sudo add-apt-repository -y ppa:george-edison55/cmake-3.x -sudo apt-get update -sudo apt-get upgrade -sudo apt-get install cmake +git clone git@github.com:trailofbits/remill.git ``` -#### On OS X - -##### Install Dependencies - -``` -brew install glog -``` - -### Step 2: Clone and Enter the Repository - -#### Clone the repository -```shell -This will also clone the cxx_common module (used to generate a library repository) and mcsema. -git clone --resursive git@github.com:trailofbits/remill.git -``` - -#### Enter the repository -```shell -cd remill -``` - -#### Build the code. -```shell -./build.sh -``` - -## Building and Running the Test Suite - -### Build Google Test - -#### On Linux - -This script will build and install the Google Test framework. It will request administrator permissions. +Next, we build Remill. This script will create another directory, `remill-build` +at the same level as the `remill` directory. All remaining dependencies needed +by Remill will be built in the `remill-build` directory. ```shell -./scripts/unix/install_gtest.sh +./remill/scripts/build.sh ``` -### Generate and Run the Test Cases +Next, we can install Remill. Remill itself is a library, and so there is no real way +to try it. However, you can head on over to the [McSema](https://github.com/trailofbits/mcsema) repository, which uses Remill for lifting instructions. ```shell -sudo make build_x86_tests -ctest +cd ./remill-build +sudo make install ``` -## Try it Out +We can also build and run Remill's test suite. + +```shell +cd ./remill-build +make build_x86_tests +./cmake-3.2.0-Linux-x86_64/bin/ctest +``` + + + -Remill is a library, and so there is no single way to try it. However, you can head on over to the [McSema](https://github.com/trailofbits/mcsema) repository and try that! diff --git a/cmake/BCCompiler/CMakeBCCompiler.cmake.in b/cmake/BCCompiler/CMakeBCCompiler.cmake.in index 58d71ec2..c3b84661 100644 --- a/cmake/BCCompiler/CMakeBCCompiler.cmake.in +++ b/cmake/BCCompiler/CMakeBCCompiler.cmake.in @@ -1,3 +1,17 @@ +# Copyright (c) 2017 Trail of Bits, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + set(CMAKE_BC_COMPILER "@CMAKE_BC_COMPILER@") set(CMAKE_BC_LINKER "@CMAKE_BC_LINKER@") diff --git a/cmake/BCCompiler/CMakeBCInformation.cmake b/cmake/BCCompiler/CMakeBCInformation.cmake index fdffa7c9..a6945263 100644 --- a/cmake/BCCompiler/CMakeBCInformation.cmake +++ b/cmake/BCCompiler/CMakeBCInformation.cmake @@ -1,11 +1,26 @@ -set(DEFAULT_BC_COMPILER_FLAGS "-std=gnu++11 -emit-llvm -Wno-unknown-warning-option -Werror -Wall -Wshadow -Wconversion -Wpadded -pedantic -Wshorten-64-to-32 -Wno-gnu-anonymous-struct -Wno-return-type-c-linkage -Wno-gnu-zero-variadic-macro-arguments -Wno-nested-anon-types -Wno-extended-offsetof -Wno-gnu-statement-expression -Wno-c99-extensions -Wno-ignored-attributes -mtune=generic -fno-vectorize -fno-slp-vectorize -ffreestanding -fno-common -fno-builtin -fno-exceptions -fno-rtti -fno-asynchronous-unwind-tables -Wno-unneeded-internal-declaration -Wno-unused-function") +# Copyright (c) 2017 Trail of Bits, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set(DEFAULT_BC_COMPILER_FLAGS "-std=gnu++11 -emit-llvm -Wno-unknown-warning-option -Wall -Wshadow -Wconversion -Wpadded -pedantic -Wshorten-64-to-32 -Wno-gnu-anonymous-struct -Wno-return-type-c-linkage -Wno-gnu-zero-variadic-macro-arguments -Wno-nested-anon-types -Wno-extended-offsetof -Wno-gnu-statement-expression -Wno-c99-extensions -Wno-ignored-attributes -mtune=generic -fno-vectorize -fno-slp-vectorize -ffreestanding -fno-common -fno-builtin -fno-exceptions -fno-rtti -fno-asynchronous-unwind-tables -Wno-unneeded-internal-declaration -Wno-unused-function") if (NOT CMAKE_BC_COMPILE_OBJECT) if (NOT DEFINED CMAKE_BC_COMPILER) message(SEND_ERROR "The bitcode compiler was not found!") endif () - set(CMAKE_BC_COMPILE_OBJECT "${CMAKE_BC_COMPILER} ${DEFAULT_BC_COMPILER_FLAGS} -c -o ") + # TODO(pag): Removed because it caused warnings. + set(CMAKE_BC_COMPILE_OBJECT "${CMAKE_BC_COMPILER} ${DEFAULT_BC_COMPILER_FLAGS} -c -o ") endif () if (NOT CMAKE_BC_LINK_EXECUTABLE) diff --git a/cmake/BCCompiler/CMakeDetermineBCCompiler.cmake b/cmake/BCCompiler/CMakeDetermineBCCompiler.cmake index 6d73dac2..c9a086ac 100644 --- a/cmake/BCCompiler/CMakeDetermineBCCompiler.cmake +++ b/cmake/BCCompiler/CMakeDetermineBCCompiler.cmake @@ -1,3 +1,17 @@ +# Copyright (c) 2017 Trail of Bits, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + if (NOT DEFINED CMAKE_BC_COMPILER) if (NOT "$ENV{LLVM_INSTALL_PREFIX}" STREQUAL "") message(STATUS "Setting LLVM_INSTALL_PREFIX from the environment variable...") diff --git a/cmake/BCCompiler/CMakeTestBCCompiler.cmake b/cmake/BCCompiler/CMakeTestBCCompiler.cmake index 0154ae26..91c144a3 100644 --- a/cmake/BCCompiler/CMakeTestBCCompiler.cmake +++ b/cmake/BCCompiler/CMakeTestBCCompiler.cmake @@ -1 +1,15 @@ +# Copyright (c) 2017 Trail of Bits, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + set(CMAKE_BC_COMPILER_WORKS 1 CACHE INTERNAL "") diff --git a/distro_packages/ArchLinux/.gitignore b/distro_packages/ArchLinux/.gitignore index 3e931029..72ee4aea 100644 --- a/distro_packages/ArchLinux/.gitignore +++ b/distro_packages/ArchLinux/.gitignore @@ -1,3 +1,17 @@ +# Copyright (c) 2017 Trail of Bits, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + pkg remill src diff --git a/docs/LIFE_OF_AN_INSTRUCTION.md b/docs/LIFE_OF_AN_INSTRUCTION.md index e25930fc..21a7bd41 100644 --- a/docs/LIFE_OF_AN_INSTRUCTION.md +++ b/docs/LIFE_OF_AN_INSTRUCTION.md @@ -8,74 +8,63 @@ This document will use the instructions in the following basic block as a runnin ![Sample basic block](images/instruction_life_block.png) -## Step 1: CFG protocol buffer representation +## Decoding instructions -The first step to lifting involves getting the machine code instructions of some binary file into a format the Remill understands. We use a common and simple format so that Remill doesn't need to understand the many executable file formats ([ELF](https://en.wikipedia.org/wiki/Executable_and_Linkable_Format), [Mach-O](https://en.wikipedia.org/wiki/Mach-O), [PE](https://en.wikipedia.org/wiki/Portable_Executable), etc.). +The first step to lifting is to decode the bytes of an instruction. This decoding step takes raw instruction bytes, and turns them into a higher-level [`Instruction`](/remill/Arch/Instruction.h) data structure. This data structure represents the logical operands to the machine code instructions. These operands have a one-to-one correspondence with arguments that will be passed to semantics functions. -The file format used by Remill is a [CFG protocol buffer](CFG_FORMAT.md). This file is produced using plug-in scripts for [Binary Ninja](https://binary.ninja) and [IDA Pro](https://www.hex-rays.com/products/ida). The CFG file for the above code block contains the following data. +Below is a string representation of the data structures representing our example assembly. -```protobuf -Module { - blocks = [ - Block { - address = 0x804b7a3; - is_addressable = false; - instructions = [ +```lisp +;; mov eax, 1 +(X86_INSTR 8048098 5 MOV_GPRv_IMMv_32 + (WRITE_OP (REG_32 EAX)) + (READ_OP (IMM_32 1))) - // mov eax, 0x1 - Instr { - bytes = "\xb8\x01\x00\x00\x00"; - address = 0x804b7a3; - }, +;; push ebx +(X86_INSTR 804809d 1 PUSH_GPRv_50_32 + (READ_OP (REG_32 EBX))) - // push ebx - Instr { - bytes = "\x53"; - address = 0x0804b7a8; - }, +;; mov ebx, dword ptr [esp + 8] +(X86_INSTR 804809e 4 MOV_GPRv_MEMv_32 + (WRITE_OP (REG_32 EBX)) + (READ_OP (ADDR_32 DWORD (SEGMENT SS_BASE) ESP + 0x8)))) - // mov ebx, dword [esp+0x8] - Instr { - bytes = "\x8b\x5c\x24\x08"; - address = 0x0804b7a9; - }, - - // int 0x80 - Instr { - bytes = "\xcd\x80"; - address = 0x0804b7ad; - } ] - } ] - named_blocks = [ ]; - referenced_blocks = [ ]; -} +;; int 0x80 +(X86_INSTR 80480a2 2 INT_IMMb + (READ_OP (IMM_8 80))) ``` -## Step 2: Creating LLVM functions for each `Block` +## From architecture-specific to architecture-neutral -Remill's unit of translation is a [basic block](https://en.wikipedia.org/wiki/Basic_block). Whereas [McSema](https://github.com/trailofbits/mcsema) translates a function in a binary into an LLVM bitcode function, Remill translates each basic block of code into an LLVM bitcode function. +Decoded instructions must be lifted into a compatible function. Compatible functions are clones of the [`__remill_basic_block`](/remill/Arch/X86/Runtime/BasicBlock.cpp) function. The `__remill_basic_block` function is special because it defines local variables that "point into" the [`State`](/remill/Arch/X86/Runtime/State.h)) structure, which represents the machine's register state. -The translator creates an LLVM function for every `Block` in the CFG protocol buffer's `Module` message. In this example, Remill will create the function `__remill_sub_804b7a3`. This function will be a copy of the [`__remill_basic_block`](https://github.com/trailofbits/remill/blob/master/remill/Arch/X86/Runtime/BasicBlock.cpp) function. What this means is that all of the code and variable definitions within `__remill_basic_block` will also be in `__remill_sub_804b7a3`. - -### The `__remill_basic_block` function - -The `__remill_basic_block` function is very important. There are *a lot* of variables defined in this function, and the comments are mildy useful. For now, we will focus on the variable definitions that relate to machine code registers. Specifically, we will look at how the translator maps `ebx` from `push ebx` into `EBX`. +The following is an example of the `__remill_basic_block` function for X86. ```C++ -// Method that will implement a basic block. We will clone this method for -// each basic block in the code being lifted. -[[gnu::used]] -void __remill_basic_block(State &state, Memory &memory, addr_t curr_pc) { +// Instructions will be lifted into clones of this function. +Memory *__remill_basic_block(Memory *memory, State &state, addr_t curr_pc) { + ... + auto &EAX = state.gpr.rax.dword; - auto &EBX = state.gpr.rbx.dword; + auto &EBX = state.gpr.rbx.dword;\ + auto &ESP = state.gpr.rsp.dword; + ... + + auto &SS_BASE = zero; + + ... + + // Lifted code will be placed here in clones versions of this function. + return memory; } ``` -Remill is designed to be able to translate arbitrary machine code to LLVM bitcode. As of this writing, only the x86 and x64 architectures are supported. Nevertheless, there needs to be a kind of "common language" that the architecture-neutral LLVM side of things, and the architecture-specific semantics functions and machine instruction decoders can use to negotiate the translation process. This common language is variable names within the `__remill_basic_block` function. +In the case of the `push ebx` instruction from our example block, our decoder understands that `ebx` is a register. Surprisingly, the lifting side of Remill has no concept of what `ebx` is! Remill is designed to be able to translate arbitrary machine code to LLVM bitcode. To that end, there needs to be a kind of "common language" that the architecture-neutral LLVM side of things, and the architecture-specific semantics functions and machine instruction decoders can use to negotiate the translation process. This common language is variable names within the `__remill_basic_block` function. The instruction decoder ensures that decoded register names correspond to variables defined in `__remill_basic_block`. The programmer implementing `__remill_basic_block` ensures the same things. The conversion from `Instruction` data structures to LLVM bitcode _assumes_ that this correspondence exists. -Ideally, we don't want the LLVM translation side to understand machine registers. And yet, if you take a look at the [`Instruction`](/remill/Arch/Instruction.h) data structure, then you will see something like the following: + +Let's hammer this home. If we scroll up, we see that the `Instruction` data structure corresponding to `push ebx` has `EBX` in one of its `(READ_OP (REG_32 EBX))` register operands. This operand corresponds to the following `Register` data structure. ```C++ class Register { @@ -86,84 +75,23 @@ class Register { }; ``` -If we look deep into the bowels of [`DecodeRegister`](/remill/Arch/X86/Arch.cpp#L406), the function that pulls out architecture-specific register operand information from [Intel's XED](https://software.intel.com/en-us/articles/xed-x86-encoder-decoder-software-library), we find that that a register operand can be built as follows: +The decoder initialized the `name` field with `"EBX"`, and the lifter can look up the variable name `EBX` in any cloned copies of `__remill_basic_block`. + +## What does lifted code look like? + +In spirit, the lifted code for the instructions in our running example looks like the following C++ code. ```C++ - if (xed_operand_read(xedo)) { - ... - op.reg = ReadOp(reg); // Returns `EBX`. - instr->operands.push_back(op); - } -``` - -In the case of the `push ebx` instruction from our example block, `ReadOp` will actually return the string `"EBX"`. Recall that each basic block of machine code is lifted into an LLVM function, and that function is a clone of `__remill_basic_block`. This means that the LLVM function implementing the basic block will have `EBX` defined. So, the translator doesn't actually know what `ebx` is, it just understands variable names, and the instruction decoder will produce variable names for registers, knowing that those variables are defined in `__remill_basic_block`, and therefore will be available within all of its clones. - -## Step 3: Calling semantics functions for each `Instr` - -The previous section described how the architecture-specific instruction decoder can communicate things like registers to the higher-level translation system. The decoder tells the translator to look for specific variables representing register values, knowing that those variables will be available to each block function. - -The "[How to add a new instruction](ADD_AN_INSTRUCTION.md)" document describes the different between *SEM*s, functions implementing instruction semantics, and *ISEL*s, specialisations of those functions that are specific to an instruction form or encoding. The instruction decoder knows about the names of *ISEL*s, and it uses these names to [direct the translator](/remill/Arch/Instruction.h#L79-L80) to find and call the right function for each instruction being lifted. - -The [operands](/remill/Arch/Instruction.h#L123) of the instruction data structure tell the translator how to find and compute values to pass as arguments to the semantics functions. In the last section, we showed how the `ebx` register in `push ebx` is represented as a register operand, whose variable name is `EBX`. The translator loads this value and passes it as an argument to the [`PUSH_GPRv_32`](/remill/Arch/X86/Semantics/PUSH.cpp#L28) *ISEL* function, as implemented by the [`PUSH`](/remill/Arch/X86/Semantics/PUSH.cpp#L17-L20) *SEM* function. - -Here's what the unoptimised bitcode for the example basic block looks like: - -```llvm -; Function Attrs: alwaysinline inlinehint nounwind -define private fastcc void @__remill_sub_804b7a3(%struct.State* dereferenceable(3200) %state, %struct.Memory* nonnull %memory, i32 %pc) #7 { - %MEMORY = alloca %struct.Memory*, align 4 - store %struct.Memory* %memory, %struct.Memory** %MEMORY, align 4 - %1 = getelementptr inbounds %struct.State, %struct.State* %state, i32 0, i32 5, i32 33 - %2 = bitcast %union.Flags* %1 to i32* - %3 = getelementptr inbounds %struct.State, %struct.State* %state, i32 0, i32 5, i32 1 - %4 = bitcast %union.Flags* %3 to i32* - %5 = getelementptr inbounds %struct.State, %struct.State* %state, i32 0, i32 5, i32 3 - %6 = bitcast %union.Flags* %5 to i32* - %7 = getelementptr inbounds %struct.State, %struct.State* %state, i32 0, i32 5, i32 3 - %8 = bitcast %union.Flags* %7 to i32* - %9 = getelementptr inbounds %struct.State, %struct.State* %state, i32 0, i32 5, i32 13 - %10 = bitcast %union.Flags* %9 to i32* - %11 = getelementptr inbounds %struct.State, %struct.State* %state, i32 0, i32 4, i32 1 - %12 = add i32 %pc, 5 - store i32 %12, i32* %2, align 4 - call void @void (anonymous namespace)::MOV, In >(State&, Memory*&, RnW, In)(%struct.State* nonnull %state, %struct.Memory** nonnull %MEMORY, i32* %4, i32 1) - %13 = load i32, i32* %2, align 4 - %14 = add i32 %13, 1 - store i32 %14, i32* %2, align 4 - %15 = load i32, i32* %6, align 4 - call void @void (anonymous namespace)::PUSH(State&, Memory*&, unsigned int)(%struct.State* nonnull %state, %struct.Memory** nonnull %MEMORY, i32 %15) - %16 = load i32, i32* %2, align 4 - %17 = add i32 %16, 4 - store i32 %17, i32* %2, align 4 - %18 = load i32, i32* %10, align 4 - %19 = load i16, i16* %11, align 2 - %20 = zext i16 %19 to i32 - %21 = add i32 %18, 8 - %22 = call i32 @__remill_compute_address(i32 %21, i32 %20) - call void @void (anonymous namespace)::MOV, Mn >(State&, Memory*&, RnW, Mn)(%struct.State* nonnull %state, %struct.Memory** nonnull %MEMORY, i32* %8, i32 %22) - %23 = load i32, i32* %2, align 4 - %24 = add i32 %23, 2 - store i32 %24, i32* %2, align 4 - call void @INT_IMMb(%struct.State* nonnull %state, %struct.Memory** nonnull %MEMORY, i32 128) - %25 = load i32, i32* %2, align 4 - %26 = load %struct.Memory*, %struct.Memory** %MEMORY, align 4 - musttail call fastcc void @__remill_interrupt_call(%struct.State* nonnull %state, %struct.Memory* %26, i32 %25) - ret void -} -``` - -Yikes! That is a lot of code. Here's what it would look like if it was written directly in C++: - -```C++ -void __remill_sub_804b7a3(State *state, Memory *memory, addr_t pc) { +void __remill_sub_804b7a3(Memory *memory, State *state, addr_t pc) { auto &EIP = state.gpr.rip.dword; auto &EAX = state.gpr.rax.dword; auto &EBX = state.gpr.rbx.dword; auto &ESP = state.gpr.rsp.dword; - // mov eax, 0x1 - pc += 5; EIP = pc; + + // mov eax, 0x1 + EIP += 5; MOV(state, &memory, &EAX, 1); // push ebx @@ -178,69 +106,59 @@ void __remill_sub_804b7a3(State *state, Memory *memory, addr_t pc) { EIP += 2; INT_IMMb(state, &memory, 0x80); - return __remill_interrupt_call(state, memory, EIP) + return __remill_interrupt_call(memory, state, EIP) } ``` -This is somewhat better. We can see that some of the mechanics of instructions happen outside of the instruction semantics function calls. Specifically, we see the program counter incremented before each semantics function call. +Earlier this documented mentioned that there is a one-to-one correspondence between operands in the `Instruction` data structure and arguments to semantics functions. We can see that with the `MOV` instruction. -There `State *state` parameter to `__remill_sub_804b7a3` holds all [machine register state](/remill/Arch/X86/Runtime/State.h). The `Memory *memory` parameter is curious. It is passed to each semantics function by pointer, implying that it is updated within. The `Memory` type is never actually defined, but it is crucial to Remill's [memory model](https://en.wikipedia.org/wiki/Memory_model_(programming)) and enforcing [memory ordering](https://en.wikipedia.org/wiki/Memory_ordering). We can see more of this when we apply and [inlining](https://en.wikipedia.org/wiki/Inline_expansion) optimisation. +The data structure of `mov ebx, dword [esp+0x8]` was: -## Step 4: Optimizing the bitcode +```lisp +;; mov ebx, dword ptr [esp + 8] +(X86_INSTR 804809e 4 MOV_GPRv_MEMv_32 + (WRITE_OP (REG_32 EBX)) + (READ_OP (ADDR_32 DWORD (SEGMENT SS_BASE) ESP + 0x8)))) +``` -```llvm -; Function Attrs: alwaysinline inlinehint nounwind -define private fastcc void @__remill_sub_804b7a3(%struct.State* dereferenceable(3200) %state, %struct.Memory* nonnull %memory, i32 %pc) #8 { - %1 = getelementptr inbounds %struct.State, %struct.State* %state, i64 0, i32 5, i32 33 - %2 = bitcast %union.Flags* %1 to i32* - %3 = getelementptr inbounds %struct.State, %struct.State* %state, i64 0, i32 5, i32 1 - %4 = bitcast %union.Flags* %3 to i32* - %5 = getelementptr inbounds %struct.State, %struct.State* %state, i64 0, i32 5, i32 3 - %6 = bitcast %union.Flags* %5 to i32* - %7 = getelementptr inbounds %struct.State, %struct.State* %state, i64 0, i32 5, i32 13 - %8 = bitcast %union.Flags* %7 to i32* - %9 = getelementptr inbounds %struct.State, %struct.State* %state, i64 0, i32 4, i32 1 - store i32 1, i32* %4, align 4 - %10 = load i32, i32* %6, align 4 - %11 = load i32, i32* %8, align 8 - %12 = add i32 %11, -4 - %13 = load i16, i16* %9, align 2 - %14 = zext i16 %13 to i32 - %15 = tail call i32 @__remill_compute_address(i32 %12, i32 %14) #15 - %16 = tail call %struct.Memory* @__remill_write_memory_32(%struct.Memory* nonnull %memory, i32 %15, i32 %10) #15 - store i32 %12, i32* %8, align 4 - %17 = add i32 %11, 4 - %18 = tail call i32 @__remill_compute_address(i32 %17, i32 %14) - %19 = tail call i32 @__remill_read_memory_32(%struct.Memory* %16, i32 %18) #15 - store i32 %19, i32* %6, align 4 - %20 = add i32 %pc, 12 - store i32 %20, i32* %2, align 4 - %21 = getelementptr inbounds %struct.State, %struct.State* %state, i64 0, i32 9 - store volatile i32 128, i32* %21, align 4 - %22 = getelementptr inbounds %struct.State, %struct.State* %state, i64 0, i32 12 - store volatile i8 1, i8* %22, align 2 - %23 = load i32, i32* %2, align 4 - musttail call fastcc void @__remill_interrupt_call(%struct.State* nonnull %state, %struct.Memory* %16, i32 %23) - ret void +The semantics function implementing the `mov` instruction is: + +```c++ +template +DEF_SEM(MOV, D dst, const S src) { + WriteZExt(dst, Read(src)); + return memory; } ``` -After inlining, the bitcode starts to show off its memory model. Again, lets see what things would look like if we translated this code into C++. +The `(REG_32 EBX)` corresponds with the `dst` parameter, and the `(ADDR_32 DWORD (SEGMENT SS_BASE) ESP + 0x8))` corresponds with the `src` paremter. What links the `Instruction` data structure and the `MOV` semantics function together is the weird `MOV_GPRv_MEMv_32` name that we see. This name is stored in the `Instruction::function` field. The lifting code "discovers" the `MOV` semantics function via this name, and that association is made using the following special syntax. + +```c++ +DEF_ISEL(MOV_GPRv_MEMv_32) = MOV; +``` + +You can head on over to the [how to add an instruction](ADD_AN_INSTRUCTION.md) document to better understand the meaning of `DEF_SEM` and `DEF_ISEL`. + + +## We must go deeper + +The spiritual lifted code makes one function call per lifted instruction, where the actual implementation of each function can be arbitrarily complex. If we optimize the bitcode that Remill produces for our few example instructions, then what we get, if translated back to C++, looks like the following. + ```C++ -void __remill_sub_804b7a3(State *state, Memory *memory, addr_t pc) { +void __remill_sub_804b7a3(Memory *memory, State *state, addr_t pc) { auto &EIP = state.gpr.rip.dword; auto &EAX = state.gpr.rax.dword; auto &EBX = state.gpr.rbx.dword; auto &ESP = state.gpr.rsp.dword; - auto &SS = state.seg.ss; + auto &SS_BASE = 0; // mov eax, 0x1 EAX = 1; // push ebx ESP -= 4; - addr_t push_addr = __remill_compute_address(ESP, SS); + addr_t push_addr = ESP + SS_BASE; memory = __remill_write_memory_32(memory, push_addr, EBX); // mov ebx, dword [esp+0x8] @@ -248,39 +166,36 @@ void __remill_sub_804b7a3(State *state, Memory *memory, addr_t pc) { EBX = __remill_read_memory_32(memory, read_addr); // int 0x80 - state.interrupt_vector = 0x80; // This is a wart. See Issue #53. + state.hyper_call = AsyncHyperCall::kX86IntN; + state.interrupt_vector = 0x80; EIP = pc + 12; - return __remill_interrupt_call(state, memory, EIP) + return __remill_async_hyper_call(state, memory, EIP) } ``` -### The Remill runtime exposed +## The Remill runtime exposed In the case of Remill, we want to be explicit about accesses to the "modelled program's memory", and to the "runtime's memory". Take another look above at the optimized bitcode. We can see many `load` and `store` instructions. These instruction's are LLVM's way of representing memory reads and writes. Conceptually, the `State` structure is not part of a program's memory -- if you ran the program natively, then our `State` structure would not be present. In Remill, we consider the "runtime" to be made of the following: 1. Memory accesses to `alloca`'d space: local variables needed to support more elaborate computations. 2. Memory accesses to the `State` structure. The `State` structure is passed by pointer, so it must be accessed via `load` and `store` instructions. -3. All of the various intrinsics, e.g. `__remill_interrupt_call`, `__remill_read_memory_32`, etc. +3. All of the various intrinsics, e.g. `__remill_async_hyper_call`, `__remill_read_memory_32`, etc. The naming of "runtime" does not mean that the bitcode itself needs to be executed, nor does it mean that the intrinsics must be implemented in any specific way. Remill's intrinsics provide semantic value (in terms of their naming). A user of Remill bitcode can do anything they want with these intrinsics. For example, they could convert all of the memory intrinsics into LLVM `load` and `store` instructions. Anyway, back to the memory model and memory ordering. Notice that the `memory` pointer is passed into every memory access intrinsic. The `memory` pointer is also replaced in the case of memory write intrinsics. This is to enforce a total order across all memory writes, and a partial order between memory reads and writes. -The `__remill_interrupt_call` and `__remill_compute_address` intrinsics are examples of lower level abstractions that cannot be directly modelled by Remill. - -The `__remill_compute_address` is used to model [memory segmentation](https://en.wikipedia.org/wiki/Memory_segmentation#x86_architecture). In x86 (32-bit) code, segmentation is widespread -- this intrinsic rarely shows up in x64 code. Memory segmentation is supported by the [local](https://en.wikipedia.org/wiki/Global_Descriptor_Table#Local_Descriptor_Table) and [global](https://en.wikipedia.org/wiki/Global_Descriptor_Table) descriptor tables. - -The `__remill_interrupt_call` instruction instructs the "runtime" that an explicit [interrupt](https://en.wikipedia.org/wiki/Interrupt) (`int 0x80`) happens. Again, Remill has no way of knowing what this actually means or how it works -- that falls under the purview of the operating system kernel. What Remill *does* know is that an interrupt is a kind of ["indirect" control flow](https://en.wikipedia.org/wiki/Indirect_branch), and so it models is like all other indirect control flows. +The `__remill_async_hyper_call` instruction instructs the "runtime" that an explicit [interrupt](https://en.wikipedia.org/wiki/Interrupt) (`int 0x80`) happens. Again, Remill has no way of knowing what this actually means or how it works -- that falls under the purview of the operating system kernel. What Remill *does* know is that an interrupt is a kind of ["indirect" control flow](https://en.wikipedia.org/wiki/Indirect_branch), and so it models is like all other indirect control flows. All Remill control-flow intrinsics and Remill lifted basic block functions share the same argument structure: -1. A pointer to the `State` structure. -2. A pointer to the opaque `Memory` structure. +1. A pointer to the opaque `Memory` structure. +2. A pointer to the `State` structure. 3. The program counter on entry to the lifted basic block. -In the case of the `__remill_interrupt_call`, the third argument, the program counter address, is computed to be the address following the `int 0x80` instruction. +In the case of the `__remill_async_hyper_call`, the third argument, the program counter address, is computed to be the address following the `int 0x80` instruction. ## Concluding remarks diff --git a/remill/Arch/AArch64/Arch.cpp b/remill/Arch/AArch64/Arch.cpp new file mode 100644 index 00000000..929b142c --- /dev/null +++ b/remill/Arch/AArch64/Arch.cpp @@ -0,0 +1,672 @@ +/* + * Copyright (c) 2017 Trail of Bits, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "remill/Arch/AArch64/Decode.h" +#include "remill/Arch/Arch.h" +#include "remill/Arch/Instruction.h" +#include "remill/Arch/Name.h" +#include "remill/BC/Version.h" +#include "remill/OS/OS.h" + +namespace remill { +namespace { + +//template +//static inline uint32_t Extract(uint32_t inst) { +// auto num_bits = (from_high - to_low) + 1U; +// return (inst >> to_low) & ((1U << num_bits) - 1U); +//} + +template +static inline T Select(T val) { + return (val >> bit) & T(1); +} + +Instruction::Category InstCategory(const aarch64::InstData &inst) { + switch (inst.iclass) { + case aarch64::InstName::INVALID: + return Instruction::kCategoryInvalid; + + // TODO(pag): B.cond. + case aarch64::InstName::B: + if (aarch64::InstForm::B_ONLY_CONDBRANCH == inst.iform) { + return Instruction::kCategoryConditionalBranch; + } else { + return Instruction::kCategoryDirectJump; + } + + case aarch64::InstName::BR: + return Instruction::kCategoryIndirectJump; + + case aarch64::InstName::CBZ: + case aarch64::InstName::CBNZ: + case aarch64::InstName::TBZ: + case aarch64::InstName::TBNZ: + return Instruction::kCategoryConditionalBranch; + + case aarch64::InstName::BL: + return Instruction::kCategoryDirectFunctionCall; + + case aarch64::InstName::BLR: + return Instruction::kCategoryIndirectFunctionCall; + + case aarch64::InstName::RET: + return Instruction::kCategoryFunctionReturn; + + case aarch64::InstName::HLT: + return Instruction::kCategoryError; + + case aarch64::InstName::HVC: + case aarch64::InstName::SMC: + case aarch64::InstName::SVC: + case aarch64::InstName::SYS: // Has aliases `IC`, `DC`, `AT`, and `TLBI`. + case aarch64::InstName::SYSL: + return Instruction::kCategoryAsyncHyperCall; + + case aarch64::InstName::NOP: + return Instruction::kCategoryNoOp; + + // Note: These are implemented with synchronous hyper calls. + case aarch64::InstName::BRK: + return Instruction::kCategoryNormal; + + default: + return Instruction::kCategoryNormal; + } +} + +class AArch64Arch : public Arch { + public: + AArch64Arch(OSName os_name_, ArchName arch_name_); + + virtual ~AArch64Arch(void); + + void PrepareModule(llvm::Module *mod) const override; + + // Decode an instruction. + bool DecodeInstruction( + uint64_t address, const std::string &instr_bytes, + Instruction &inst) const override; + + private: + AArch64Arch(void) = delete; +}; + +AArch64Arch::AArch64Arch(OSName os_name_, ArchName arch_name_) + : Arch(os_name_, arch_name_) {} + +AArch64Arch::~AArch64Arch(void) {} + +void AArch64Arch::PrepareModule(llvm::Module *mod) const { + std::string dl; + llvm::Triple triple("aarch64-unknown-unknown-"); + + switch (os_name) { + case kOSLinux: + triple.setOS(llvm::Triple::Linux); + + switch (arch_name) { + case kArchAArch64LittleEndian: + triple.setArch(llvm::Triple::aarch64); + dl = "e-m:e-i64:64-i128:128-n32:64-S128"; + break; + + default: + LOG(FATAL) + << "Cannot prepare AArch64 module for architecture " + << GetArchName(arch_name); + break; + } + break; + + default: + LOG(FATAL) + << "Cannot prepare module for AArch64 code on OS " + << GetOSName(os_name); + break; + } + + mod->setDataLayout(dl); + mod->setTargetTriple(triple.normalize()); + + // Go and remove compile-time attributes added into the semantics. These + // can screw up later compilation. We purposefully compile semantics with + // things like auto-vectorization disabled so that it keeps the bitcode + // to a simpler subset of the available LLVM instruction set. If/when we + // compile this bitcode back into machine code, we may want to use those + // features, and clang will complain if we try to do so if these metadata + // remain present. + auto &context = mod->getContext(); + + llvm::AttributeSet target_attribs; + target_attribs = target_attribs.addAttribute( + context, llvm::AttributeSet::FunctionIndex, "target-features"); + target_attribs = target_attribs.addAttribute( + context, llvm::AttributeSet::FunctionIndex, "target-cpu"); + + for (llvm::Function &func : *mod) { + auto attribs = func.getAttributes(); + attribs = attribs.removeAttributes( + context, llvm::AttributeSet::FunctionIndex, target_attribs); + func.setAttributes(attribs); + } +} + +enum RegClass { + kRegX, + kRegW +}; + +static const char * const kPrefixX = "X\0X\0X\0X\0X\0X\0X\0X\0X\0X\0X\0X\0X\0X" + "\0X\0X\0X\0X\0X\0X\0X\0X\0X\0X\0X\0X\0X" + "\0X\0X\0X\0\0\0\0\0"; +static const char * const kPrefixW = "WWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWW"; + +static const char *RegPrefix(RegClass size, uint32_t number) { + return kRegX == size ? &(kPrefixX[number * 2]) : &(kPrefixW[number]); +} + +static const char * const kNumberName[] = { + "0", "1", "2", "3", "4", "5", "6", "7", "8", "9", + "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", + "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", + "LP", "SP" +}; + + +using RegNum = uint8_t; + +static const char *RegNumberName(RegNum number) { + CHECK_LE(number, 31U); + return kNumberName[number]; +} + +static std::string RegName(RegClass rclass, RegNum number) { + std::stringstream ss; + ss << RegPrefix(rclass, number); + ss << RegNumberName(number); + return ss.str(); +} + +static size_t ReadRegSize(RegClass rclass) { + switch (rclass) { + case kRegX: + return 64; + case kRegW: + return 32; + } +} + +static size_t WriteRegSize(RegClass rclass) { + switch (rclass) { + case kRegX: + case kRegW: + return 64; + } +} + +enum Action { + kActionRead, + kActionWrite, + kActionReadWrite +}; + +static Operand::Register Reg(Action action, RegClass rclass, RegNum reg_num) { + Operand::Register reg; + if (kActionWrite == action) { + reg.name = RegName(rclass, reg_num); + reg.size = WriteRegSize(rclass); + } else if (kActionRead == action) { + reg.name = RegName(rclass, reg_num); + reg.size = ReadRegSize(rclass); + } else { + LOG(FATAL) + << "Reg function only takes a simple read or write action."; + } + return reg; +} + +static void AddRegOperand(Instruction &inst, Action action, + RegClass rclass, RegNum reg_num) { + Operand op; + op.type = Operand::kTypeRegister; + + if (kActionWrite == action || kActionReadWrite == action) { + op.reg = Reg(kActionWrite, rclass, reg_num); + op.size = op.reg.size; + op.action = Operand::kActionWrite; + inst.operands.push_back(op); + } + + if (kActionRead == action || kActionReadWrite == action) { + op.reg = Reg(kActionRead, rclass, reg_num); + op.size = op.reg.size; + op.action = Operand::kActionRead; + inst.operands.push_back(op); + } +} + +static void AddNextPC(Instruction &inst) { + Operand op; + op.type = Operand::kTypeAddress; + op.size = 64; + op.addr.address_size = 64; + op.addr.base_reg.name = "PC"; + op.addr.base_reg.size = 64; + op.addr.displacement = 4; + + op.action = Operand::kActionRead; + op.addr.kind = Operand::Address::kAddressCalculation; + inst.operands.push_back(op); +} + +// Base+offset memory operands are equivalent to indexing into an array. +// +// We have something like this: +// [, #] +// +// Which gets is: +// addr = Xn + imm +// ... deref addr and do stuff ... +static void AddBasePlusOffsetMemOp(Instruction &inst, Action action, + uint64_t access_size, + RegNum base_reg, uint64_t disp) { + Operand op; + op.type = Operand::kTypeAddress; + op.size = access_size; + op.addr.address_size = 64; + op.addr.base_reg = Reg(kActionRead, kRegX, base_reg); + op.addr.displacement = disp; + + if (kActionWrite == action || kActionReadWrite == action) { + op.action = Operand::kActionWrite; + op.addr.kind = Operand::Address::kMemoryWrite; + inst.operands.push_back(op); + } + + if (kActionRead == action || kActionReadWrite == action) { + op.action = Operand::kActionRead; + op.addr.kind = Operand::Address::kMemoryRead; + inst.operands.push_back(op); + } +} + +// Pre-index memory operands write back the result of the displaced address +// to the base register. +// +// We have something like this: +// [, #]! +// +// Which gets us: +// addr = Xn + imm +// ... deref addr and do stuff ... +// Xn = addr + imm +// +// So we add in two operands: one that is a register write operand for Xn, +// the other that is the value of (Xn + imm + imm). +static void AddPreIndexMemOp(Instruction &inst, Action action, + uint64_t access_size, + RegNum base_reg, uint64_t disp) { + AddBasePlusOffsetMemOp(inst, action, access_size, base_reg, disp); + auto addr_op = inst.operands[inst.operands.size() - 1]; + + Operand reg_op; + reg_op.type = Operand::kTypeRegister; + reg_op.action = Operand::kActionWrite; + reg_op.reg = Reg(kActionWrite, kRegX, base_reg); + reg_op.size = reg_op.reg.size; + inst.operands.push_back(reg_op); + + addr_op.addr.kind = Operand::Address::kAddressCalculation; + addr_op.addr.address_size = 64; + addr_op.addr.base_reg = Reg(kActionRead, kRegX, base_reg); + addr_op.addr.displacement *= 2; + inst.operands.push_back(addr_op); +} + +// Post-index memory operands write back the result of the displaced address +// to the base register. +// +// We have something like this: +// [], # +// +// Which gets us: +// addr = Xn +// ... deref addr and do stuff ... +// Xn = addr + imm +// +// So we add in two operands: one that is a register write operand for Xn, +// the other that is the value of (Xn + imm). +static void AddPostIndexMemOp(Instruction &inst, Action action, + uint64_t access_size, + RegNum base_reg, uint64_t disp) { + AddBasePlusOffsetMemOp(inst, action, access_size, base_reg, 0); + auto addr_op = inst.operands[inst.operands.size() - 1]; + + Operand reg_op; + reg_op.type = Operand::kTypeRegister; + reg_op.action = Operand::kActionWrite; + reg_op.reg = Reg(kActionWrite, kRegX, base_reg); + reg_op.size = reg_op.reg.size; + inst.operands.push_back(reg_op); + + addr_op.addr.kind = Operand::Address::kAddressCalculation; + addr_op.addr.address_size = 64; + addr_op.addr.base_reg = Reg(kActionRead, kRegX, base_reg); + addr_op.addr.displacement = disp; + inst.operands.push_back(addr_op); +} + +enum { + kInstructionSize = 4 // Number of bytes in an instruction. +}; + +bool AArch64Arch::DecodeInstruction( + uint64_t address, const std::string &inst_bytes, + Instruction &inst) const { + + aarch64::InstData dinst = {}; + auto bytes = reinterpret_cast(inst_bytes.data()); + + inst.arch_name = arch_name; + inst.pc = address; + inst.next_pc = address + kInstructionSize; + + if (kInstructionSize != inst_bytes.size()) { + inst.function = "INVALID_SIZE"; + return false; + + } else if (0 != (address % kInstructionSize)) { + inst.function = "INVALID_UNALIGNED"; + return false; + + } else if (!aarch64::TryExtract(bytes, dinst)) { + inst.function = "INVALID_ENCODING"; + return false; + } + + inst.category = InstCategory(dinst); + inst.function = aarch64::InstFormToString(dinst.iform); + + if (!aarch64::TryDecode(dinst, inst)) { + inst.category = Instruction::kCategoryInvalid; + return false; + } + + return true; +} + +} // namespace + +namespace aarch64 { + +// RET {} +bool TryDecodeRET_64R_BRANCH_REG(const InstData &data, Instruction &inst) { + AddRegOperand(inst, kActionRead, kRegX, data.Rn); + return true; +} + + +// BLR +bool TryDecodeBLR_64_BRANCH_REG(const InstData &data, Instruction &inst) { + AddRegOperand(inst, kActionRead, kRegX, data.Rn); + AddNextPC(inst); + return true; +} + +// STP , , [, #]! +bool TryDecodeSTP_32_LDSTPAIR_PRE(const InstData &data, Instruction &inst) { + AddRegOperand(inst, kActionRead, kRegW, data.Rt); + AddRegOperand(inst, kActionRead, kRegW, data.Rt2); + uint64_t offset = static_cast(data.imm7.simm7); + AddPreIndexMemOp(inst, kActionWrite, 64, data.Rn, offset << 2); + return true; +} + +// STP , , [, #]! +bool TryDecodeSTP_64_LDSTPAIR_PRE(const InstData &data, Instruction &inst) { + AddRegOperand(inst, kActionRead, kRegX, data.Rt); + AddRegOperand(inst, kActionRead, kRegX, data.Rt2); + uint64_t offset = static_cast(data.imm7.simm7); + AddPreIndexMemOp(inst, kActionWrite, 128, data.Rn, offset << 3); + return true; +} + +// STP , , [], # +bool TryDecodeSTP_32_LDSTPAIR_POST(const InstData &data, Instruction &inst) { + AddRegOperand(inst, kActionRead, kRegW, data.Rt); + AddRegOperand(inst, kActionRead, kRegW, data.Rt2); + uint64_t offset = static_cast(data.imm7.simm7); + AddPostIndexMemOp(inst, kActionWrite, 64, data.Rn, offset << 2); + return true; +} + +// STP , , [], # +bool TryDecodeSTP_64_LDSTPAIR_POST(const InstData &data, Instruction &inst) { + AddRegOperand(inst, kActionRead, kRegX, data.Rt); + AddRegOperand(inst, kActionRead, kRegX, data.Rt2); + uint64_t offset = static_cast(data.imm7.simm7); + AddPostIndexMemOp(inst, kActionWrite, 128, data.Rn, offset << 3); + return true; +} + +// LDP , , [], # +bool TryDecodeLDP_32_LDSTPAIR_POST(const InstData &data, Instruction &inst) { + AddRegOperand(inst, kActionWrite, kRegW, data.Rt); + AddRegOperand(inst, kActionWrite, kRegW, data.Rt2); + AddPostIndexMemOp(inst, kActionRead, 64, data.Rn, + static_cast(data.imm7.simm7) << 2); + return true; +} + +// LDP , , [], # +bool TryDecodeLDP_64_LDSTPAIR_POST(const InstData &data, Instruction &inst) { + AddRegOperand(inst, kActionWrite, kRegX, data.Rt); + AddRegOperand(inst, kActionWrite, kRegX, data.Rt2); + AddPostIndexMemOp(inst, kActionRead, 128, data.Rn, + static_cast(data.imm7.simm7) << 3); + return true; +} + +// LDP , , [, #]! +bool TryDecodeLDP_32_LDSTPAIR_PRE(const InstData &data, Instruction &inst) { + AddRegOperand(inst, kActionWrite, kRegW, data.Rt); + AddRegOperand(inst, kActionWrite, kRegW, data.Rt2); + AddPreIndexMemOp(inst, kActionRead, 64, data.Rn, + static_cast(data.imm7.simm7) << 2); + return true; +} + +// LDP , , [, #]! +bool TryDecodeLDP_64_LDSTPAIR_PRE(const InstData &data, Instruction &inst) { + AddRegOperand(inst, kActionWrite, kRegX, data.Rt); + AddRegOperand(inst, kActionWrite, kRegX, data.Rt2); + AddPreIndexMemOp(inst, kActionRead, 128, data.Rn, + static_cast(data.imm7.simm7) << 3); + return true; +} + +// LDP , , [{, #}] +bool TryDecodeLDP_32_LDSTPAIR_OFF(const InstData &data, Instruction &inst) { + AddRegOperand(inst, kActionWrite, kRegW, data.Rt); + AddRegOperand(inst, kActionWrite, kRegW, data.Rt2); + AddBasePlusOffsetMemOp(inst, kActionRead, 64, data.Rn, + static_cast(data.imm7.simm7) << 2); + return true; +} + +// LDP , , [{, #}] +bool TryDecodeLDP_64_LDSTPAIR_OFF(const InstData &data, Instruction &inst) { + AddRegOperand(inst, kActionWrite, kRegX, data.Rt); + AddRegOperand(inst, kActionWrite, kRegX, data.Rt2); + AddBasePlusOffsetMemOp(inst, kActionRead, 128, data.Rn, + static_cast(data.imm7.simm7) << 3); + return true; +} + +// LDR , [{, #}] +bool TryDecodeLDR_32_LDST_POS(const InstData &data, Instruction &inst) { + AddRegOperand(inst, kActionWrite, kRegW, data.Rt); + AddBasePlusOffsetMemOp(inst, kActionRead, 32, data.Rn, + data.imm12.uimm << 2); + return true; +} + +// LDR , [{, #}] +bool TryDecodeLDR_64_LDST_POS(const InstData &data, Instruction &inst) { + AddRegOperand(inst, kActionWrite, kRegX, data.Rt); + AddBasePlusOffsetMemOp(inst, kActionRead, 64, data.Rn, + data.imm12.uimm << 3); + return true; +} + +static void AddPCRegMemOp(Instruction &inst, Action action, uint64_t disp) { + Operand op; + op.type = Operand::kTypeAddress; + op.size = 64; + op.addr.address_size = 64; + op.addr.base_reg.name = "PC"; + op.addr.base_reg.size = 64; + op.addr.displacement = disp; + if (kActionRead == action) { + op.action = Operand::kActionRead; + op.addr.kind = Operand::Address::kMemoryRead; + } else if (kActionWrite == action) { + op.action = Operand::kActionWrite; + op.addr.kind = Operand::Address::kMemoryWrite; + } else { + LOG(FATAL) + << "AddPCRegMemOp only accepts simple operand actions."; + } + inst.operands.push_back(op); +} + +// LDR ,