From 2eaf52ca72d332ee8db3869474e6f66a1c2f4bc7 Mon Sep 17 00:00:00 2001 From: Eric Kilmer Date: Fri, 2 Apr 2021 21:21:51 -0400 Subject: [PATCH] Remove old cxx common logic (#500) * Remove old cxx-common references * Test on this branch first * Revert "Test on this branch first" This reverts commit 888b584f8a1deeb8beba986a41f22614448e39e5. --- .github/workflows/{vcpkg_ci.yml => ci.yml} | 18 + CMakeLists.txt | 196 +++-------- CMakeLists_vcpkg.txt | 339 ------------------- Dockerfile | 76 ++--- README.md | 17 +- cmake/remillConfig.cmake.in | 16 - cmake/settings.cmake | 24 -- scripts/docker-lifter-entrypoint.sh | 35 +- scripts/travis.sh | 362 --------------------- tests/AArch64/CMakeLists.txt | 10 +- tests/X86/CMakeLists.txt | 20 +- 11 files changed, 135 insertions(+), 978 deletions(-) rename .github/workflows/{vcpkg_ci.yml => ci.yml} (85%) delete mode 100644 CMakeLists_vcpkg.txt delete mode 100755 scripts/travis.sh diff --git a/.github/workflows/vcpkg_ci.yml b/.github/workflows/ci.yml similarity index 85% rename from .github/workflows/vcpkg_ci.yml rename to .github/workflows/ci.yml index cee6309d..9461ecd3 100644 --- a/.github/workflows/vcpkg_ci.yml +++ b/.github/workflows/ci.yml @@ -224,3 +224,21 @@ jobs: asset_path: remill_macos-10.15_packages.zip asset_name: remill_macos-10.15_packages.zip asset_content_type: application/gzip + + + Docker_Linux: + runs-on: ubuntu-latest + strategy: + matrix: + llvm: ["11"] + ubuntu: ["18.04"] + steps: + - uses: actions/checkout@v2 + - name: Build LLVM ${{ matrix.llvm }} on ${{ matrix.ubuntu }} + run: | + docker build . -t docker.pkg.github.com/lifting-bits/remill/remill-llvm${{ matrix.llvm }}-ubuntu${{ matrix.ubuntu }}-amd64:latest -f Dockerfile --build-arg UBUNTU_VERSION=${{ matrix.ubuntu }} --build-arg ARCH=amd64 --build-arg LLVM_VERSION=${{ matrix.llvm }} + - name: Test Docker image + run: | + docker run --rm docker.pkg.github.com/lifting-bits/remill/remill-llvm${{ matrix.llvm }}-ubuntu${{ matrix.ubuntu }}-amd64:latest --arch amd64 --ir_out /dev/stdout --bytes c704ba01000000 + docker run --rm docker.pkg.github.com/lifting-bits/remill/remill-llvm${{ matrix.llvm }}-ubuntu${{ matrix.ubuntu }}-amd64:latest --arch aarch64 --ir_out /dev/stdout --address 0x400544 --bytes FD7BBFA90000009000601891FD030091B7FFFF97E0031F2AFD7BC1A8C0035FD6 + docker run --rm docker.pkg.github.com/lifting-bits/remill/remill-llvm${{ matrix.llvm }}-ubuntu${{ matrix.ubuntu }}-amd64:latest --arch aarch32 -ir_out /dev/stderr --bytes 0cd04de208008de504108de500208de508309de504009de500109de5903122e0c20fa0e110109fe5001091e5002081e5040081e50cd08de21eff2fe14000000000000000 diff --git a/CMakeLists.txt b/CMakeLists.txt index 281becae..6b9cf63e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -12,41 +12,18 @@ # See the License for the specific language governing permissions and # limitations under the License. -if (NOT DEFINED ENV{TRAILOFBITS_LIBRARIES}) - message(STATUS "Using new vcpkg build system") - include(CMakeLists_vcpkg.txt) - return() -endif() +include(cmake/vcpkg_helper.cmake) project(remill) cmake_minimum_required(VERSION 3.14) - include(GNUInstallDirs) include("${CMAKE_CURRENT_SOURCE_DIR}/cmake/settings.cmake") -include("${CMAKE_CURRENT_SOURCE_DIR}/cmake/utils.cmake") -include("${CMAKE_CURRENT_SOURCE_DIR}/cmake/BCCompiler.cmake") +include("${CMAKE_CURRENT_SOURCE_DIR}/cmake/utils_vcpkg.cmake") include("${CMAKE_CURRENT_SOURCE_DIR}/cmake/ccache.cmake") list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake/modules") include(CTest) -if (LLVM_Z3_INSTALL_DIR) - find_package(Z3 4.7.1) - set(need_z3 TRUE) -elseif(DEFINED CXX_COMMON_REPOSITORY_ROOT) - set(LLVM_Z3_INSTALL_DIR "${CXX_COMMON_REPOSITORY_ROOT}/z3" CACHE PATH "Path to the z3 installation") - set(need_z3 TRUE) -else() - set(need_z3 FALSE) -endif() - -if(need_z3) - find_package(Z3 4.7.1) - if (NOT Z3_FOUND) - message(WARNING "Z3 >= 4.7.1 has not been found in LLVM_Z3_INSTALL_DIR: ${LLVM_Z3_INSTALL_DIR}.") - endif() -endif() - configureCcache() FindAndSelectClangCompiler() @@ -54,33 +31,19 @@ enable_language(C CXX ASM) set(REMILL_SOURCE_DIR "${PROJECT_SOURCE_DIR}") -if(DEFINED WIN32) - set(dynamic_lib_prefix "") - set(dynamic_lib_extension "dll") - set(static_lib_prefix "") - set(static_lib_extension "lib") - set(executable_extension ".exe") - set(install_folder "${CMAKE_INSTALL_PREFIX}/remill") -else() - set(dynamic_lib_prefix "lib") - set(dynamic_lib_extension "so") - set(static_lib_prefix "lib") - set(static_lib_extension "a") - set(executable_extension "") - set(install_folder "${CMAKE_INSTALL_PREFIX}") -endif() - -set(REMILL_INSTALL_LIB_DIR "${install_folder}/lib" CACHE PATH "Directory in which remill libraries will be installed") -set(REMILL_INSTALL_BIN_DIR "${install_folder}/bin" CACHE PATH "Directory in which remill binaries will be installed") -set(REMILL_INSTALL_INCLUDE_DIR "${install_folder}/include" CACHE PATH "Directory in which remill headers will be installed") -set(REMILL_INSTALL_SHARE_DIR "${install_folder}/share" CACHE PATH "Directory in which remill cmake files will be installed") +set(REMILL_INSTALL_LIB_DIR "${CMAKE_INSTALL_LIBDIR}" CACHE PATH "Directory in which remill libraries will be installed") +set(REMILL_INSTALL_BIN_DIR "${CMAKE_INSTALL_BINDIR}" CACHE PATH "Directory in which remill binaries will be installed") +set(REMILL_INSTALL_INCLUDE_DIR "${CMAKE_INSTALL_INCLUDEDIR}/remill" CACHE PATH "Directory in which remill headers will be installed") +set(REMILL_INSTALL_SHARE_DIR "${CMAKE_INSTALL_DATADIR}" CACHE PATH "Directory in which remill cmake files will be installed") # # libraries # # LLVM -find_package(LLVM REQUIRED CONFIG HINTS ${FINDPACKAGE_LLVM_HINTS}) +find_package(LLVM CONFIG REQUIRED) +message(STATUS "Found LLVM ${LLVM_PACKAGE_VERSION}") +message(STATUS "Using LLVMConfig.cmake in: ${LLVM_DIR}") string(REPLACE "." ";" LLVM_VERSION_LIST ${LLVM_PACKAGE_VERSION}) list(GET LLVM_VERSION_LIST 0 LLVM_MAJOR_VERSION) @@ -93,6 +56,7 @@ target_include_directories(thirdparty_llvm SYSTEM INTERFACE target_compile_definitions(thirdparty_llvm INTERFACE ${LLVM_DEFINITIONS} ) +include("${CMAKE_CURRENT_SOURCE_DIR}/cmake/BCCompiler.cmake") # Go find only the static libraries of LLVM, and link against those. foreach(LLVM_LIB IN LISTS LLVM_AVAILABLE_LIBS) @@ -105,40 +69,41 @@ endforeach() # These are out-of-order in `LLVM_AVAILABLE_LIBS` and should always be last. list(REMOVE_ITEM LLVM_LIBRARIES LLVMMC LLVMCore LLVMSupport) list(APPEND LLVM_LIBRARIES LLVMMC LLVMCore LLVMSupport) + target_link_libraries(thirdparty_llvm INTERFACE ${LLVM_LIBRARIES} ) -# Microsoft Z3 -add_library(thirdparty_z3 INTERFACE) -if(Z3_FOUND) - target_include_directories(thirdparty_z3 SYSTEM INTERFACE - ${Z3_INCLUDE_DIR} - ) - target_link_libraries(thirdparty_z3 INTERFACE - ${Z3_LIBRARIES} - ) +# Microsoft Z3 with LLVM. Not exactly used in remill, but LLVM doesn't link +# against it correctly +# NOTE: If changing this, also replicate in remillConfig file +if (LLVM_WITH_Z3) + find_package(Z3 CONFIG REQUIRED 4.7.1) + get_target_property(LLVMSupport_LIBS LLVMSupport INTERFACE_LINK_LIBRARIES) + list(REMOVE_ITEM LLVMSupport_LIBS Z3) + list(APPEND LLVMSupport_LIBS z3::libz3) + set_target_properties(LLVMSupport PROPERTIES + INTERFACE_LINK_LIBRARIES "${LLVMSupport_LIBS}") endif() +message(STATUS "LLVM Libraries: ${LLVM_LIBRARIES}") + # Intel XED -find_package(XED REQUIRED) +find_package(XED CONFIG REQUIRED) add_library(thirdparty_xed INTERFACE) -target_include_directories(thirdparty_xed SYSTEM INTERFACE - ${XED_INCLUDE_DIRS} -) target_link_libraries(thirdparty_xed INTERFACE - ${XED_LIBRARIES} + XED::XED ) # Google glog module -find_package(glog REQUIRED) +find_package(glog CONFIG REQUIRED) add_library(thirdparty_glog INTERFACE) target_link_libraries(thirdparty_glog INTERFACE glog::glog ) # Google gflags -find_package(gflags REQUIRED) +find_package(gflags CONFIG REQUIRED) add_library(thirdparty_gflags INTERFACE) target_link_libraries(thirdparty_gflags INTERFACE gflags @@ -173,14 +138,11 @@ option(REMILL_BUILD_SPARC32_RUNTIME "Build the Runtime for SPARC32. Turn this of # target settings # -set(REMILL_LLVM_VERSION "${LLVM_MAJOR_VERSION}.${LLVM_MINOR_VERSION}") +set(REMILL_LLVM_VERSION "${LLVM_MAJOR_VERSION}") +message("Remill llvm version: ${REMILL_LLVM_VERSION}") math(EXPR REMILL_LLVM_VERSION_NUMBER "${LLVM_MAJOR_VERSION} * 100 + ${LLVM_MINOR_VERSION}") -if(DEFINED WIN32) - set(REMILL_INSTALL_SEMANTICS_DIR "${install_folder}/${REMILL_LLVM_VERSION}/semantics" CACHE PATH "Directory into which semantics are installed") -else() - set(REMILL_INSTALL_SEMANTICS_DIR "${REMILL_INSTALL_SHARE_DIR}/remill/${REMILL_LLVM_VERSION}/semantics" CACHE PATH "Directory into which semantics are installed") -endif() +set(REMILL_INSTALL_SEMANTICS_DIR "${CMAKE_INSTALL_PREFIX}/${REMILL_INSTALL_SHARE_DIR}/remill/${REMILL_LLVM_VERSION}/semantics" CACHE PATH "Directory into which semantics are installed") set(REMILL_BUILD_SEMANTICS_DIR_X86 "${CMAKE_CURRENT_BINARY_DIR}/lib/Arch/X86/Runtime") set(REMILL_BUILD_SEMANTICS_DIR_AARCH32 "${CMAKE_CURRENT_BINARY_DIR}/lib/Arch/AArch32/Runtime") @@ -188,17 +150,21 @@ set(REMILL_BUILD_SEMANTICS_DIR_AARCH64 "${CMAKE_CURRENT_BINARY_DIR}/lib/Arch/AAr set(REMILL_BUILD_SEMANTICS_DIR_SPARC32 "${CMAKE_CURRENT_BINARY_DIR}/lib/Arch/SPARC32/Runtime") set(REMILL_BUILD_SEMANTICS_DIR_SPARC64 "${CMAKE_CURRENT_BINARY_DIR}/lib/Arch/SPARC64/Runtime") +# add everything as public. set(REMILL_INCLUDE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/include") set(REMILL_LIB_DIR "${CMAKE_CURRENT_SOURCE_DIR}/lib") add_library(remill_settings INTERFACE) -target_include_directories(remill_settings INTERFACE "${REMILL_INCLUDE_DIR}") +target_include_directories(remill_settings INTERFACE + $ + $) if(WIN32) # warnings and compiler settings target_compile_options(remill_settings INTERFACE - /MD /nologo /W3 /EHsc /wd4141 /wd4146 /wd4180 /wd4244 + "$<$:/MDd>$<$:/MD>" + /nologo /W3 /EHsc /wd4141 /wd4146 /wd4180 /wd4244 /wd4258 /wd4267 /wd4291 /wd4345 /wd4351 /wd4355 /wd4456 /wd4457 /wd4458 /wd4459 /wd4503 /wd4624 /wd4722 /wd4800 /wd4100 /wd4127 /wd4512 /wd4505 /wd4610 /wd4510 /wd4702 @@ -264,7 +230,7 @@ else() endif() target_compile_definitions(remill_settings INTERFACE - "REMILL_INSTALL_SEMANTICS_DIR=\"${REMILL_INSTALL_SEMANTICS_DIR}/\"" + "REMILL_INSTALL_SEMANTICS_DIR=\"${REMILL_INSTALL_SEMANTICS_DIR}\"" "REMILL_BUILD_SEMANTICS_DIR_X86=\"${REMILL_BUILD_SEMANTICS_DIR_X86}\"" "REMILL_BUILD_SEMANTICS_DIR_AARCH32=\"${REMILL_BUILD_SEMANTICS_DIR_AARCH32}\"" "REMILL_BUILD_SEMANTICS_DIR_AARCH64=\"${REMILL_BUILD_SEMANTICS_DIR_AARCH64}\"" @@ -272,15 +238,12 @@ target_compile_definitions(remill_settings INTERFACE "REMILL_BUILD_SEMANTICS_DIR_SPARC64=\"${REMILL_BUILD_SEMANTICS_DIR_SPARC64}\"" ) -set(THIRDPARTY_LIBRARY_LIST thirdparty_z3 - thirdparty_llvm - thirdparty_xed - thirdparty_glog - thirdparty_gflags) - +set(THIRDPARTY_LIBRARY_LIST thirdparty_llvm thirdparty_xed thirdparty_glog thirdparty_gflags) target_link_libraries(remill_settings INTERFACE ${THIRDPARTY_LIBRARY_LIST} ) +install(TARGETS remill_settings ${THIRDPARTY_LIBRARY_LIST} + EXPORT remillTargets) add_subdirectory(lib/Arch) add_subdirectory(lib/BC) @@ -297,69 +260,18 @@ target_link_libraries(remill INTERFACE remill_version ${LINKER_END_GROUP} ) +install(TARGETS remill EXPORT remillTargets) # # Also install clang, libllvm and llvm-link # -set(INSTALLED_CLANG_NAME "remill-clang-${REMILL_LLVM_VERSION}${executable_extension}") -set(INSTALLED_LLVMLINK_NAME "remill-llvm-link-${REMILL_LLVM_VERSION}${executable_extension}") +set(INSTALLED_CLANG_NAME "remill-clang-${REMILL_LLVM_VERSION}${CMAKE_EXECUTABLE_SUFFIX}") +set(INSTALLED_LLVMLINK_NAME "remill-llvm-link-${REMILL_LLVM_VERSION}${CMAKE_EXECUTABLE_SUFFIX}") -if("${CXX_COMMON_REPOSITORY_ROOT}" STREQUAL "" OR NOT EXISTS "${CXX_COMMON_REPOSITORY_ROOT}/llvm") - set(INSTALLED_LIBLLVM_NAME "${dynamic_lib_prefix}LLVM-${REMILL_LLVM_VERSION}.${dynamic_lib_extension}") +InstallExternalTarget("ext_clang" "${CLANG_PATH}" "BIN" "${INSTALLED_CLANG_NAME}") - # system binaries are not built statically, so we need to fix the rpath - find_program("clang_location" "clang-${REMILL_LLVM_VERSION}${executable_extension}") - if("${clang_location}" STREQUAL "clang_location-NOTFOUND") - message(FATAL_ERROR "InstallExternalTarget: Failed to locate the clang executable") - endif() - - message("InstallExternalTarget: Found clang executable: ${clang_location}") - - find_program("llvmlink_location" "llvm-link${executable_extension}") - if("${llvmlink_location}" STREQUAL "llvmlink_location-NOTFOUND") - message(FATAL_ERROR "InstallExternalTarget: Failed to locate the llvm-link executable") - endif() - - message("InstallExternalTarget: Found llvm-link executable: ${llvmlink_location}") - - find_library("libllvm_location" "${dynamic_lib_prefix}LLVM-${REMILL_LLVM_VERSION}.${dynamic_lib_extension}") - if("${libllvm_location}" STREQUAL "libllvm_location-NOTFOUND") - message(FATAL_ERROR "InstallExternalTarget: Failed to locate the libLLVM dynamic library") - endif() - - message("InstallExternalTarget: Found libLLVM location: ${libllvm_location}") - - InstallExternalTarget("ext_clang" "${clang_location}" "${REMILL_INSTALL_BIN_DIR}" - "${INSTALLED_CLANG_NAME}" "${REMILL_INSTALL_LIB_DIR}") - - InstallExternalTarget("ext_llvmlink" "${llvmlink_location}" "${REMILL_INSTALL_BIN_DIR}" - "${INSTALLED_LLVMLINK_NAME}" "${REMILL_INSTALL_LIB_DIR}") - -else() - # The executable in our binary repository are statically built, meaning that we don't need - # to change the rpath - InstallExternalTarget("ext_clang" "${CXX_COMMON_REPOSITORY_ROOT}/llvm/bin/clang${executable_extension}" - "${REMILL_INSTALL_BIN_DIR}" "${INSTALLED_CLANG_NAME}") - - InstallExternalTarget("ext_llvmlink" "${CXX_COMMON_REPOSITORY_ROOT}/llvm/bin/llvm-link${executable_extension}" - "${REMILL_INSTALL_BIN_DIR}" "${INSTALLED_LLVMLINK_NAME}") -endif() - -set(REMILL_BC_LIBRARY_LOCATION "${REMILL_INSTALL_LIB_DIR}/${static_lib_prefix}remill_bc.${static_lib_extension}") -set(REMILL_ARCH_LIBRARY_LOCATION "${REMILL_INSTALL_LIB_DIR}/${static_lib_prefix}remill_arch.${static_lib_extension}") -set(REMILL_ARCH_X86_LIBRARY_LOCATION "${REMILL_INSTALL_LIB_DIR}/${static_lib_prefix}remill_arch_x86.${static_lib_extension}") -set(REMILL_ARCH_AARCH32_LIBRARY_LOCATION "${REMILL_INSTALL_LIB_DIR}/${static_lib_prefix}remill_arch_aarch32.${static_lib_extension}") -set(REMILL_ARCH_AARCH64_LIBRARY_LOCATION "${REMILL_INSTALL_LIB_DIR}/${static_lib_prefix}remill_arch_aarch64.${static_lib_extension}") -set(REMILL_ARCH_SPARC32_LIBRARY_LOCATION "${REMILL_INSTALL_LIB_DIR}/${static_lib_prefix}remill_arch_sparc32.${static_lib_extension}") -set(REMILL_ARCH_SPARC64_LIBRARY_LOCATION "${REMILL_INSTALL_LIB_DIR}/${static_lib_prefix}remill_arch_sparc64.${static_lib_extension}") -set(REMILL_OS_LIBRARY_LOCATION "${REMILL_INSTALL_LIB_DIR}/${static_lib_prefix}remill_os.${static_lib_extension}") -set(REMILL_VERSION_LIBRARY_LOCATION "${REMILL_INSTALL_LIB_DIR}/${static_lib_prefix}remill_version.${static_lib_extension}") - - -set(REMILL_INCLUDE_LOCATION "${REMILL_INSTALL_INCLUDE_DIR}") -get_target_property(REMILL_COMPILE_OPTIONS remill_settings INTERFACE_COMPILE_OPTIONS) -get_target_property(REMILL_COMPILE_DEFINITIONS remill_settings INTERFACE_COMPILE_DEFINITIONS) +InstallExternalTarget("ext_llvmlink" "${LLVMLINK_PATH}" "BIN" "${INSTALLED_LLVMLINK_NAME}") GetTargetTree(THIRDPARTY_LIBRARIES ${THIRDPARTY_LIBRARY_LIST}) GetPublicIncludeFolders(THIRDPARTY_INCLUDE_DIRECTORIES ${THIRDPARTY_LIBRARIES}) @@ -381,22 +293,17 @@ list(REMOVE_DUPLICATES THIRDPARTY_LIBRARY_FILES) # First do the basic substitutions. configure_file( - "${CMAKE_CURRENT_SOURCE_DIR}/cmake/remillConfig.cmake.in" - "${CMAKE_CURRENT_BINARY_DIR}/remillConfig.cmake.pregen" + "${CMAKE_CURRENT_SOURCE_DIR}/cmake/remillConfig_vcpkg.cmake.in" + "${CMAKE_CURRENT_BINARY_DIR}/remillConfig.cmake" @ONLY ) -# Then expand the generator expressions added to `THIRDPARTY_LIBRARY_FILES`. -file(GENERATE - OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/remillConfig.cmake" - INPUT "${CMAKE_CURRENT_BINARY_DIR}/remillConfig.cmake.pregen" -) - install(FILES "${CMAKE_CURRENT_BINARY_DIR}/remillConfig.cmake" + "${CMAKE_CURRENT_LIST_DIR}/cmake/vcpkg_helper.cmake" DESTINATION "${REMILL_INSTALL_LIB_DIR}/cmake/remill" ) -install(DIRECTORY "${REMILL_INCLUDE_DIR}/remill" +install(DIRECTORY "${REMILL_INCLUDE_DIR}/remill/" DESTINATION "${REMILL_INSTALL_INCLUDE_DIR}" ) @@ -409,8 +316,13 @@ add_custom_target(semantics) # tools add_subdirectory(bin) +install(EXPORT remillTargets + DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/remill") + # tests +message("compiler ID ${CMAKE_C_COMPILER_ID}") if ("${CMAKE_C_COMPILER_ID}" STREQUAL "Clang" OR "${CMAKE_C_COMPILER_ID}" STREQUAL "AppleClang") + find_package(Threads REQUIRED) add_custom_target(test_dependencies) if(NOT "${PLATFORM_NAME}" STREQUAL "windows") diff --git a/CMakeLists_vcpkg.txt b/CMakeLists_vcpkg.txt deleted file mode 100644 index 6b9cf63e..00000000 --- a/CMakeLists_vcpkg.txt +++ /dev/null @@ -1,339 +0,0 @@ -# Copyright (c) 2018 Trail of Bits, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -include(cmake/vcpkg_helper.cmake) - -project(remill) -cmake_minimum_required(VERSION 3.14) -include(GNUInstallDirs) - -include("${CMAKE_CURRENT_SOURCE_DIR}/cmake/settings.cmake") -include("${CMAKE_CURRENT_SOURCE_DIR}/cmake/utils_vcpkg.cmake") -include("${CMAKE_CURRENT_SOURCE_DIR}/cmake/ccache.cmake") -list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake/modules") -include(CTest) - -configureCcache() -FindAndSelectClangCompiler() - -enable_language(C CXX ASM) - -set(REMILL_SOURCE_DIR "${PROJECT_SOURCE_DIR}") - -set(REMILL_INSTALL_LIB_DIR "${CMAKE_INSTALL_LIBDIR}" CACHE PATH "Directory in which remill libraries will be installed") -set(REMILL_INSTALL_BIN_DIR "${CMAKE_INSTALL_BINDIR}" CACHE PATH "Directory in which remill binaries will be installed") -set(REMILL_INSTALL_INCLUDE_DIR "${CMAKE_INSTALL_INCLUDEDIR}/remill" CACHE PATH "Directory in which remill headers will be installed") -set(REMILL_INSTALL_SHARE_DIR "${CMAKE_INSTALL_DATADIR}" CACHE PATH "Directory in which remill cmake files will be installed") - -# -# libraries -# - -# LLVM -find_package(LLVM CONFIG REQUIRED) -message(STATUS "Found LLVM ${LLVM_PACKAGE_VERSION}") -message(STATUS "Using LLVMConfig.cmake in: ${LLVM_DIR}") - -string(REPLACE "." ";" LLVM_VERSION_LIST ${LLVM_PACKAGE_VERSION}) -list(GET LLVM_VERSION_LIST 0 LLVM_MAJOR_VERSION) -list(GET LLVM_VERSION_LIST 1 LLVM_MINOR_VERSION) - -add_library(thirdparty_llvm INTERFACE) -target_include_directories(thirdparty_llvm SYSTEM INTERFACE - ${LLVM_INCLUDE_DIRS} -) -target_compile_definitions(thirdparty_llvm INTERFACE - ${LLVM_DEFINITIONS} -) -include("${CMAKE_CURRENT_SOURCE_DIR}/cmake/BCCompiler.cmake") - -# Go find only the static libraries of LLVM, and link against those. -foreach(LLVM_LIB IN LISTS LLVM_AVAILABLE_LIBS) - get_target_property(LLVM_LIB_TYPE ${LLVM_LIB} TYPE) - if(LLVM_LIB_TYPE STREQUAL "STATIC_LIBRARY") - list(APPEND LLVM_LIBRARIES "${LLVM_LIB}") - endif() -endforeach() - -# These are out-of-order in `LLVM_AVAILABLE_LIBS` and should always be last. -list(REMOVE_ITEM LLVM_LIBRARIES LLVMMC LLVMCore LLVMSupport) -list(APPEND LLVM_LIBRARIES LLVMMC LLVMCore LLVMSupport) - -target_link_libraries(thirdparty_llvm INTERFACE - ${LLVM_LIBRARIES} -) - -# Microsoft Z3 with LLVM. Not exactly used in remill, but LLVM doesn't link -# against it correctly -# NOTE: If changing this, also replicate in remillConfig file -if (LLVM_WITH_Z3) - find_package(Z3 CONFIG REQUIRED 4.7.1) - get_target_property(LLVMSupport_LIBS LLVMSupport INTERFACE_LINK_LIBRARIES) - list(REMOVE_ITEM LLVMSupport_LIBS Z3) - list(APPEND LLVMSupport_LIBS z3::libz3) - set_target_properties(LLVMSupport PROPERTIES - INTERFACE_LINK_LIBRARIES "${LLVMSupport_LIBS}") -endif() - -message(STATUS "LLVM Libraries: ${LLVM_LIBRARIES}") - -# Intel XED -find_package(XED CONFIG REQUIRED) -add_library(thirdparty_xed INTERFACE) -target_link_libraries(thirdparty_xed INTERFACE - XED::XED -) - -# Google glog module -find_package(glog CONFIG REQUIRED) -add_library(thirdparty_glog INTERFACE) -target_link_libraries(thirdparty_glog INTERFACE - glog::glog -) - -# Google gflags -find_package(gflags CONFIG REQUIRED) -add_library(thirdparty_gflags INTERFACE) -target_link_libraries(thirdparty_gflags INTERFACE - gflags -) - -# Windows SDK -add_library(thirdparty_win32 INTERFACE) -if(DEFINED WIN32) - target_link_libraries(thirdparty_win32 INTERFACE - "Kernel32.lib" - ) -endif() - - -# For Linux builds, group LLVM libraries into a single group -# that avoids frustrating library ordering issues. -if(UNIX AND NOT APPLE) - set(LINKER_START_GROUP "-Wl,--start-group") - set(LINKER_END_GROUP "-Wl,--end-group") -else() - set(LINKER_START_GROUP "") - set(LINKER_END_GROUP "") -endif() - -# -# Configuration options for semantics -# -option(REMILL_BARRIER_AS_NOP "Remove compiler barriers (inline assembly) in semantics" OFF) -option(REMILL_BUILD_SPARC32_RUNTIME "Build the Runtime for SPARC32. Turn this off if you have include errors with , or read the README for a fix" ON) - -# -# target settings -# - -set(REMILL_LLVM_VERSION "${LLVM_MAJOR_VERSION}") -message("Remill llvm version: ${REMILL_LLVM_VERSION}") -math(EXPR REMILL_LLVM_VERSION_NUMBER "${LLVM_MAJOR_VERSION} * 100 + ${LLVM_MINOR_VERSION}") - -set(REMILL_INSTALL_SEMANTICS_DIR "${CMAKE_INSTALL_PREFIX}/${REMILL_INSTALL_SHARE_DIR}/remill/${REMILL_LLVM_VERSION}/semantics" CACHE PATH "Directory into which semantics are installed") - -set(REMILL_BUILD_SEMANTICS_DIR_X86 "${CMAKE_CURRENT_BINARY_DIR}/lib/Arch/X86/Runtime") -set(REMILL_BUILD_SEMANTICS_DIR_AARCH32 "${CMAKE_CURRENT_BINARY_DIR}/lib/Arch/AArch32/Runtime") -set(REMILL_BUILD_SEMANTICS_DIR_AARCH64 "${CMAKE_CURRENT_BINARY_DIR}/lib/Arch/AArch64/Runtime") -set(REMILL_BUILD_SEMANTICS_DIR_SPARC32 "${CMAKE_CURRENT_BINARY_DIR}/lib/Arch/SPARC32/Runtime") -set(REMILL_BUILD_SEMANTICS_DIR_SPARC64 "${CMAKE_CURRENT_BINARY_DIR}/lib/Arch/SPARC64/Runtime") - -# add everything as public. -set(REMILL_INCLUDE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/include") -set(REMILL_LIB_DIR "${CMAKE_CURRENT_SOURCE_DIR}/lib") - -add_library(remill_settings INTERFACE) - -target_include_directories(remill_settings INTERFACE - $ - $) - -if(WIN32) - # warnings and compiler settings - target_compile_options(remill_settings INTERFACE - "$<$:/MDd>$<$:/MD>" - /nologo /W3 /EHsc /wd4141 /wd4146 /wd4180 /wd4244 - /wd4258 /wd4267 /wd4291 /wd4345 /wd4351 /wd4355 /wd4456 - /wd4457 /wd4458 /wd4459 /wd4503 /wd4624 /wd4722 /wd4800 - /wd4100 /wd4127 /wd4512 /wd4505 /wd4610 /wd4510 /wd4702 - /wd4245 /wd4706 /wd4310 /wd4701 /wd4703 /wd4389 /wd4611 - /wd4805 /wd4204 /wd4577 /wd4091 /wd4592 /wd4324 - ) - - target_compile_definitions(remill_settings INTERFACE - _CRT_SECURE_NO_DEPRECATE - _CRT_SECURE_NO_WARNINGS - _CRT_NONSTDC_NO_DEPRECATE - _CRT_NONSTDC_NO_WARNINGS - _SCL_SECURE_NO_DEPRECATE - _SCL_SECURE_NO_WARNINGS - GOOGLE_PROTOBUF_NO_RTTI - ) - -else() - # warnings and compiler settings - target_compile_options(remill_settings INTERFACE - -Wall -Wextra -Wno-unused-parameter -Wno-c++98-compat - -Wno-unreachable-code-return -Wno-nested-anon-types - -Wno-extended-offsetof - -Wno-variadic-macros -Wno-return-type-c-linkage - -Wno-c99-extensions -Wno-ignored-attributes -Wno-unused-local-typedef - -Wno-unknown-pragmas -Wno-unknown-warning-option -fPIC - -fno-omit-frame-pointer -fvisibility-inlines-hidden - -fno-asynchronous-unwind-tables - ) - - # Clang-specific warnings/error options - if("${CMAKE_C_COMPILER_ID}" STREQUAL "Clang" OR "${CMAKE_C_COMPILER_ID}" STREQUAL "AppleClang") - target_compile_options(remill_settings INTERFACE - -Wgnu-alignof-expression -Wno-gnu-anonymous-struct -Wno-gnu-designator - -Wno-gnu-zero-variadic-macro-arguments -Wno-gnu-statement-expression - -fno-aligned-allocation - ) - endif() - - # debug symbols - if(CMAKE_BUILD_TYPE STREQUAL "Debug" OR CMAKE_BUILD_TYPE STREQUAL "RelWithDebInfo") - target_compile_options(remill_settings INTERFACE - -gdwarf-2 -g3 - ) - endif() - - # optimization flags and definitions - if(CMAKE_BUILD_TYPE STREQUAL "Debug") - target_compile_options(remill_settings INTERFACE - -O0 - ) - target_compile_definitions(remill_settings INTERFACE - "DEBUG" - ) - else() - target_compile_options(remill_settings INTERFACE - -O2 - ) - target_compile_definitions(remill_settings INTERFACE - "NDEBUG" - ) - endif() -endif() - -target_compile_definitions(remill_settings INTERFACE - "REMILL_INSTALL_SEMANTICS_DIR=\"${REMILL_INSTALL_SEMANTICS_DIR}\"" - "REMILL_BUILD_SEMANTICS_DIR_X86=\"${REMILL_BUILD_SEMANTICS_DIR_X86}\"" - "REMILL_BUILD_SEMANTICS_DIR_AARCH32=\"${REMILL_BUILD_SEMANTICS_DIR_AARCH32}\"" - "REMILL_BUILD_SEMANTICS_DIR_AARCH64=\"${REMILL_BUILD_SEMANTICS_DIR_AARCH64}\"" - "REMILL_BUILD_SEMANTICS_DIR_SPARC32=\"${REMILL_BUILD_SEMANTICS_DIR_SPARC32}\"" - "REMILL_BUILD_SEMANTICS_DIR_SPARC64=\"${REMILL_BUILD_SEMANTICS_DIR_SPARC64}\"" -) - -set(THIRDPARTY_LIBRARY_LIST thirdparty_llvm thirdparty_xed thirdparty_glog thirdparty_gflags) -target_link_libraries(remill_settings INTERFACE - ${THIRDPARTY_LIBRARY_LIST} -) -install(TARGETS remill_settings ${THIRDPARTY_LIBRARY_LIST} - EXPORT remillTargets) - -add_subdirectory(lib/Arch) -add_subdirectory(lib/BC) -add_subdirectory(lib/OS) -add_subdirectory(lib/Version) - -add_library(remill INTERFACE) -target_link_libraries(remill INTERFACE - ${LINKER_START_GROUP} - ${THIRDPARTY_LIBRARY_LIST} - remill_bc - remill_os - remill_arch - remill_version - ${LINKER_END_GROUP} -) -install(TARGETS remill EXPORT remillTargets) - -# -# Also install clang, libllvm and llvm-link -# - -set(INSTALLED_CLANG_NAME "remill-clang-${REMILL_LLVM_VERSION}${CMAKE_EXECUTABLE_SUFFIX}") -set(INSTALLED_LLVMLINK_NAME "remill-llvm-link-${REMILL_LLVM_VERSION}${CMAKE_EXECUTABLE_SUFFIX}") - -InstallExternalTarget("ext_clang" "${CLANG_PATH}" "BIN" "${INSTALLED_CLANG_NAME}") - -InstallExternalTarget("ext_llvmlink" "${LLVMLINK_PATH}" "BIN" "${INSTALLED_LLVMLINK_NAME}") - -GetTargetTree(THIRDPARTY_LIBRARIES ${THIRDPARTY_LIBRARY_LIST}) -GetPublicIncludeFolders(THIRDPARTY_INCLUDE_DIRECTORIES ${THIRDPARTY_LIBRARIES}) -foreach(THIRDPARTY_LIB IN LISTS THIRDPARTY_LIBRARIES) - string(SUBSTRING "${THIRDPARTY_LIB}" 0 1 THIRDPARTY_LIB_PREFIX) - if(TARGET ${THIRDPARTY_LIB}) - get_target_property(THIRDPARTY_LIB_TYPE ${THIRDPARTY_LIB} TYPE) - if(THIRDPARTY_LIB_TYPE STREQUAL "STATIC_LIBRARY" OR THIRDPARTY_LIB_TYPE STREQUAL "SHARED_LIBRARY") - list(APPEND THIRDPARTY_LIBRARY_FILES "$${}") - endif() - elseif("${THIRDPARTY_LIB_PREFIX}" STREQUAL "$${}") - # E.g. $ - else() - list(APPEND THIRDPARTY_LIBRARY_FILES "${THIRDPARTY_LIB}") - endif() -endforeach() - -list(REMOVE_DUPLICATES THIRDPARTY_LIBRARY_FILES) - -# First do the basic substitutions. -configure_file( - "${CMAKE_CURRENT_SOURCE_DIR}/cmake/remillConfig_vcpkg.cmake.in" - "${CMAKE_CURRENT_BINARY_DIR}/remillConfig.cmake" - @ONLY -) - -install(FILES "${CMAKE_CURRENT_BINARY_DIR}/remillConfig.cmake" - "${CMAKE_CURRENT_LIST_DIR}/cmake/vcpkg_helper.cmake" - DESTINATION "${REMILL_INSTALL_LIB_DIR}/cmake/remill" -) - -install(DIRECTORY "${REMILL_INCLUDE_DIR}/remill/" - DESTINATION "${REMILL_INSTALL_INCLUDE_DIR}" -) - -# -# additional targets -# - -add_custom_target(semantics) - -# tools -add_subdirectory(bin) - -install(EXPORT remillTargets - DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/remill") - -# tests -message("compiler ID ${CMAKE_C_COMPILER_ID}") -if ("${CMAKE_C_COMPILER_ID}" STREQUAL "Clang" OR "${CMAKE_C_COMPILER_ID}" STREQUAL "AppleClang") - find_package(Threads REQUIRED) - add_custom_target(test_dependencies) - - if(NOT "${PLATFORM_NAME}" STREQUAL "windows") - if("${CMAKE_HOST_SYSTEM_PROCESSOR}" STREQUAL "AMD64" OR "${CMAKE_HOST_SYSTEM_PROCESSOR}" STREQUAL "x86_64") - message(STATUS "X86 tests enabled") - add_subdirectory(tests/X86) - endif() - endif() - - if("${CMAKE_HOST_SYSTEM_PROCESSOR}" STREQUAL "aarch64" AND "${PLATFORM_NAME}" STREQUAL "linux") - message(STATUS "aarch64 tests enabled") - add_subdirectory(tests/AArch64) - endif() -endif() diff --git a/Dockerfile b/Dockerfile index 6b395e72..98afa7dd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,57 +1,51 @@ -ARG LLVM_VERSION=1000 +# Choose your LLVM version +ARG LLVM_VERSION=11 ARG ARCH=amd64 ARG UBUNTU_VERSION=18.04 ARG DISTRO_BASE=ubuntu${UBUNTU_VERSION} ARG BUILD_BASE=ubuntu:${UBUNTU_VERSION} -ARG LIBRARIES=/opt/trailofbits/libraries +ARG LIBRARIES=/opt/trailofbits + # Run-time dependencies go here FROM ${BUILD_BASE} as base -RUN apt-get update && \ - apt-get install -qqy --no-install-recommends libtinfo5 zlib1g libz3-4 && \ - rm -rf /var/lib/apt/lists/* - - # Build-time dependencies go here -FROM trailofbits/cxx-common:llvm${LLVM_VERSION}-${DISTRO_BASE}-${ARCH} as deps +# See here for full list of those dependencies +# https://github.com/trailofbits/cxx-common/blob/master/docker/Dockerfile.ubuntu.vcpkg +FROM trailofbits/cxx-common-vcpkg-builder-ubuntu:${UBUNTU_VERSION} as deps +ARG UBUNTU_VERSION +ARG ARCH +ARG LLVM_VERSION +ARG LIBRARIES -ENV DEBIAN_FRONTEND=noninteractive - -RUN if [ "$(uname -m)" = "aarch64" ]; then dpkg --add-architecture armhf; fi && \ - apt-get update && \ - if [ "$(uname -m)" = "x86_64" ]; then apt-get install -qqy gcc-multilib g++-multilib; fi && \ - if [ "$(uname -m)" = "aarch64" ]; then apt-get install -qqy gcc-arm-linux-gnueabihf libstdc++-8-dev:armhf; fi && \ - apt-get install -qqy zlib1g-dev libz3-4 ninja-build ccache git python3 curl coreutils build-essential libtinfo-dev lsb-release && \ +RUN apt-get update && \ + apt-get install -qqy python3 python3-pip libc6-dev wget liblzma-dev zlib1g-dev curl git build-essential ninja-build libselinux1-dev libbsd-dev ccache pixz xz-utils make rpm && \ + if [ "$(uname -m)" = "x86_64" ]; then dpkg --add-architecture i386 && apt-get update && apt-get install -qqy gcc-multilib g++-multilib zip zlib1g-dev:i386; fi && \ rm -rf /var/lib/apt/lists/* - # Source code build FROM deps as build -ARG LIBRARIES - -WORKDIR /remill -COPY . ./ - -ENV PATH="${LIBRARIES}/llvm/bin:${LIBRARIES}/cmake/bin:${LIBRARIES}/protobuf/bin:${PATH}" -ENV CC="${LIBRARIES}/llvm/bin/clang" -ENV CXX="${LIBRARIES}/llvm/bin/clang++" -ENV TRAILOFBITS_LIBRARIES="${LIBRARIES}" - -RUN mkdir build && cd build && \ - cmake -G Ninja -DCMAKE_VERBOSE_MAKEFILE=True -DCMAKE_INSTALL_PREFIX=/opt/trailofbits/remill .. && \ - cmake --build . --target install - -RUN cd build && \ - cmake --build . --target test_dependencies && \ - env CTEST_OUTPUT_ON_FAILURE=1 cmake --build . --target test - - -FROM base as dist ARG LLVM_VERSION -COPY scripts/docker-lifter-entrypoint.sh /opt/trailofbits/remill/docker-lifter-entrypoint.sh -COPY --from=build /opt/trailofbits/remill /opt/trailofbits/remill -ENV PATH=/opt/trailofbits/remill/bin:${PATH} \ - LLVM_VERSION=llvm${LLVM_VERSION} -ENTRYPOINT ["/opt/trailofbits/remill/docker-lifter-entrypoint.sh"] +WORKDIR /rellic +COPY ./ ./ +RUN ./scripts/build.sh \ + --llvm-version ${LLVM_VERSION} \ + --prefix /opt/trailofbits \ + --extra-cmake-args "-DCMAKE_BUILD_TYPE=Release" + +RUN cd remill-build && \ + cmake --build . --target test_dependencies && \ + CTEST_OUTPUT_ON_FAILURE=1 cmake --build . --verbose --target test && \ + cmake --build . --target install + +# Small installation image +FROM base as install +ARG LLVM_VERSION + +COPY --from=build /opt/trailofbits /opt/trailofbits +COPY scripts/docker-lifter-entrypoint.sh /opt/trailofbits +ENV LLVM_VERSION=llvm${LLVM_VERSION} \ + PATH=/opt/trailofbits/bin +ENTRYPOINT ["/opt/trailofbits/docker-lifter-entrypoint.sh"] diff --git a/README.md b/README.md index 7122d688..898720e3 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,7 @@ Remill now comes with a Dockerfile for easier testing. This Dockerfile reference The Dockerfile allows for quick builds of multiple supported LLVM, architecture, and Linux configurations. -Quickstart (builds Remill against LLVM 8.0 on Ubuntu 18.04 for AMD64): +Quickstart (builds Remill against LLVM 11 on Ubuntu 18.04 for AMD64): Clone Remill: ```shell @@ -67,21 +67,21 @@ cd remill Build Remill Docker container: ```shell # do the build -docker build . -t remill:llvm800-ubuntu18.04-amd64 \ +docker build . -t remill \ -f Dockerfile \ --build-arg UBUNTU_VERSION=18.04 \ --build-arg ARCH=amd64 \ - --build-arg LLVM_VERSION=800 + --build-arg LLVM_VERSION=11 ``` Ensure remill works: ```shell # Decode some AMD64 instructions to LLVM -docker run --rm -it remill:llvm800-ubuntu18.04-amd64 \ +docker run --rm -it remill \ --arch amd64 --ir_out /dev/stdout --bytes c704ba01000000 # Decode some AArch64 instructions to LLVM -docker run --rm -it remill:llvm800-ubuntu18.04-amd64 \ +docker run --rm -it remill \ --arch aarch64 --address 0x400544 --ir_out /dev/stdout \ --bytes FD7BBFA90000009000601891FD030091B7FFFF97E0031F2AFD7BC1A8C0035FD6 ``` @@ -97,20 +97,15 @@ sudo apt-get upgrade sudo apt-get install \ git \ - python2.7 \ + python3 \ wget \ curl \ build-essential \ - libtinfo-dev \ lsb-release \ - zlib1g-dev \ ccache \ libc6-dev:i386 \ 'libstdc++-*-dev:i386' \ g++-multilib - -# Ubuntu 14.04, 16.04 -sudo apt-get install realpath ``` Next, clone the repository. This will clone the code into the `remill` directory. diff --git a/cmake/remillConfig.cmake.in b/cmake/remillConfig.cmake.in index 78c26ea0..d488b78f 100644 --- a/cmake/remillConfig.cmake.in +++ b/cmake/remillConfig.cmake.in @@ -33,22 +33,6 @@ if(NOT TARGET remill) set(LINKER_END_GROUP "") endif() - if(NOT "x@CXX_COMMON_REPOSITORY_ROOT@x" STREQUAL "xx") - if (EXISTS "@CXX_COMMON_REPOSITORY_ROOT@") - set(CXX_COMMON_REPOSITORY_ROOT "@CXX_COMMON_REPOSITORY_ROOT@" - CACHE PATH "Location of cxx-common libraries" - ) - endif() - endif() - - if(NOT DEFINED CXX_COMMON_REPOSITORY_ROOT OR NOT EXISTS "${CXX_COMMON_REPOSITORY_ROOT}") - if(DEFINED ENV{TRAILOFBITS_LIBRARIES}) - set(CXX_COMMON_REPOSITORY_ROOT $ENV{TRAILOFBITS_LIBRARIES} - CACHE PATH "Location of cxx-common libraries." - ) - endif() - endif() - set(LLVM_MAJOR_VERSION @LLVM_MAJOR_VERSION@) set(LLVM_MINOR_VERSION @LLVM_MINOR_VERSION@) set(REMILL_LLVM_VERSION "@LLVM_MAJOR_VERSION@.@LLVM_MINOR_VERSION@") diff --git a/cmake/settings.cmake b/cmake/settings.cmake index 4f6719e4..c031379c 100644 --- a/cmake/settings.cmake +++ b/cmake/settings.cmake @@ -24,30 +24,6 @@ macro(main) # generate a compile commands JSON file. set(CMAKE_EXPORT_COMPILE_COMMANDS ON) - # - # cxx-common - # - - if(NOT DEFINED CXX_COMMON_REPOSITORY_ROOT) - if(DEFINED ENV{TRAILOFBITS_LIBRARIES}) - set(CXX_COMMON_REPOSITORY_ROOT $ENV{TRAILOFBITS_LIBRARIES} - CACHE PATH "Location of cxx-common libraries." - ) - endif() - endif() - - if(DEFINED CXX_COMMON_REPOSITORY_ROOT) - set(TOB_CMAKE_INCLUDE "${CXX_COMMON_REPOSITORY_ROOT}/cmake_modules/repository.cmake") - if(NOT EXISTS "${TOB_CMAKE_INCLUDE}") - message(FATAL_ERROR "The library repository could not be found!") - endif() - - include("${TOB_CMAKE_INCLUDE}") - - else() - message(STATUS "Using system libraries") - endif() - # # compiler and linker flags # diff --git a/scripts/docker-lifter-entrypoint.sh b/scripts/docker-lifter-entrypoint.sh index c7865dba..28cf2ec2 100755 --- a/scripts/docker-lifter-entrypoint.sh +++ b/scripts/docker-lifter-entrypoint.sh @@ -3,7 +3,7 @@ # Needed to process multiple arguments to docker image V="" -case ${LLVM_VERSION} in +case ${LLVM_VERSION} in llvm35*) V=3.5 ;; @@ -19,30 +19,29 @@ case ${LLVM_VERSION} in llvm39*) V=3.9 ;; - # There is an llvm401 that we treat as 4.0 - llvm40*) - V=4.0 + llvm4*) + V=4 ;; - llvm50*) - V=5.0 + llvm5*) + V=5 ;; - llvm60*) - V=6.0 + llvm6*) + V=6 ;; - llvm70*) - V=7.0 + llvm7*) + V=7 ;; - llvm80*) - V=8.0 + llvm8*) + V=8 ;; - llvm90*) - V=9.0 + llvm9*) + V=9 ;; - llvm100*) - V=10.0 + llvm10*) + V=10 ;; - llvm110*) - V=11.0 + llvm11*) + V=11 ;; *) echo "Unknown LLVM version: ${LLVM_VERSION}" diff --git a/scripts/travis.sh b/scripts/travis.sh deleted file mode 100755 index da49949b..00000000 --- a/scripts/travis.sh +++ /dev/null @@ -1,362 +0,0 @@ -#!/usr/bin/env bash - -# Copyright (c) 2017 Trail of Bits, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specifi - -main() { - if [ $# -ne 2 ] ; then - printf "Usage:\n\ttravis.sh \n" - return 1 - fi - - local platform_name="$1" - local operation_type="$2" - - # This makes life simpler for github actions - if [[ "${platform_name}" == "macos-latest" ]] ; then - platform_name="osx" - elif [[ "${platform_name}" == "ubuntu-latest" ]] ; then - platform_name="linux" - fi - - if [[ "${platform_name}" != "osx" && "${platform_name}" != "linux" ]] ; then - printf "Invalid platform: ${platform_name}\n" - return 1 - fi - - if [[ "${operation_type}" == "initialize" ]] ; then - "${platform_name}_initialize" - return $? - - elif [[ "$operation_type" == "build" ]] ; then - "${platform_name}_build" - return $? - - else - printf "Invalid operation\n" - return 1 - fi -} - -linux_initialize() { - printf "Initializing platform: linux\n" - - printf " > Updating the system...\n" - sudo apt-get -qq update - if [ $? -ne 0 ] ; then - printf " x The package database could not be updated\n" - return 1 - fi - - printf " > Installing the required packages...\n" - sudo apt-get install -qqy git python2.7 curl coreutils build-essential gcc-multilib g++-multilib libtinfo-dev lsb-release ccache - if [ $? -ne 0 ] ; then - printf " x Could not install the required dependencies\n" - return 1 - fi - - printf " > The system has been successfully initialized\n" - return 0 -} - -osx_initialize() { - printf "Initializing platform: osx\n" - if [[ "x${SDKROOT}x" = "xx" ]] ; then - export SDKROOT=$(xcrun -sdk macosx --show-sdk-path) - fi - printf " > The macOS SDK is located at ${SDKROOT}\n" - - # Mainly for realpath - brew install coreutils cmake - if [ $? -ne 0 ] ; then - printf " x Could not install the required dependencies\n" - return 1 - fi - - return 0 -} - -linux_build() { - local os_version=`cat /etc/issue | awk '{ print $2 }' | cut -d '.' -f 1-2 | tr -d '.'` - - llvm_version_list=( "40" "50" "60" ) - for llvm_version in "${llvm_version_list[@]}" ; do - common_build "ubuntu${os_version}" "${llvm_version}" - if [ $? -ne 0 ] ; then - return 1 - fi - - printf "\n\n" - done - - return 0 -} - -osx_build() { - llvm_version_list=( "40" ) - for llvm_version in "${llvm_version_list[@]}" ; do - common_build "osx" "${llvm_version}" - if [ $? -ne 0 ] ; then - return 1 - fi - - printf "\n\n" - done - - return 0 -} - -common_build() { - if [ $# -ne 2 ] ; then - printf "Usage:\n\tcommon_build \n\nllvm_version: 35, 40, ...\n" - return 1 - fi - - local original_path="${PATH}" - local log_file=`mktemp` - local os_version="$1" - local llvm_version="$2" - - printf "#\n" - printf "# Running CI tests for LLVM version ${llvm_version}...\n" - printf "#\n\n" - - printf " > Cleaning up the environment variables...\n" - export PATH="${original_path}" - - unset TRAILOFBITS_LIBRARIES - unset CC - unset CXX - - printf " > Cleaning up the build folders...\n" - if [ -d "build" ] ; then - sudo rm -rf build > "${log_file}" 2>&1 - if [ $? -ne 0 ] ; then - printf " x Failed to remove the existing build folder. Error output follows:\n" - printf "===\n" - cat "${log_file}" - return 1 - fi - fi - - if [ -d "libraries" ] ; then - sudo rm -rf libraries > "${log_file}" 2>&1 - if [ $? -ne 0 ] ; then - printf " x Failed to remove the existing libraries folder. Error output follows:\n" - printf "===\n" - cat "${log_file}" - return 1 - fi - fi - - # acquire the cxx-common package - printf " > Acquiring the cxx-common package: LLVM${llvm_version} for ${os_version}\n" - - if [ ! -d "cxxcommon" ] ; then - mkdir "cxxcommon" > "${log_file}" 2>&1 - if [ $? -ne 0 ] ; then - printf " x Failed to create the cxxcommon folder. Error output follows:\n" - printf "===\n" - cat "${log_file}" - return 1 - fi - fi - - local cxx_common_tarball_name="libraries-llvm${llvm_version}-${os_version}-amd64.tar.gz" - if [ ! -f "cxxcommon/${cxx_common_tarball_name}" ] ; then - ( cd "cxxcommon" && curl -C - "https://s3.amazonaws.com/cxx-common/${cxx_common_tarball_name}" -O ) > "${log_file}" 2>&1 - if [ $? -ne 0 ] ; then - printf " x Failed to download the cxx-common package. Error output follows:\n" - printf "===\n" - cat "${log_file}" - - rm "cxxcommon/${cxx_common_tarball_name}" - return 1 - fi - fi - - if [ ! -d "libraries" ] ; then - tar xzf "cxxcommon/${cxx_common_tarball_name}" > "${log_file}" 2>&1 - if [ $? -ne 0 ] ; then - printf " x The archive appears to be corrupted. Error output follows:\n" - printf "===\n" - cat "${log_file}" - - rm "cxxcommon/${cxx_common_tarball_name}" - rm -rf libraries - return 1 - fi - fi - - export CCACHE_DIR="ccache_llvm${llvm_version}" - - if [ ! -d "${CCACHE_DIR}" ] ; then - printf " > Creating ccache folder\n" - else - printf " > Using existing ccache folder\n" - fi - - export CCACHE_DIR="$(realpath ${CCACHE_DIR})" - printf " i ${CCACHE_DIR}\n" - - export TRAILOFBITS_LIBRARIES=`GetRealPath libraries` - export PATH="${TRAILOFBITS_LIBRARIES}/llvm/bin:${TRAILOFBITS_LIBRARIES}/protobuf/bin:${PATH}" - # Use brew-installed cmake instead of outdated version here - if [[ "${platform_name}" != "osx" ]] ; then - export PATH="${TRAILOFBITS_LIBRARIES}/cmake/bin:${PATH}" - fi - - export CC="${TRAILOFBITS_LIBRARIES}/llvm/bin/clang" - export CXX="${TRAILOFBITS_LIBRARIES}/llvm/bin/clang++" - - printf " > Generating the project...\n" - mkdir build > "${log_file}" 2>&1 - if [ $? -ne 0 ] ; then - printf " x Failed to create the build folder. Error output follows:\n" - printf "===\n" - cat "${log_file}" - return 1 - fi - - ( cd build && cmake -DCMAKE_VERBOSE_MAKEFILE=True .. ) > "${log_file}" 2>&1 - if [ $? -ne 0 ] ; then - printf " x Failed to generate the project. Error output follows:\n" - printf "===\n" - cat "${log_file}" - return 1 - fi - - printf " > Building remill...\n" - if [ "${llvm_version:0:1}" == "3" ] ; then - printf " i Clang static analyzer not supported on this LLVM release (${llvm_version})\n" - ( cd build && make -j `nproc` ) > "${log_file}" 2>&1 & - else - printf " i Clang static analyzer enabled\n" - ( cd build && scan-build --show-description --status-bugs make -j `GetProcessorCount` ) > "${log_file}" 2>&1 & - fi - - local build_pid="$!" - - printf "\nWaiting..." - while [ true ] ; do - kill -s 0 "${build_pid}" > /dev/null 2>&1 - if [ $? -ne 0 ] ; then - break - fi - - printf "." - sleep 5 - done - printf "\n\n" - - wait "${build_pid}" - if [ $? -ne 0 ] ; then - printf " x Failed to build the project. Error output follows:\n" - printf "===\n" - cat "${log_file}" - return 1 - fi - - if [ "${llvm_version:0:1}" != "3" ] ; then - if [ `cat "${log_file}" | grep 'scan-build: No bugs found.' | wc -l` != 0 ] ; then - printf " i scan-build didn't find any bug\n" - else - printf " ! scan-build output follows\n" - if [ "${llvm_version:0:1}" != "3" ] ; then - cat "${log_file}" | while read line ; do printf " %s\n" "${line}" ; done - printf "\n" - fi - fi - fi - - # Some LLVM versions can't compile the tests - if [ "${llvm_version}" == "35" ] || [ "${llvm_version}" == "36" ] || [ "${llvm_version}" == "37" ] || [ "${llvm_version}" == "38" ] ; then - printf " ! Tests are not compatible with this LLVM version (${llvm_version})\n" - printf " > Build succeeded\n" - return 0 - fi - - # Some LLVM versions aren't (yet) compatible with the tests - if [ "${llvm_version}" == "50" ] ; then - printf " ! Tests are blacklisted for this LLVM version (${llvm_version})\n" - printf " > Build succeeded\n" - return 0 - fi - - which sw_vers > /dev/null 2>&1 - if [ $? -eq 0 ] ; then - printf " ! Skipping the install step on macOS\n" - else - printf " > Installing...\n" - ( cd build && sudo make install ) > "${log_file}" 2>&1 - if [ $? -ne 0 ] ; then - printf " x Failed to install the project. Error output follows:\n" - printf "===\n" - cat "${log_file}" - return 1 - fi - fi - - which sw_vers > /dev/null 2>&1 - if [ $? -eq 0 ] ; then - printf " ! Skipping the tests on macOS\n" - return 0 - fi - - printf " > Building and running the tests...\n\nWaiting..." - ( cd build && make -j `GetProcessorCount` test_dependencies && env CTEST_OUTPUT_ON_FAILURE=1 make test ) > "${log_file}" 2>&1 & - local test_pid="$!" - - while [ true ] ; do - kill -s 0 "${test_pid}" > /dev/null 2>&1 - if [ $? -ne 0 ] ; then - break - fi - - printf "." - sleep 5 - done - printf "\n\n" - - wait "${test_pid}" - if [ $? -ne 0 ] ; then - printf " x Failed to build and run the tests. Error output follows:\n" - printf "===\n" - cat "${log_file}" - return 1 - fi - - printf " > Build succeeded\n" - return 0 -} - -GetProcessorCount() { - which nproc > /dev/null 2>&1 - if [ $? -eq 0 ] ; then - nproc - else - sysctl -n hw.ncpu - fi -} - -GetRealPath() { - which realpath > /dev/null 2>&1 - if [ $? -eq 0 ] ; then - realpath $1 - else - [[ $1 = /* ]] && echo "$1" || echo "$PWD/${1#./}" - fi -} - -main $@ -exit $? diff --git a/tests/AArch64/CMakeLists.txt b/tests/AArch64/CMakeLists.txt index 1a071572..3bf5f63a 100644 --- a/tests/AArch64/CMakeLists.txt +++ b/tests/AArch64/CMakeLists.txt @@ -12,14 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -if (DEFINED ENV{TRAILOFBITS_LIBRARIES}) - find_package(gtest REQUIRED) - list(APPEND PROJECT_LIBRARIES ${gtest_LIBRARIES}) - list(APPEND PROJECT_INCLUDEDIRECTORIES ${gtest_INCLUDE_DIRS}) -else() - find_package(GTest CONFIG REQUIRED) - list(APPEND PROJECT_LIBRARIES GTest::gtest) -endif() +find_package(GTest CONFIG REQUIRED) +list(APPEND PROJECT_LIBRARIES GTest::gtest) enable_testing() enable_language(ASM) diff --git a/tests/X86/CMakeLists.txt b/tests/X86/CMakeLists.txt index 2c344bfd..4367ae1c 100644 --- a/tests/X86/CMakeLists.txt +++ b/tests/X86/CMakeLists.txt @@ -41,12 +41,7 @@ function(COMPILE_X86_TESTS name address_size has_avx has_avx512) set_target_properties(lift-${name}-tests PROPERTIES OBJECT_DEPENDS "${X86_TEST_FILES}") - if (DEFINED ENV{TRAILOFBITS_LIBRARIES}) - target_link_libraries(lift-${name}-tests PUBLIC remill ${gtest_LIBRARIES}) - target_include_directories(lift-${name}-tests PUBLIC ${gtest_INCLUDE_DIRS}) - else() - target_link_libraries(lift-${name}-tests PRIVATE remill GTest::gtest) - endif() + target_link_libraries(lift-${name}-tests PRIVATE remill GTest::gtest) target_compile_definitions(lift-${name}-tests PUBLIC ${PROJECT_DEFINITIONS}) add_custom_command( @@ -64,12 +59,7 @@ function(COMPILE_X86_TESTS name address_size has_avx has_avx512) add_executable(run-${name}-tests EXCLUDE_FROM_ALL Run.cpp Tests.S tests_${name}.S) set_target_properties(run-${name}-tests PROPERTIES OBJECT_DEPENDS "${X86_TEST_FILES}") - if (DEFINED ENV{TRAILOFBITS_LIBRARIES}) - target_link_libraries(run-${name}-tests PUBLIC remill ${gtest_LIBRARIES}) - target_include_directories(run-${name}-tests PUBLIC ${gtest_INCLUDE_DIRS}) - else() - target_link_libraries(run-${name}-tests PUBLIC remill GTest::gtest) - endif() + target_link_libraries(run-${name}-tests PUBLIC remill GTest::gtest) target_compile_definitions(run-${name}-tests PUBLIC ${PROJECT_DEFINITIONS}) target_compile_options(run-${name}-tests @@ -81,11 +71,7 @@ function(COMPILE_X86_TESTS name address_size has_avx has_avx512) add_dependencies(test_dependencies "run-${name}-tests") endfunction() -if (DEFINED ENV{TRAILOFBITS_LIBRARIES}) - find_package(gtest REQUIRED) -else() - find_package(GTest CONFIG REQUIRED) -endif() +find_package(GTest CONFIG REQUIRED) enable_testing()