From d5c50357287cae0072938477b3b525bafcce8069 Mon Sep 17 00:00:00 2001 From: Alex Cameron Date: Sun, 30 Oct 2022 20:42:23 +0000 Subject: [PATCH] Get Remill building with LLVM 15 (#631) * Get Remill building with LLVM 15 * Add missing header for x86 lift test * Fix `enableOpaquePointers` calls * Define a non-extern `__remill_state` in each Instructions module * Remove `__remill_state` variables in tests * Update build script to support LLVM 15 * Build with LLVM 15 in CI * Bump CXX Common version * Update Docker script * Add comment explaining the definition of the state variable * Correct wording --- .github/workflows/ci.yml | 6 +-- .../LiftAndCompare.cpp | 3 ++ bin/lift/Lift.cpp | 3 ++ cmake/BCCompiler.cmake | 2 +- include/remill/Arch/X86/Runtime/State.h | 2 + lib/Arch/AArch32/Runtime/Instructions.cpp | 4 ++ lib/Arch/AArch64/Runtime/Instructions.cpp | 4 ++ lib/Arch/Runtime/Intrinsics.cpp | 4 +- lib/Arch/SPARC32/Runtime/Instructions.cpp | 4 ++ lib/Arch/SPARC64/Runtime/Instructions.cpp | 4 ++ lib/Arch/X86/Runtime/Instructions.cpp | 4 ++ lib/BC/SleighLifter.cpp | 1 + lib/BC/Util.cpp | 40 ---------------- scripts/build.sh | 10 ++-- scripts/docker-lifter-entrypoint.sh | 48 ++----------------- tests/AArch64/Lift.cpp | 2 + tests/AArch64/Run.cpp | 2 - tests/Thumb/TestLifting.cpp | 25 ++++++---- tests/X86/Lift.cpp | 3 ++ tests/X86/Run.cpp | 2 - 20 files changed, 68 insertions(+), 105 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9975475b..6a3f41bb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,7 +23,7 @@ jobs: matrix: image: - { name: 'ubuntu', tag: '20.04' } - llvm: ['14'] + llvm: ['14', '15'] runs-on: ubuntu-20.04 container: @@ -103,7 +103,7 @@ jobs: fail-fast: false matrix: os: ['macos-11'] - llvm: ['14'] + llvm: ['14', '15'] runs-on: ${{ matrix.os }} @@ -235,7 +235,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - llvm: ["14"] + llvm: ["14", "15"] ubuntu: ["20.04"] steps: - uses: actions/checkout@v2 diff --git a/bin/differential_tester_x86/LiftAndCompare.cpp b/bin/differential_tester_x86/LiftAndCompare.cpp index 2909ba85..6ec9732d 100644 --- a/bin/differential_tester_x86/LiftAndCompare.cpp +++ b/bin/differential_tester_x86/LiftAndCompare.cpp @@ -36,6 +36,7 @@ #include #include #include +#include #include #include @@ -106,7 +107,9 @@ class DifferentialModuleBuilder { // it is expected that compatible arches share a semantics module. std::unique_ptr context = std::make_unique(); +#if LLVM_VERSION_NUMBER < LLVM_VERSION(15, 0) context->enableOpaquePointers(); +#endif auto tmp_arch = remill::Arch::Build(context.get(), os_name_1, arch_name_1); std::shared_ptr semantics_module = remill::LoadArchSemantics(tmp_arch.get()); diff --git a/bin/lift/Lift.cpp b/bin/lift/Lift.cpp index f407e683..3e51b23d 100644 --- a/bin/lift/Lift.cpp +++ b/bin/lift/Lift.cpp @@ -33,6 +33,7 @@ #include #include #include +#include #include #include @@ -240,7 +241,9 @@ int main(int argc, char *argv[]) { // Make sure `--address` and `--entry_address` are in-bounds for the target // architecture's address size. llvm::LLVMContext context; +#if LLVM_VERSION_NUMBER < LLVM_VERSION(15, 0) context.enableOpaquePointers(); +#endif auto arch = remill::Arch::Get(context, FLAGS_os, FLAGS_arch); const uint64_t addr_mask = ~0ULL >> (64UL - arch->address_size); if (FLAGS_address != (FLAGS_address & addr_mask)) { diff --git a/cmake/BCCompiler.cmake b/cmake/BCCompiler.cmake index 055b80ff..a806d93a 100644 --- a/cmake/BCCompiler.cmake +++ b/cmake/BCCompiler.cmake @@ -12,7 +12,7 @@ set(DEFAULT_BC_COMPILER_FLAGS -Wno-gnu-anonymous-struct -Wno-return-type-c-linkage -Wno-gnu-zero-variadic-macro-arguments -Wno-nested-anon-types -Wno-extended-offsetof -Wno-gnu-statement-expression -Wno-c99-extensions - -Wno-ignored-attributes -mtune=generic -fno-vectorize -fno-slp-vectorize + -Wno-ignored-attributes -fno-vectorize -fno-slp-vectorize -Wno-variadic-macros -Wno-c11-extensions -Wno-c++11-extensions -ffreestanding -fno-common -fno-builtin -fno-exceptions -fno-rtti -fno-asynchronous-unwind-tables -Wno-unneeded-internal-declaration diff --git a/include/remill/Arch/X86/Runtime/State.h b/include/remill/Arch/X86/Runtime/State.h index abed01e0..49dbe23a 100644 --- a/include/remill/Arch/X86/Runtime/State.h +++ b/include/remill/Arch/X86/Runtime/State.h @@ -168,6 +168,7 @@ static_assert(2 == sizeof(FPUControlWord), "Invalid structure packing of `FPUControl`."); struct FPUStackElem final { + FPUStackElem() {} union { float80_t st; struct { @@ -322,6 +323,7 @@ struct FpuFXSAVE64 { // FP register state that conforms with `FXSAVE` and `FXSAVE64`. union alignas(16) FPU final { + FPU() {} struct : public FpuFSAVE { uint8_t _padding0[512 - sizeof(FpuFSAVE)]; } __attribute__((packed)) fsave; diff --git a/lib/Arch/AArch32/Runtime/Instructions.cpp b/lib/Arch/AArch32/Runtime/Instructions.cpp index c6898176..79d7f516 100644 --- a/lib/Arch/AArch32/Runtime/Instructions.cpp +++ b/lib/Arch/AArch32/Runtime/Instructions.cpp @@ -29,6 +29,10 @@ // clang-format on +// A definition is required to ensure that LLVM doesn't optimize the `State` type out of the bytecode +// See https://github.com/lifting-bits/remill/pull/631#issuecomment-1279989004 +State __remill_state; + #define REG_PC state.gpr.r15.dword #define REG_LR state.gpr.r14.dword #define REG_SP state.gpr.r13.dword diff --git a/lib/Arch/AArch64/Runtime/Instructions.cpp b/lib/Arch/AArch64/Runtime/Instructions.cpp index a486b4b4..82db2118 100644 --- a/lib/Arch/AArch64/Runtime/Instructions.cpp +++ b/lib/Arch/AArch64/Runtime/Instructions.cpp @@ -29,6 +29,10 @@ // clang-format on +// A definition is required to ensure that LLVM doesn't optimize the `State` type out of the bytecode +// See https://github.com/lifting-bits/remill/pull/631#issuecomment-1279989004 +State __remill_state; + #define REG_PC state.gpr.pc.qword #define REG_SP state.gpr.sp.qword #define REG_LP state.gpr.x30.qword diff --git a/lib/Arch/Runtime/Intrinsics.cpp b/lib/Arch/Runtime/Intrinsics.cpp index cb99bdc4..b83e973c 100644 --- a/lib/Arch/Runtime/Intrinsics.cpp +++ b/lib/Arch/Runtime/Intrinsics.cpp @@ -27,6 +27,8 @@ // addresses taken, and so this prevents dead argument elimination. extern "C" void __remill_mark_as_used(const void *); +// Each architecture's semantics module defines this variable +// See https://github.com/lifting-bits/remill/pull/631#issuecomment-1279989004 extern State __remill_state; #if defined(REMILL_ON_SPARC32) || defined(REMILL_ON_SPARC64) @@ -35,7 +37,7 @@ extern RegisterWindow __remill_register_window; // This is just a hack to make sure all these functions appear in the bitcode // file! -[[gnu::used]] extern "C" void __remill_intrinsics(void) { +extern "C" [[gnu::used]] void __remill_intrinsics(void) { USED(__remill_state); diff --git a/lib/Arch/SPARC32/Runtime/Instructions.cpp b/lib/Arch/SPARC32/Runtime/Instructions.cpp index a80129d5..d348821b 100644 --- a/lib/Arch/SPARC32/Runtime/Instructions.cpp +++ b/lib/Arch/SPARC32/Runtime/Instructions.cpp @@ -24,6 +24,10 @@ #include "remill/Arch/SPARC32/Runtime/State.h" #include "remill/Arch/SPARC32/Runtime/Types.h" +// A definition is required to ensure that LLVM doesn't optimize the `State` type out of the bytecode +// See https://github.com/lifting-bits/remill/pull/631#issuecomment-1279989004 +State __remill_state; + #define REG_PC state.pc.aword #define REG_NPC state.next_pc.aword #define REG_SP state.gpr.o6.aword diff --git a/lib/Arch/SPARC64/Runtime/Instructions.cpp b/lib/Arch/SPARC64/Runtime/Instructions.cpp index f1b70eb5..019952d2 100644 --- a/lib/Arch/SPARC64/Runtime/Instructions.cpp +++ b/lib/Arch/SPARC64/Runtime/Instructions.cpp @@ -24,6 +24,10 @@ #include "remill/Arch/SPARC64/Runtime/State.h" #include "remill/Arch/SPARC64/Runtime/Types.h" +// A definition is required to ensure that LLVM doesn't optimize the `State` type out of the bytecode +// See https://github.com/lifting-bits/remill/pull/631#issuecomment-1279989004 +State __remill_state; + #define REG_PC state.pc.aword #define REG_NPC state.next_pc.aword #define REG_SP state.gpr.o6.aword diff --git a/lib/Arch/X86/Runtime/Instructions.cpp b/lib/Arch/X86/Runtime/Instructions.cpp index 895f0ec9..1a18e4c8 100644 --- a/lib/Arch/X86/Runtime/Instructions.cpp +++ b/lib/Arch/X86/Runtime/Instructions.cpp @@ -28,6 +28,10 @@ // clang-format on +// A definition is required to ensure that LLVM doesn't optimize the `State` type out of the bytecode +// See https://github.com/lifting-bits/remill/pull/631#issuecomment-1279989004 +State __remill_state; + #define REG_IP state.gpr.rip.word #define REG_EIP state.gpr.rip.dword #define REG_RIP state.gpr.rip.qword diff --git a/lib/BC/SleighLifter.cpp b/lib/BC/SleighLifter.cpp index 37219933..8b719ed6 100644 --- a/lib/BC/SleighLifter.cpp +++ b/lib/BC/SleighLifter.cpp @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include diff --git a/lib/BC/Util.cpp b/lib/BC/Util.cpp index b7157dcd..e43c2a20 100644 --- a/lib/BC/Util.cpp +++ b/lib/BC/Util.cpp @@ -1204,46 +1204,6 @@ MoveConstantIntoModule(llvm::Constant *c, llvm::Module *dest_module, moved_c = ret; return ret; } - case llvm::Instruction::UDiv: { - const auto b = llvm::dyn_cast(ce); - auto ret = llvm::ConstantExpr::getUDiv( - MoveConstantIntoModule(ce->getOperand(0), dest_module, value_map, - type_map), - MoveConstantIntoModule(ce->getOperand(1), dest_module, value_map, - type_map), - b->isExact()); - moved_c = ret; - return ret; - } - case llvm::Instruction::SDiv: { - const auto b = llvm::dyn_cast(ce); - auto ret = llvm::ConstantExpr::getSDiv( - MoveConstantIntoModule(ce->getOperand(0), dest_module, value_map, - type_map), - MoveConstantIntoModule(ce->getOperand(1), dest_module, value_map, - type_map), - b->isExact()); - moved_c = ret; - return ret; - } - case llvm::Instruction::URem: { - auto ret = llvm::ConstantExpr::getURem( - MoveConstantIntoModule(ce->getOperand(0), dest_module, value_map, - type_map), - MoveConstantIntoModule(ce->getOperand(1), dest_module, value_map, - type_map)); - moved_c = ret; - return ret; - } - case llvm::Instruction::SRem: { - auto ret = llvm::ConstantExpr::getSRem( - MoveConstantIntoModule(ce->getOperand(0), dest_module, value_map, - type_map), - MoveConstantIntoModule(ce->getOperand(1), dest_module, value_map, - type_map)); - moved_c = ret; - return ret; - } case llvm::Instruction::IntToPtr: { auto ret = llvm::ConstantExpr::getIntToPtr( MoveConstantIntoModule(ce->getOperand(0), dest_module, value_map, diff --git a/scripts/build.sh b/scripts/build.sh index 708c2fc7..8f9cf0f0 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -23,11 +23,11 @@ DOWNLOAD_DIR="$( cd "$( dirname "${SRC_DIR}" )" && pwd )/lifting-bits-downloads" CURR_DIR=$( pwd ) BUILD_DIR="${CURR_DIR}/remill-build" INSTALL_DIR=/usr/local -LLVM_VERSION=llvm-14 +LLVM_VERSION=llvm-15 OS_VERSION= ARCH_VERSION= BUILD_FLAGS= -CXX_COMMON_VERSION="0.2.10" +CXX_COMMON_VERSION="0.2.12" # There are pre-build versions of various libraries for specific # Ubuntu releases. @@ -276,6 +276,10 @@ function GetLLVMVersion LLVM_VERSION=llvm-14 return 0 ;; + 15) + LLVM_VERSION=llvm-15 + return 0 + ;; *) # unknown option echo "[x] Unknown or unsupported LLVM version ${1}. You may be able to manually build it with cxx-common." @@ -291,7 +295,7 @@ function Help echo "" echo "Options:" echo " --prefix Change the default (${INSTALL_DIR}) installation prefix." - echo " --llvm-version Change the default (14) LLVM version." + echo " --llvm-version Change the default (15) LLVM version." echo " --build-dir Change the default (${BUILD_DIR}) build directory." echo " --debug Build with Debug symbols." echo " --extra-cmake-args Extra CMake arguments to build with." diff --git a/scripts/docker-lifter-entrypoint.sh b/scripts/docker-lifter-entrypoint.sh index c4ec830e..9f751af4 100755 --- a/scripts/docker-lifter-entrypoint.sh +++ b/scripts/docker-lifter-entrypoint.sh @@ -4,54 +4,12 @@ V="" case ${LLVM_VERSION} in - llvm35*) - V=3.5 - ;; - llvm36*) - V=3.6 - ;; - llvm37*) - V=3.7 - ;; - llvm38*) - V=3.8 - ;; - llvm39*) - V=3.9 - ;; - llvm4*) - V=4 - ;; - llvm5*) - V=5 - ;; - llvm6*) - V=6 - ;; - llvm7*) - V=7 - ;; - llvm8*) - V=8 - ;; - llvm9*) - V=9 - ;; - llvm10*) - V=10 - ;; - llvm11*) - V=11 - ;; - llvm12*) - V=12 - ;; - llvm13*) - V=13 - ;; llvm14*) V=14 ;; + llvm15*) + V=15 + ;; *) echo "Unknown LLVM version: ${LLVM_VERSION}" exit 1 diff --git a/tests/AArch64/Lift.cpp b/tests/AArch64/Lift.cpp index 2033c3e7..1acb4b60 100644 --- a/tests/AArch64/Lift.cpp +++ b/tests/AArch64/Lift.cpp @@ -123,7 +123,9 @@ extern "C" int main(int argc, char *argv[]) { } llvm::LLVMContext context; +#if LLVM_VERSION_NUMBER < LLVM_VERSION(15, 0) context.enableOpaquePointers(); +#endif auto os_name = remill::GetOSName(REMILL_OS); auto arch_name = remill::GetArchName(FLAGS_arch); auto arch = remill::Arch::Build(&context, os_name, arch_name); diff --git a/tests/AArch64/Run.cpp b/tests/AArch64/Run.cpp index 2b2c8780..c0a5317f 100644 --- a/tests/AArch64/Run.cpp +++ b/tests/AArch64/Run.cpp @@ -143,8 +143,6 @@ MAKE_RW_FP_MEMORY(32) MAKE_RW_FP_MEMORY(64) MAKE_RW_FP_MEMORY(128) -State __remill_state{}; - NEVER_INLINE Memory *__remill_read_memory_f80(Memory *, addr_t addr, native_float80_t &out) { out = AccessMemory(addr); diff --git a/tests/Thumb/TestLifting.cpp b/tests/Thumb/TestLifting.cpp index 544131d0..59db1b8e 100644 --- a/tests/Thumb/TestLifting.cpp +++ b/tests/Thumb/TestLifting.cpp @@ -33,6 +33,7 @@ #include #include #include +#include #include #include @@ -59,7 +60,9 @@ std::optional GetFlows(std::string_view bytes, uint64_t address, uint64_t tm_val) { llvm::LLVMContext context; +#if LLVM_VERSION_NUMBER < LLVM_VERSION(15, 0) context.enableOpaquePointers(); +#endif auto arch = remill::Arch::Build(&context, remill::OSName::kOSLinux, remill::ArchName::kArchAArch32LittleEndian); auto sems = remill::LoadArchSemantics(arch.get()); @@ -237,7 +240,9 @@ TEST(ThumbRandomizedLifts, PopPC) { spec.AddPrecWrite(10, 16); llvm::LLVMContext context; +#if LLVM_VERSION_NUMBER < LLVM_VERSION(15, 0) context.enableOpaquePointers(); +#endif TestSpecRunner runner(context); runner.RunTestSpec(spec); } @@ -254,14 +259,18 @@ TEST(ThumbRandomizedLifts, RelPcTest) { spec.AddPrecWrite(28, 0xdeadc0de); llvm::LLVMContext context; +#if LLVM_VERSION_NUMBER < LLVM_VERSION(15, 0) context.enableOpaquePointers(); +#endif TestSpecRunner runner(context); runner.RunTestSpec(spec); } TEST(RegressionTests, AARCH64RegSize) { llvm::LLVMContext context; +#if LLVM_VERSION_NUMBER < LLVM_VERSION(15, 0) context.enableOpaquePointers(); +#endif auto arch = remill::Arch::Build(&context, remill::OSName::kOSLinux, remill::ArchName::kArchAArch64LittleEndian); auto sems = remill::LoadArchSemantics(arch.get()); @@ -282,7 +291,7 @@ TEST(RegressionTests, AARCH64RegSize) { } -/* These tests are transcribed from the behaviors described in: A2.3.1 +/* These tests are transcribed from the behaviors described in: A2.3.1 MOV(reg, thumb) ignores last bit, but does not mode-switch @@ -300,15 +309,15 @@ TEST(RegressionTests, AARCH64RegSize) { Thumb -> Thumb (1, blx 1) -> {true: 1} Indirects (LDR, MOV(reg,ARM), BX): - + Arm -> (Thumb/Arm) (0, LDR PC, [r0]) -> {true: non_constant} Arm -> (Thumb/Arm) (0, BX r1) -> {true: non_constant} Thumb -> (Thumb/Arm) (1, LDR PC, [0]) -> {true: non_constant} Thumb -> (Thumb/Arm) (1, BX r1) -> {true: non_constant} - + Arm only Arm -> (Thumb/Arm) (0, mov pc, r1) -> {true: non_constant} - + Same but with conditionals: @@ -323,13 +332,13 @@ TEST(RegressionTests, AARCH64RegSize) { Arm -> Arm (0, blxne 1) -> {true: 0} Indirects (LDR, MOV(reg,ARM), BX): - + Arm -> (Thumb/Arm) (0, LDRNE PC, [r0]) -> {!=branch_not_taken_pc: non_constant, ==branch_not_taken_pc: 0} Arm -> (Thumb/Arm) (0, BXNE r1) -> {!=branch_not_taken_pc: non_constant, ==branch_not_taken_pc: 0} - + Arm only Arm -> (Thumb/Arm) (0, movne pc, r1) -> {!=branch_not_taken_pc: non_constant, ==branch_not_taken_pc: 0} - + */ /* @@ -652,4 +661,4 @@ TEST(ArmContextTests, ThumbBPLRegressionTest) { std::get(act_insn.flows); EXPECT_EQ(expect_cond_flow, act_insn.flows); -} \ No newline at end of file +} diff --git a/tests/X86/Lift.cpp b/tests/X86/Lift.cpp index 29faac65..b313f008 100644 --- a/tests/X86/Lift.cpp +++ b/tests/X86/Lift.cpp @@ -37,6 +37,7 @@ #include "remill/BC/IntrinsicTable.h" #include "remill/BC/Lifter.h" #include "remill/BC/Util.h" +#include "remill/BC/Version.h" #include "remill/OS/OS.h" #include "tests/X86/Test.h" @@ -123,7 +124,9 @@ extern "C" int main(int argc, char *argv[]) { } llvm::LLVMContext context; +#if LLVM_VERSION_NUMBER < LLVM_VERSION(15, 0) context.enableOpaquePointers(); +#endif auto os_name = remill::GetOSName(REMILL_OS); auto arch_name = remill::GetArchName(FLAGS_arch); auto arch = remill::Arch::Build(&context, os_name, arch_name); diff --git a/tests/X86/Run.cpp b/tests/X86/Run.cpp index 9ffcab67..d05b8122 100644 --- a/tests/X86/Run.cpp +++ b/tests/X86/Run.cpp @@ -181,8 +181,6 @@ MAKE_RW_FP_MEMORY(64) //MAKE_RW_FP_MEMORY(80) MAKE_RW_FP_MEMORY(128) -State __remill_state{}; - NEVER_INLINE Memory *__remill_read_memory_f80(Memory *, addr_t addr, native_float80_t &out) { out = AccessMemory(addr);