diff --git a/CMakeLists.txt b/CMakeLists.txt index beabae320..a071330f1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -246,20 +246,33 @@ foreach( # `analyze-helper-arguments` can understand. # * `analyze-helper-arguments` discovers, for every helper, which arguments # alias CSVs at the call site. - # * `loop-unroll` unrolls loops so the subsequent `fix-helpers` pass can - # rewrite the accesses contained in bounded loops bodies. + # * `loop-unroll` (x86 only, see below) unrolls loops so the subsequent + # `fix-helpers` pass can rewrite the accesses contained in bounded loop + # bodies. # * `fix-helpers` rewrites helper bodies to access CSVs via the env pointer, # switching on env-relative offsets. # * `detect-uninlinable-helpers` excludes not viable helpers from inlining. + + # Only the x86 softfloat helpers iterate over the vector-register lanes with + # bounded `for` loops (e.g. the packed `helper_*p[sd]` operate on + # `XMM/ZMM_D(i)`). `fix-helpers` can only rewrite the per-lane CSV accesses + # once those loops are unrolled. This step is not needed on other archs, so we + # avoid it to save space on the generated libtcg helpers and build time. + set(EXTRA_HELPER_PASSES "") + if(ARCH STREQUAL "x86_64" OR ARCH STREQUAL "i386") + set(EXTRA_HELPER_PASSES -loop-unroll) + endif() + add_custom_command( OUTPUT "${LIBTCG_HELPERS_FULL}" DEPENDS "${LIBTCG_HELPERS}" revng-all-binaries revngHelperArgumentsAnalysis revngHelperInliningAnalyses COMMAND ./bin/revng opt -mark-inline-helpers-up-to -sroa -instsimplify - -cpu-loop-exit -analyze-helper-arguments -loop-unroll -fix-helpers - -fix-helpers-architecture="${ARCH}" -dce -detect-uninlinable-helpers - "${LIBTCG_HELPERS}" -o "${LIBTCG_HELPERS_FULL}") + -cpu-loop-exit -analyze-helper-arguments ${EXTRA_HELPER_PASSES} + -fix-helpers -fix-helpers-architecture="${ARCH}" -dce + -detect-uninlinable-helpers "${LIBTCG_HELPERS}" -o + "${LIBTCG_HELPERS_FULL}") add_custom_target("annotated-libtcg-helpers-${ARCH}" ALL DEPENDS "${LIBTCG_HELPERS_FULL}") diff --git a/include/revng/FunctionIsolation/InlineHelpers.h b/include/revng/FunctionIsolation/InlineHelpers.h deleted file mode 100644 index a1dea7539..000000000 --- a/include/revng/FunctionIsolation/InlineHelpers.h +++ /dev/null @@ -1,27 +0,0 @@ -#pragma once - -// -// This file is distributed under the MIT License. See LICENSE.md for details. -// - -#include "llvm/Pass.h" - -#include "revng/BasicAnalyses/GeneratedCodeBasicInfo.h" - -class InlineHelpersPass : public llvm::ModulePass { -public: - static char ID; - - // This map stores the loaded helper module(s) to be inlined. Generally state - // in a pass is frowned upon, but since this is only ever read it's simpler to - // store it at the pass level. - llvm::StringMap> HelpersModules; - -public: - InlineHelpersPass() : ModulePass(ID) {} - - bool runOnModule(llvm::Module &M) override; - -private: - void linkRequiredHelpers(llvm::Module &M); -}; diff --git a/include/revng/InlineHelpers/DeleteHelperBodies.h b/include/revng/InlineHelpers/DeleteHelperBodies.h new file mode 100644 index 000000000..862a1b66a --- /dev/null +++ b/include/revng/InlineHelpers/DeleteHelperBodies.h @@ -0,0 +1,18 @@ +#pragma once + +// +// This file is distributed under the MIT License. See LICENSE.md for details. +// + +#include "llvm/IR/Module.h" +#include "llvm/Pass.h" + +class DeleteHelperBodiesPass : public llvm::ModulePass { +public: + static char ID; + +public: + DeleteHelperBodiesPass() : llvm::ModulePass(ID) {} + + bool runOnModule(llvm::Module &M) override; +}; diff --git a/include/revng/InlineHelpers/InlineHelpers.h b/include/revng/InlineHelpers/InlineHelpers.h new file mode 100644 index 000000000..4eabd7db0 --- /dev/null +++ b/include/revng/InlineHelpers/InlineHelpers.h @@ -0,0 +1,25 @@ +#pragma once + +// +// This file is distributed under the MIT License. See LICENSE.md for details. +// + +#include "llvm/IR/Module.h" +#include "llvm/Pass.h" + +#include "revng/BasicAnalyses/GeneratedCodeBasicInfo.h" + +/// Inline every `revng_inline` helper at its call site in the `Isolated` +/// functions, where the critical arguments are constant at the call site. Does +/// not link helper bodies (use `LinkHelpersToInlinePass` first) and does not +/// delete inlined helper bodies (use `DeleteHelperBodiesPass` once at the end +/// of the pipeline). +class InlineHelpersPass : public llvm::ModulePass { +public: + static char ID; + +public: + InlineHelpersPass() : ModulePass(ID) {} + + bool runOnModule(llvm::Module &M) override; +}; diff --git a/include/revng/InlineHelpers/LinkHelpersToInline.h b/include/revng/InlineHelpers/LinkHelpersToInline.h new file mode 100644 index 000000000..c03f71dd4 --- /dev/null +++ b/include/revng/InlineHelpers/LinkHelpersToInline.h @@ -0,0 +1,21 @@ +#pragma once + +// +// This file is distributed under the MIT License. See LICENSE.md for details. +// + +#include "llvm/IR/Module.h" +#include "llvm/Pass.h" + +/// Link the bodies of every `revng_inline`-tagged helper that's still a +/// declaration in `M`, cloning them from the per-architecture +/// `libtcg-helpers-to-inline-.bc` shipped with revng. +class LinkHelpersToInlinePass : public llvm::ModulePass { +public: + static char ID; + +public: + LinkHelpersToInlinePass() : llvm::ModulePass(ID) {} + + bool runOnModule(llvm::Module &M) override; +}; diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt index 25254fedf..081d93ee5 100644 --- a/lib/CMakeLists.txt +++ b/lib/CMakeLists.txt @@ -37,6 +37,7 @@ add_subdirectory(HeadersGeneration) add_subdirectory(HelperArgumentsAnalysis) add_subdirectory(ImportFromC) add_subdirectory(InitModelTypes) +add_subdirectory(InlineHelpers) add_subdirectory(LocalVariables) add_subdirectory(LLMRename) add_subdirectory(Lift) diff --git a/lib/EarlyFunctionAnalysis/CMakeLists.txt b/lib/EarlyFunctionAnalysis/CMakeLists.txt index 1797a0745..603072cfe 100644 --- a/lib/EarlyFunctionAnalysis/CMakeLists.txt +++ b/lib/EarlyFunctionAnalysis/CMakeLists.txt @@ -35,6 +35,7 @@ target_link_libraries( revngEarlyFunctionAnalysis revngBasicAnalyses revngABI + revngInlineHelpers revngSupport revngModel revngPipes diff --git a/lib/EarlyFunctionAnalysis/DetectABI.cpp b/lib/EarlyFunctionAnalysis/DetectABI.cpp index 7fe5a0712..81222049f 100644 --- a/lib/EarlyFunctionAnalysis/DetectABI.cpp +++ b/lib/EarlyFunctionAnalysis/DetectABI.cpp @@ -30,8 +30,10 @@ #include "revng/EarlyFunctionAnalysis/DetectABI.h" #include "revng/EarlyFunctionAnalysis/FunctionEdgeBase.h" #include "revng/EarlyFunctionAnalysis/FunctionSummaryOracle.h" +#include "revng/InlineHelpers/InlineHelpers.h" +#include "revng/InlineHelpers/LinkHelpersToInline.h" #include "revng/Model/Binary.h" -#include "revng/Model/NameBuilder.h" +#include "revng/Model/IRHelpers.h" #include "revng/Model/Pass/DeduplicateCollidingNames.h" #include "revng/Model/Register.h" #include "revng/Pipeline/Pipe.h" @@ -39,6 +41,7 @@ #include "revng/Pipes/Kinds.h" #include "revng/Pipes/ModelGlobal.h" #include "revng/Support/BasicBlockID.h" +#include "revng/Support/CommonOptions.h" #include "revng/Support/Debug.h" #include "revng/Support/IRBuilder.h" #include "revng/Support/IRHelpers.h" @@ -82,6 +85,14 @@ static opt ABIEnforcement("abi-enforcement-level", "found.")), init(ABIOpt::FullABIEnforcement)); +static opt DumpPostInline("detect-abi-dump-post-helpers-inlining", + desc("Path of a file the whole " + "module is written to right " + "after helper inlining inside " + "`analyzeABI` (for debugging " + "and testing)"), + init("")); + static Logger Log("detect-abi"); struct Changes { @@ -107,6 +118,7 @@ public: Manager.add(new LoadModelWrapperPass(ModelWrapper(Global->get()))); Manager.add(new CollectFunctionsFromCalleesWrapperPass()); Manager.add(new ControlFlowGraphCachePass(CFGs)); + Manager.add(new LinkHelpersToInlinePass()); Manager.add(new efa::DetectABIPass()); Manager.add(new CollectFunctionsFromUnusedAddressesWrapperPass()); Manager.add(new efa::DetectABIPass()); @@ -451,7 +463,7 @@ void DetectABI::analyzeABI() { revng_log(Log, "Running ABI analyses"); LoggerIndent Indent(Log); - llvm::Task Task(2, "analyzeABI"); + llvm::Task Task(3, "analyzeABI"); std::map> Functions; // Create all temporary functions @@ -463,6 +475,39 @@ void DetectABI::analyzeABI() { Functions[Function.Entry()] = std::move(NewFunction); } + // Inline `revng_inline`-tagged helper calls so the subsequent ABI dataflow + // analysis can observe the helpers' register reads and writes directly. + // `InlineHelpersPass` only inlines into `Isolated` functions, so we tag the + // outlined stubs as such. These stubs are temporary and are discarded with + // the cloned module once the analysis ends, so the tag never escapes. + // Helpers whose per-call critical arguments are not LLVM constants here + // survive un-inlined and are consumed as opaque calls by the analysis. + for (auto &[Entry, OutlinedFn] : Functions) + FunctionTags::Isolated.addTo(OutlinedFn->Function.get()); + + Task.advance("Inline helpers"); + { + llvm::legacy::PassManager PM; + PM.add(new InlineHelpersPass()); + PM.run(M); + } + + // When `--debug-names` is set, rename each outlined stub from the + // metaaddress-based identifier to its source-symbol name, using the same + // `llvmName` scheme as the rest of the pipeline. + if (DebugNames) { + for (auto &[Entry, OutlinedFn] : Functions) { + llvm::Function *F = OutlinedFn->Function.get(); + F->setName(llvmName(Binary->Functions().at(Entry))); + } + } + + // Dump the whole module right after helper inlining, for debugging and + // testing. The outlined stubs live in the module at this point, so a single + // module dump captures them all. + if (not DumpPostInline.empty()) + dumpModule(&M, DumpPostInline.c_str()); + // Push this into analyzeFunction OpaqueRegisterUser RegisterUser(&M); @@ -482,11 +527,10 @@ void DetectABI::analyzeABI() { } unsigned Runs = 0; - model::CNameBuilder NameBuilder = *Binary; while (not ToAnalyze.empty()) { model::Function &Function = *ToAnalyze.pop(); revng_log(Log, "Analyzing " << Function.Entry().toString()); - FixedPointTask.advance(NameBuilder.name(Function)); + FixedPointTask.advance(llvmName(Function)); OutlinedFunction &OutlinedFunction = *Functions.at(Function.Entry()); Changes Changes = analyzeFunctionABI(Function, OutlinedFunction, @@ -1117,6 +1161,14 @@ llvm::Error DetectABI::run(Model &Model, revng::pipes::CFGMap CFGs(""); ControlFlowGraphCache FMC(CFGs); + // Link helper bodies once at the start of the analysis, to avoid relinking + // for every `inline-helpers` call + { + llvm::legacy::PassManager PM; + PM.add(new LinkHelpersToInlinePass()); + PM.run(Module); + } + collectFunctionsFromCallees(Module, GCBI, Binary); efa::runDetectABI(Module, GCBI, FMC, TupleModel); collectFunctionsFromUnusedAddresses(Module, GCBI, Binary, FMC); diff --git a/lib/FunctionIsolation/CMakeLists.txt b/lib/FunctionIsolation/CMakeLists.txt index 1eb21b6e8..fa16096bc 100644 --- a/lib/FunctionIsolation/CMakeLists.txt +++ b/lib/FunctionIsolation/CMakeLists.txt @@ -5,10 +5,8 @@ revng_add_analyses_library_internal( revngFunctionIsolation EnforceABI.cpp - InlineHelpers.cpp InvokeIsolatedFunctions.cpp IsolateFunctions.cpp - PostInlineHelpersVerifyPass.cpp PromoteCSVs.cpp RemoveExceptionalCalls.cpp StripDebugInfoFromHelpers.cpp @@ -21,6 +19,7 @@ target_link_libraries( revngABI revngModel revngEarlyFunctionAnalysis + revngInlineHelpers revngSupport revngPipes ${LLVM_LIBRARIES}) diff --git a/lib/InlineHelpers/CMakeLists.txt b/lib/InlineHelpers/CMakeLists.txt new file mode 100644 index 000000000..788142d1f --- /dev/null +++ b/lib/InlineHelpers/CMakeLists.txt @@ -0,0 +1,16 @@ +# +# This file is distributed under the MIT License. See LICENSE.md for details. +# + +# The inline-helpers chain (link / inline / delete), exposed both as LLVM passes +# and as free functions. It lives in its own library so that +# EarlyFunctionAnalysis can depend on it without creating a cycle with +# FunctionIsolation, which depends on EarlyFunctionAnalysis. +revng_add_analyses_library_internal( + revngInlineHelpers DeleteHelperBodies.cpp InlineHelpers.cpp + LinkHelpersToInline.cpp PostInlineHelpersVerifyPass.cpp) + +llvm_map_components_to_libnames(LLVM_LIBRARIES TransformUtils) + +target_link_libraries(revngInlineHelpers revngBasicAnalyses revngModel + revngSupport ${LLVM_LIBRARIES}) diff --git a/lib/InlineHelpers/DeleteHelperBodies.cpp b/lib/InlineHelpers/DeleteHelperBodies.cpp new file mode 100644 index 000000000..03b515362 --- /dev/null +++ b/lib/InlineHelpers/DeleteHelperBodies.cpp @@ -0,0 +1,41 @@ +// +// This file is distributed under the MIT License. See LICENSE.md for details. +// + +#include "llvm/IR/Function.h" +#include "llvm/IR/Module.h" +#include "llvm/Pass.h" + +#include "revng/InlineHelpers/DeleteHelperBodies.h" +#include "revng/Support/IRHelpers.h" + +#include "PostInlineHelpersVerifyPass.h" + +using namespace llvm; + +// Drop the body of every `revng_inline`-tagged function in `M`, turning each +// one back into a declaration. +static void deleteHelperBodies(llvm::Module &M) { + for (llvm::Function &F : M) { + if (F.getSection() == InlineHelpersSection) + deleteOnlyBody(F); + } +} + +char DeleteHelperBodiesPass::ID = 0; + +using Register = RegisterPass; +static Register + X("delete-helper-bodies", "Delete Helper Bodies Pass", true, true); + +bool DeleteHelperBodiesPass::runOnModule(llvm::Module &M) { + deleteHelperBodies(M); + + // Verify that no `getelementptr` instruction was dragged into an Isolated + // function by the inlining performed by the upstream `-inline-helpers` + // pass. The downstream pipeline relies on this invariant. + // TODO: convert this from a pass to a free-standing function + PostInlineHelpersVerifyPass{}.runOnModule(M); + + return true; +} diff --git a/lib/FunctionIsolation/InlineHelpers.cpp b/lib/InlineHelpers/InlineHelpers.cpp similarity index 66% rename from lib/FunctionIsolation/InlineHelpers.cpp rename to lib/InlineHelpers/InlineHelpers.cpp index 1fa4dff3e..3248b3621 100644 --- a/lib/FunctionIsolation/InlineHelpers.cpp +++ b/lib/InlineHelpers/InlineHelpers.cpp @@ -7,7 +7,6 @@ #include "llvm/ADT/BitVector.h" #include "llvm/ADT/SmallVector.h" -#include "llvm/ADT/StringSet.h" #include "llvm/IR/Instructions.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/FileSystem.h" @@ -15,16 +14,10 @@ #include "llvm/Support/raw_ostream.h" #include "llvm/Transforms/Utils/Cloning.h" -#include "revng/ADT/Queue.h" -#include "revng/BasicAnalyses/GeneratedCodeBasicInfo.h" -#include "revng/FunctionIsolation/InlineHelpers.h" +#include "revng/InlineHelpers/InlineHelpers.h" #include "revng/Support/CommandLine.h" #include "revng/Support/Debug.h" #include "revng/Support/IRHelpers.h" -#include "revng/Support/OpaqueFunctionsPool.h" -#include "revng/Support/ResourceFinder.h" - -#include "PostInlineHelpersVerifyPass.h" using namespace llvm; @@ -194,67 +187,9 @@ void InlineHelpers::run(Function *F) { dumpFunctionToDir(*F, DumpAfterDir); } -void InlineHelpersPass::linkRequiredHelpers(llvm::Module &M) { - NamedMDNode *Node = M.getNamedMetadata(QemuArchitectureMD); - revng_assert(Node != nullptr); - revng_assert(Node->getNumOperands() > 0); - - // Multiple linkings could have lead to this node having more than one - // operand, if so check that they are all the same. - llvm::StringSet Strings; - for (size_t I = 0; I < Node->getNumOperands(); I++) { - revng_assert(Node->getOperand(0)->getNumOperands() == 1); - const MDOperand &StringNode = Node->getOperand(0)->getOperand(0); - Strings.insert(cast(StringNode)->getString()); - } - - revng_assert(Strings.size() == 1, - "QEMU helpers of multiple architectures were linked inside the " - "module"); - - // Collect the names of every `revng_inline` helper referenced in `M`; - // they all lack a body in `M` and need to be cloned from the - // per-architecture to-inline bitcode. - SmallVector HelpersToInline; - for (llvm::Function &F : M) { - if (F.isDeclaration() and F.getSection() == InlineHelpersSection) - HelpersToInline.push_back(F.getName()); - } - - // Skip the expensive bitcode parse + clone + link if there's nothing to do. - if (HelpersToInline.empty()) - return; - - // Given the architecture MD, retrieve the correct libtcg module (if not - // already in the `HelpersModules` map) and link it into the main module - StringRef ArchName = Strings.begin()->first(); - if (HelpersModules.count(ArchName) == 0) { - const std::string LibHelpersName = ("/share/revng/" - "libtcg-helpers-to-inline-" - + ArchName + ".bc") - .str(); - auto OptionalHelpers = revng::ResourceFinder.findFile(LibHelpersName); - revng_assert(OptionalHelpers.has_value(), "Cannot find libtcg helpers"); - HelpersModules[ArchName] = parseIR(M.getContext(), OptionalHelpers.value()); - } - - llvm::Module &HelpersModule = *HelpersModules[ArchName]; - std::set ToClone; - for (StringRef Name : HelpersToInline) { - llvm::Function *HelperFunction = HelpersModule.getFunction(Name); - revng_assert(HelperFunction != nullptr - and not HelperFunction->isDeclaration(), - "revng_inline helper missing body in to-inline.bc"); - ToClone.insert(HelperFunction); - } - - auto ClonedHelpersModule = ::cloneFiltered(HelpersModule, ToClone); - linkModules(std::move(ClonedHelpersModule), M); -} - bool InlineHelpersPass::runOnModule(llvm::Module &M) { - linkRequiredHelpers(M); - + // Inline `revng_inline` helper calls into the `Isolated` functions, where the + // per-call critical arguments are constant at the call site. SmallVector Isolated; for (Function &F : M) if (FunctionTags::Isolated.isTagOf(&F)) @@ -267,18 +202,5 @@ bool InlineHelpersPass::runOnModule(llvm::Module &M) { IH.run(F); } - // Since we're done inlining helpers they are no longer needed, delete their - // body - for (Function &F : M) { - if (F.getSection() == InlineHelpersSection) - deleteOnlyBody(F); - } - - // Verify that no `getelementptr` instruction was dragged into an Isolated - // function by the inlining above. The downstream pipeline relies on this - // invariant. - // TODO: convert this from a pass to a free-standing function - PostInlineHelpersVerifyPass{}.runOnModule(M); - return true; } diff --git a/lib/InlineHelpers/LinkHelpersToInline.cpp b/lib/InlineHelpers/LinkHelpersToInline.cpp new file mode 100644 index 000000000..7c6aebae3 --- /dev/null +++ b/lib/InlineHelpers/LinkHelpersToInline.cpp @@ -0,0 +1,92 @@ +// +// This file is distributed under the MIT License. See LICENSE.md for details. +// + +#include +#include +#include + +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/ADT/StringSet.h" +#include "llvm/IR/Function.h" +#include "llvm/IR/Metadata.h" +#include "llvm/IR/Module.h" +#include "llvm/Pass.h" + +#include "revng/InlineHelpers/LinkHelpersToInline.h" +#include "revng/Support/Assert.h" +#include "revng/Support/IRHelpers.h" +#include "revng/Support/ResourceFinder.h" + +using namespace llvm; + +char LinkHelpersToInlinePass::ID = 0; + +using Register = RegisterPass; +static Register X("link-helpers-to-inline", + "Link Required Helpers To Inline Pass", + true, + true); + +bool LinkHelpersToInlinePass::runOnModule(llvm::Module &M) { + NamedMDNode *Node = M.getNamedMetadata(QemuArchitectureMD); + revng_assert(Node != nullptr); + revng_assert(Node->getNumOperands() > 0); + + // Multiple linkings could have lead to this node having more than one + // operand, if so check that they are all the same. + llvm::StringSet Strings; + for (size_t I = 0; I < Node->getNumOperands(); I++) { + revng_assert(Node->getOperand(0)->getNumOperands() == 1); + const MDOperand &StringNode = Node->getOperand(0)->getOperand(0); + Strings.insert(cast(StringNode)->getString()); + } + + revng_assert(Strings.size() == 1, + "QEMU helpers of multiple architectures were linked inside the " + "module"); + + // Collect the names of revng_inline helpers that lack a body in `M` and + // therefore need to be cloned from the per-architecture to-inline bitcode. + SmallVector MissingBodies; + for (llvm::Function &F : M) { + if (F.isDeclaration() and F.getSection() == InlineHelpersSection) { + MissingBodies.push_back(F.getName()); + } + } + + // Nothing to link — every referenced revng_inline helper already has a + // body in `M`. Skip the expensive bitcode parse + clone + link in this case. + if (MissingBodies.empty()) { + return true; + } + + // Given the architecture MD, retrieve the correct libtcg module and link + // it into the main module. + StringRef ArchName = Strings.begin()->first(); + const std::string LibHelpersName = ("/share/revng/" + "libtcg-helpers-to-inline-" + + ArchName + ".bc") + .str(); + auto OptionalHelpers = revng::ResourceFinder.findFile(LibHelpersName); + revng_assert(OptionalHelpers.has_value(), "Cannot find libtcg helpers"); + std::unique_ptr + HelpersModule = parseIR(M.getContext(), OptionalHelpers.value()); + + std::set ToClone; + for (StringRef Name : MissingBodies) { + llvm::Function *HelperFunction = HelpersModule->getFunction(Name); + revng_assert(HelperFunction != nullptr + and not HelperFunction->isDeclaration(), + "revng_inline helper missing body in to-inline.bc; the " + "static-policy / slim-down chain produced an " + "inconsistent state"); + ToClone.insert(HelperFunction); + } + + auto ClonedHelpersModule = ::cloneFiltered(*HelpersModule, ToClone); + linkModules(std::move(ClonedHelpersModule), M); + + return true; +} diff --git a/lib/FunctionIsolation/PostInlineHelpersVerifyPass.cpp b/lib/InlineHelpers/PostInlineHelpersVerifyPass.cpp similarity index 100% rename from lib/FunctionIsolation/PostInlineHelpersVerifyPass.cpp rename to lib/InlineHelpers/PostInlineHelpersVerifyPass.cpp diff --git a/lib/FunctionIsolation/PostInlineHelpersVerifyPass.h b/lib/InlineHelpers/PostInlineHelpersVerifyPass.h similarity index 100% rename from lib/FunctionIsolation/PostInlineHelpersVerifyPass.h rename to lib/InlineHelpers/PostInlineHelpersVerifyPass.h diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index 5984081f3..e3dc772b0 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -351,6 +351,7 @@ set(REVNG_CLI_COMMANDS_MODULE_FILES revng/internal/cli/_commands/test_docs/__init__.py revng/internal/cli/_commands/test_docs/doctest_runner.py revng/internal/cli/_commands/trace_run.py + revng/internal/cli/_commands/merge-irs.py revng/internal/cli/_commands/model_compare.py revng/internal/cli/_commands/model_migrate.py revng/internal/cli/_commands/model_export_sqlite.py diff --git a/python/revng/internal/cli/_commands/merge-irs.py b/python/revng/internal/cli/_commands/merge-irs.py new file mode 100644 index 000000000..ca3adecba --- /dev/null +++ b/python/revng/internal/cli/_commands/merge-irs.py @@ -0,0 +1,96 @@ +# +# This file is distributed under the MIT License. See LICENSE.md for details. +# + +import tarfile +from argparse import ArgumentParser, RawDescriptionHelpFormatter +from base64 import b64decode +from contextlib import suppress +from pathlib import Path +from shutil import copyfileobj +from tempfile import TemporaryDirectory, TemporaryFile +from typing import IO + +import yaml + +from revng.internal.cli.commands_registry import Command, CommandsRegistry, Options +from revng.internal.cli.support import build_command_with_loads, file_wrapper, try_run +from revng.support import TarDictionary + + +def read_single_file(input_: IO[bytes]) -> list[bytes]: + # If we encounter a non-seekable file (e.g. stdin) first copy it to a + # temporary file and then re-run this function + if not input_.seekable(): + with TemporaryFile() as temp_file: + copyfileobj(input_, temp_file) + temp_file.seek(0) + return read_single_file(temp_file) + + # If here, the file is seekable, test the various formats by trying to read + # them and moving onto the next format if the previous throws an error + + # Test for YAML + with suppress(yaml.YAMLError): + data = yaml.load(input_, Loader=yaml.CSafeLoader) + return [b64decode(v) for v in data.values()] + + # Test for tar + input_.seek(0) + with suppress(tarfile.ReadError): + return list(TarDictionary(input_).values()) + + # No good format found, assume it's a single plain file + input_.seek(0) + return [input_.read()] + + +class MergeLLVMIRCommand(Command): + def __init__(self): + super().__init__(("merge", "llvm"), "Merge multiple LLVM IR files into a single one") + + def register_arguments(self, parser: ArgumentParser): + parser.formatter_class = RawDescriptionHelpFormatter + parser.description = """Links multiple LLVM IR files into a single one. +This program works differently depending on the number of inputs: +* If one input is passed it is interpreted as either: + * a YAML dict with base64-encoded values + * a tar where each file is a bitcode file + * a single LLVM module +* If no input is passed, treat stdin as a single input +* If multiple input files are passed, treat all of them as bitcode files +""" + parser.add_argument("-o", "--output", help="Where to output the file, stdout if omitted") + parser.add_argument("inputs", nargs="*", help="Input file(s), omit to use stdin") + + def run(self, options: Options): + args = options.parsed_args + if len(args.inputs) >= 2: + return self.run_merge(args.inputs, options) + + input_file = args.inputs[0] if len(args.inputs) == 1 else "-" + with file_wrapper(input_file, "rb") as input_: + data = read_single_file(input_) + + with TemporaryDirectory(prefix="tmp.revng-merge-llvm.") as temp_dir: + temp_dir_path = Path(temp_dir) + input_paths = [] + for index, value in enumerate(data): + element_path = temp_dir_path / str(index) + element_path.write_bytes(value) + input_paths.append(str(element_path.resolve())) + + return self.run_merge(input_paths, options) + + def run_merge(self, inputs: list[str], options: Options): + with file_wrapper(options.parsed_args.output, "wb") as output_file: + return try_run( + build_command_with_loads("merge-llvm", inputs, options), + options, + stdout=output_file, + ) + + +def setup(commands_registry: CommandsRegistry): + commands_registry.define_namespace(("merge",), "Tools to merge function-wise object together") + commands_registry.register_command(MergeLLVMIRCommand()) diff --git a/python/revng/internal/cli/support.py b/python/revng/internal/cli/support.py index 1e4d8effa..0b9e4acd8 100644 --- a/python/revng/internal/cli/support.py +++ b/python/revng/internal/cli/support.py @@ -2,6 +2,8 @@ # This file is distributed under the MIT License. See LICENSE.md for details. # +from __future__ import annotations + import os import shlex import signal @@ -10,8 +12,8 @@ from contextlib import contextmanager from dataclasses import dataclass from subprocess import Popen from tempfile import NamedTemporaryFile -from typing import Any, Callable, Dict, Iterable, List, Literal, Mapping, NoReturn, Optional -from typing import Tuple, Union +from typing import TYPE_CHECKING, Any, Callable, Dict, Iterable, List, Literal, Mapping, NoReturn +from typing import Optional, Tuple, Union import yaml @@ -19,6 +21,9 @@ from revng.internal.support.collect import collect_libraries from revng.internal.support.elf import is_executable from revng.support import TarDictionary, get_command +if TYPE_CHECKING: + from _typeshed import FileDescriptorLike + OptionalEnv = Optional[Mapping[str, str]] @@ -74,7 +79,7 @@ def _run_common( return command, environment -def popen(command, options: Options, environment: OptionalEnv = None, **kwargs) -> int | Popen: +def _popen(command, options: Options, environment: OptionalEnv = None, **kwargs) -> int | Popen: command, environment = _run_common(command, options, environment) if options.dry_run: @@ -83,15 +88,36 @@ def popen(command, options: Options, environment: OptionalEnv = None, **kwargs) return Popen(command, env=environment, **kwargs) -def try_run(command, options: Options, environment: OptionalEnv = None) -> int: +def popen( + command, + options: Options, + environment: OptionalEnv = None, + stdin: FileDescriptorLike | None = None, + stdout: FileDescriptorLike | None = None, + stderr: FileDescriptorLike | None = None, +) -> int | Popen: + return _popen(command, options, environment, stdin=stdin, stdout=stdout, stderr=stderr) + + +def try_run( + command, + options: Options, + environment: OptionalEnv = None, + stdin: FileDescriptorLike | None = None, + stdout: FileDescriptorLike | None = None, + stderr: FileDescriptorLike | None = None, +) -> int: try: signal.signal(signal.SIGINT, signal.SIG_IGN) - process = popen( + process = _popen( command, options, environment, preexec_fn=lambda: signal.signal(signal.SIGINT, signal.SIG_DFL), close_fds=False, + stdin=stdin, + stdout=stdout, + stderr=stderr, ) if isinstance(process, int): return process @@ -100,8 +126,15 @@ def try_run(command, options: Options, environment: OptionalEnv = None) -> int: signal.signal(signal.SIGINT, signal.SIG_DFL) -def run(command, options: Options, environment: OptionalEnv = None): - result = try_run(command, options, environment) +def run( + command, + options: Options, + environment: OptionalEnv = None, + stdin: FileDescriptorLike | None = None, + stdout: FileDescriptorLike | None = None, + stderr: FileDescriptorLike | None = None, +): + result = try_run(command, options, environment, stdin, stdout, stderr) if result != 0: sys.exit(result) diff --git a/python/revng/internal/pipeline.yml b/python/revng/internal/pipeline.yml index 007c21444..d14d8ef10 100644 --- a/python/revng/internal/pipeline.yml +++ b/python/revng/internal/pipeline.yml @@ -178,7 +178,9 @@ branches: configuration: passes: - mem2reg + - link-helpers-to-inline - inline-helpers + - delete-helper-bodies - pipe: attach-debug-info arguments: [cfg-map, llvm-functions] - pipe: promote-csvs @@ -362,6 +364,11 @@ branches: - dce - savepoint: segregate-stack-accesses containers: [llvm-functions] + artifacts: + - name: segregate-stack-accesses + container: llvm-functions + category: debug + filename: stack_accesses_segregated.bc - pipe: pure-llvm-passes-pipe arguments: [llvm-functions] configuration: diff --git a/share/revng/pipelines/revng-pipelines.yml b/share/revng/pipelines/revng-pipelines.yml index e5b2ae528..d92e6fdd7 100644 --- a/share/revng/pipelines/revng-pipelines.yml +++ b/share/revng/pipelines/revng-pipelines.yml @@ -170,7 +170,9 @@ Branches: # `cc_compute_c` and `cc_compute_all`. - promote-csvs - mem2reg + - link-helpers-to-inline - inline-helpers + - delete-helper-bodies - Type: attach-debug-info-to-abi-enforced UsedContainers: [cfg.yml.tar.gz, functions.bc.zstd] - Type: llvm-pipe diff --git a/share/revng/test/configuration/revng/detect-stack-size.yml b/share/revng/test/configuration/revng/detect-stack-size.yml index ed6aed2b6..ce5bd0606 100644 --- a/share/revng/test/configuration/revng/detect-stack-size.yml +++ b/share/revng/test/configuration/revng/detect-stack-size.yml @@ -6,13 +6,14 @@ commands: # # Run detect-stack-size and test against ground truth # - # TODO: this crashes in the new pipeline, investigate - type: revng.test-detect-stack-size from: - type: revng-qa.compiled-with-debug-info filter: for-detect-stack-size suffix: / command: |- - revng analyze --resume "$OUTPUT" import-binary "$INPUT" -o /dev/null; - revng analyze --resume "$OUTPUT" detect-abi "$INPUT" -o /dev/null; - revng analyze --resume "$OUTPUT" detect-stack-size "$INPUT" | revng model compare "${SOURCE}.model.yml" -; + cd "$OUTPUT"; + revng2 project init --no-initial-auto-analysis "$INPUT"; + revng2 project analyze parse-binary -o /dev/null; + revng2 project analyze detect-abi -o /dev/null; + revng2 project analyze detect-stack-size | revng model compare "${SOURCE}.model.yml" -; diff --git a/share/revng/test/configuration/revng/floating-point-helpers-inlining.yml b/share/revng/test/configuration/revng/floating-point-helpers-inlining.yml new file mode 100644 index 000000000..279026a63 --- /dev/null +++ b/share/revng/test/configuration/revng/floating-point-helpers-inlining.yml @@ -0,0 +1,49 @@ +# +# This file is distributed under the MIT License. See LICENSE.md for details. +# + +# End-to-end tests that verify the emergence of the softfloat calls, on the same +# `for-floating-point-helpers-inlining` ground truth, at two points of the +# pipeline: +# * `revng.test-floating-point-helpers-inlining-in-enforce-abi`: +# in the `enforce-abi` artifact; +# * `revng.test-floating-point-helpers-inlining-in-detect-abi`: +# right after helper inlining inside `DetectABI::analyzeABI()`. +# We pass the `--debug-names` flag to easily anchor the `CHECK` directives for +# multiple function names. +# The CHECK-64 prefix is used for 64-bit-only tests. + +commands: + - type: revng.test-floating-point-helpers-inlining-in-enforce-abi + from: + - type: revng-qa.compiled-with-debug-info + # i386 is excluded until x87 FPU support lands. + filter: for-floating-point-helpers-inlining and !i386 + # `revng merge llvm` merges the per-function IR modules emitted by the + # `--tar` artifact into a single module, so we can `FileCheck` them all at + # once. + # The `sed` below strips the `local_` prefix for a coherent FileCheck file. + command: |- + PREFIXES=CHECK ; + [ "$POINTER_SIZE" = "64" ] && PREFIXES=$$PREFIXES,CHECK-64 ; + revng2 quick artifact enforce-abi "$INPUT" --tar -- --debug-names | + revng merge llvm | + sed 's,@local_,@,g' | + FileCheck --check-prefixes=$$PREFIXES "${SOURCE}".filecheck + + - type: revng.test-floating-point-helpers-inlining-in-detect-abi + from: + - type: revng-qa.compiled-with-debug-info + # i386 is excluded until x87 FPU support lands. + filter: for-floating-point-helpers-inlining and !i386 + # The `sed` below strips the `local_` prefix for a coherent FileCheck file. + command: |- + DUMP="$$(temp)" ; + revng2 quick analyze --analyses=parse-binary,detect-abi "$INPUT" + -- --debug-names + --detect-abi-dump-post-helpers-inlining="$$DUMP" + > /dev/null ; + PREFIXES=CHECK ; + [ "$POINTER_SIZE" = "64" ] && PREFIXES=$$PREFIXES,CHECK-64 ; + sed 's,@local_,@,g' "$$DUMP" | + FileCheck --check-prefixes=$$PREFIXES "${SOURCE}".filecheck diff --git a/share/revng/test/configuration/revng/floating-point.yml b/share/revng/test/configuration/revng/floating-point.yml deleted file mode 100644 index da1e32e6c..000000000 --- a/share/revng/test/configuration/revng/floating-point.yml +++ /dev/null @@ -1,29 +0,0 @@ -# -# This file is distributed under the MIT License. See LICENSE.md for details. -# - -# End-to-end tests which verify the emergence of the softfloat calls in the -# `enforce-abi` artifact. -# We pass the `--debug-names` flag to easily anchor the `CHECK` directives for -# multiple function names. -# We also strip the `local_` prefix to have a more coherent FileCheck file. -# Since we use the `revng2` pipeline, we need to extract all the generated IRs -# for each isolated function and to concatenated their content to easily match -# the CHECK directives. -# The CHECK-64 prefix is used for 64-bit arch only tests. - -commands: - - type: revng.floating-point - from: - - type: revng-qa.compiled-with-debug-info - filter: for-floating-point and !i386 - command: |- - PROJECT="$$(temp -d)" ; - revng2 -C "$$PROJECT" project init "$INPUT" -- --debug-names ; - revng2 -C "$$PROJECT" project artifact enforce-abi --format yaml -o "$$PROJECT/out.yml" -- --debug-names ; - PREFIXES=CHECK ; - file "$INPUT" | grep -q "ELF 64-bit" && PREFIXES=$$PREFIXES,CHECK-64 ; - yq -r 'to_entries[].value' "$$PROJECT/out.yml" - | while IFS= read -r b64 ; do printf '%s' "$$b64" | base64 -d | revng opt -S /dev/stdin -o - ; done - | sed 's,@local_,@,g' - | FileCheck --check-prefixes=$$PREFIXES "${SOURCE}".filecheck diff --git a/share/revng/test/configuration/revng/segregate-stack-accesses.yml b/share/revng/test/configuration/revng/segregate-stack-accesses.yml index 7bacec5c8..df623f15f 100644 --- a/share/revng/test/configuration/revng/segregate-stack-accesses.yml +++ b/share/revng/test/configuration/revng/segregate-stack-accesses.yml @@ -3,29 +3,31 @@ # commands: - # TODO: this crashes with the new pipeline, investigate - type: revng.test-segregate-stack-accesses from: - type: revng-qa.compiled-with-debug-info filter: for-segregate-stack-accesses suffix: / command: |- + cd "$OUTPUT"; export REVNG_OPTIONS="$${REVNG_OPTIONS:-} --debug-names"; - revng analyze --resume "$OUTPUT" import-binary "$INPUT" -o /dev/null; + revng2 project init --no-initial-auto-analysis "$INPUT"; + revng2 project analyze parse-binary -o /dev/null; revng model override-by-name - "$OUTPUT/context/model.yml" + model.yml ${SOURCE}.override.$QEMU_NAME.yml - > "$OUTPUT/context/model-tmp.yml"; - mv "$OUTPUT/context/model-tmp.yml" "$OUTPUT/context/model.yml"; + > model-tmp.yml; + mv model-tmp.yml model.yml; + rm -r .cache; - revng analyze --resume "$OUTPUT" detect-abi "$INPUT" -o /dev/null; + revng2 project analyze detect-abi -o /dev/null; + revng2 project analyze detect-stack-size -o /dev/null; - revng analyze --resume "$OUTPUT" detect-stack-size "$INPUT" -o /dev/null; - - revng artifact --resume "$OUTPUT" segregate-stack-accesses "$INPUT" | + revng2 project artifact segregate-stack-accesses --tar | + revng merge llvm | revng opt -remove-llvmassume-calls -instcombine-noarrays -sroa-noarrays -drop-redundant-ptrint-casts -dce -type-shrinking -dce -drop-redundant-ptrint-casts -early-cse -instsimplify -dce -S | FileCheck --check-prefix=CHECK-$QEMU_NAME ${SOURCE}.filecheck.ll; - revng artifact --resume "$OUTPUT" emit-c "$INPUT" -o /dev/null; + revng2 project artifact emit-c -o /dev/null; diff --git a/share/revng/test/configuration/revng/simplify-switch.yml b/share/revng/test/configuration/revng/simplify-switch.yml index 294b3aac5..5266df7a1 100644 --- a/share/revng/test/configuration/revng/simplify-switch.yml +++ b/share/revng/test/configuration/revng/simplify-switch.yml @@ -11,15 +11,6 @@ commands: - type: revng-qa.compiled-with-debug-info filter: for-simplify-switch command: |- - WORKDIR=$$(temp -d); - revng2 quick artifact simplify-switch "$INPUT" - --tar -o "$$WORKDIR/simplify-switch.tar" - -- --abi-enforcement-level=no; - - tar -tf "$$WORKDIR/simplify-switch.tar" 2>/dev/null | - while IFS= read -r ENTRY; do - tar -xf "$$WORKDIR/simplify-switch.tar" "$$ENTRY" --to-stdout 2>/dev/null | - revng opt -S >> "$$WORKDIR/simplify-switch.ll"; - done; - - FileCheck "${SOURCE}.filecheck" < "$$WORKDIR/simplify-switch.ll"; + revng2 quick artifact simplify-switch "$INPUT" --tar -- --abi-enforcement-level=no | + revng merge llvm | + FileCheck "${SOURCE}.filecheck"; diff --git a/tests/filecheck/llvm-passes/inline-helpers.ll b/tests/filecheck/llvm-passes/inline-helpers.ll index 34c491c84..8ad19fc7f 100644 --- a/tests/filecheck/llvm-passes/inline-helpers.ll +++ b/tests/filecheck/llvm-passes/inline-helpers.ll @@ -8,7 +8,7 @@ ; where every listed argument is a compile-time constant. A helper ; with a zero policy must always be inlined. ; -; RUN: %root/bin/revng opt -inline-helpers %s -S -o - | FileCheck %s +; RUN: %root/bin/revng opt -link-helpers-to-inline -inline-helpers -delete-helper-bodies %s -S -o - | FileCheck %s !revng.qemu_architecture = !{!0} !0 = !{!"x86_64"} diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index 17115ee40..779830cc5 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -5,6 +5,7 @@ add_subdirectory(clift-opt) add_subdirectory(internal) add_subdirectory(lddtree) +add_subdirectory(merge-llvm) add_subdirectory(model) add_subdirectory(pipeline) add_subdirectory(pypeline) diff --git a/tools/merge-llvm/CMakeLists.txt b/tools/merge-llvm/CMakeLists.txt new file mode 100644 index 000000000..3f20b0fbc --- /dev/null +++ b/tools/merge-llvm/CMakeLists.txt @@ -0,0 +1,7 @@ +# +# This file is distributed under the MIT License. See LICENSE.md for details. +# + +revng_add_executable(merge-llvm Main.cpp) + +target_link_libraries(merge-llvm revngSupport ${LLVM_LIBRARIES}) diff --git a/tools/merge-llvm/Main.cpp b/tools/merge-llvm/Main.cpp new file mode 100644 index 000000000..f99b60e88 --- /dev/null +++ b/tools/merge-llvm/Main.cpp @@ -0,0 +1,56 @@ +// +// This file is distributed under the MIT License. See LICENSE.md for details. +// + +#include "llvm/IR/Module.h" +#include "llvm/IRReader/IRReader.h" +#include "llvm/Support/Error.h" +#include "llvm/Support/PluginLoader.h" +#include "llvm/Support/SourceMgr.h" + +#include "revng/Support/CommandLine.h" +#include "revng/Support/Error.h" +#include "revng/Support/IRHelpers.h" +#include "revng/Support/InitRevng.h" +#include "revng/Support/Tar.h" + +static llvm::cl::list Arguments(llvm::cl::Positional, + llvm::cl::ZeroOrMore, + llvm::cl::desc("FILES"), + llvm::cl::cat(MainCategory)); + +static llvm::ExitOnError AbortOnError; + +int main(int argc, char *argv[]) { + revng::InitRevng X(argc, argv, "", { &MainCategory }); + + llvm::LLVMContext Context; + std::unique_ptr OutputModule; + + for (llvm::StringRef Filename : Arguments) { + llvm::SMDiagnostic Err; + auto Module = llvm::parseIRFile(Filename, Err, Context); + revng_assert(Module != nullptr); + + if (OutputModule != nullptr) { + // Make the existing module be linked to the new module, this allows + // preserving the order of definitions as the order of the supplied files + std::swap(Module, OutputModule); + linkFunctionModules(std::move(Module), OutputModule); + } else { + OutputModule = std::move(Module); + } + } + + revng_assert(OutputModule != nullptr); + revng::verify(OutputModule.get()); + { + std::error_code EC; + llvm::raw_fd_ostream OS("-", EC); + revng_assert(not EC); + + OutputModule->print(OS, nullptr); + } + + return EXIT_SUCCESS; +}