Merge branch 'feature/detect-abi-inline-helpers'

CMakeLists.txt

@@ -246,20 +246,33 @@ foreach(
   #   `analyze-helper-arguments` can understand.
   # * `analyze-helper-arguments` discovers, for every helper, which arguments
   #   alias CSVs at the call site.
-  # * `loop-unroll` unrolls loops so the subsequent `fix-helpers` pass can
-  #   rewrite the accesses contained in bounded loops bodies.
+  # * `loop-unroll` (x86 only, see below) unrolls loops so the subsequent
+  #   `fix-helpers` pass can rewrite the accesses contained in bounded loop
+  #   bodies.
   # * `fix-helpers` rewrites helper bodies to access CSVs via the env pointer,
   #   switching on env-relative offsets.
   # * `detect-uninlinable-helpers` excludes not viable helpers from inlining.
+
+  # Only the x86 softfloat helpers iterate over the vector-register lanes with
+  # bounded `for` loops (e.g. the packed `helper_*p[sd]` operate on
+  # `XMM/ZMM_D(i)`). `fix-helpers` can only rewrite the per-lane CSV accesses
+  # once those loops are unrolled. This step is not needed on other archs, so we
+  # avoid it to save space on the generated libtcg helpers and build time.
+  set(EXTRA_HELPER_PASSES "")
+  if(ARCH STREQUAL "x86_64" OR ARCH STREQUAL "i386")
+    set(EXTRA_HELPER_PASSES -loop-unroll)
+  endif()
+
   add_custom_command(
     OUTPUT "${LIBTCG_HELPERS_FULL}"
     DEPENDS "${LIBTCG_HELPERS}" revng-all-binaries revngHelperArgumentsAnalysis
             revngHelperInliningAnalyses
     COMMAND
       ./bin/revng opt -mark-inline-helpers-up-to -sroa -instsimplify
-      -cpu-loop-exit -analyze-helper-arguments -loop-unroll -fix-helpers
-      -fix-helpers-architecture="${ARCH}" -dce -detect-uninlinable-helpers
-      "${LIBTCG_HELPERS}" -o "${LIBTCG_HELPERS_FULL}")
+      -cpu-loop-exit -analyze-helper-arguments ${EXTRA_HELPER_PASSES}
+      -fix-helpers -fix-helpers-architecture="${ARCH}" -dce
+      -detect-uninlinable-helpers "${LIBTCG_HELPERS}" -o
+      "${LIBTCG_HELPERS_FULL}")
   add_custom_target("annotated-libtcg-helpers-${ARCH}" ALL
                     DEPENDS "${LIBTCG_HELPERS_FULL}")
 

include/revng/FunctionIsolation/InlineHelpers.h

@@ -1,27 +0,0 @@
-#pragma once
-
-//
-// This file is distributed under the MIT License. See LICENSE.md for details.
-//
-
-#include "llvm/Pass.h"
-
-#include "revng/BasicAnalyses/GeneratedCodeBasicInfo.h"
-
-class InlineHelpersPass : public llvm::ModulePass {
-public:
-  static char ID;
-
-  // This map stores the loaded helper module(s) to be inlined. Generally state
-  // in a pass is frowned upon, but since this is only ever read it's simpler to
-  // store it at the pass level.
-  llvm::StringMap<std::unique_ptr<llvm::Module>> HelpersModules;
-
-public:
-  InlineHelpersPass() : ModulePass(ID) {}
-
-  bool runOnModule(llvm::Module &M) override;
-
-private:
-  void linkRequiredHelpers(llvm::Module &M);
-};

include/revng/InlineHelpers/DeleteHelperBodies.h

@@ -0,0 +1,18 @@
+#pragma once
+
+//
+// This file is distributed under the MIT License. See LICENSE.md for details.
+//
+
+#include "llvm/IR/Module.h"
+#include "llvm/Pass.h"
+
+class DeleteHelperBodiesPass : public llvm::ModulePass {
+public:
+  static char ID;
+
+public:
+  DeleteHelperBodiesPass() : llvm::ModulePass(ID) {}
+
+  bool runOnModule(llvm::Module &M) override;
+};

include/revng/InlineHelpers/InlineHelpers.h

@@ -0,0 +1,25 @@
+#pragma once
+
+//
+// This file is distributed under the MIT License. See LICENSE.md for details.
+//
+
+#include "llvm/IR/Module.h"
+#include "llvm/Pass.h"
+
+#include "revng/BasicAnalyses/GeneratedCodeBasicInfo.h"
+
+/// Inline every `revng_inline` helper at its call site in the `Isolated`
+/// functions, where the critical arguments are constant at the call site. Does
+/// not link helper bodies (use `LinkHelpersToInlinePass` first) and does not
+/// delete inlined helper bodies (use `DeleteHelperBodiesPass` once at the end
+/// of the pipeline).
+class InlineHelpersPass : public llvm::ModulePass {
+public:
+  static char ID;
+
+public:
+  InlineHelpersPass() : ModulePass(ID) {}
+
+  bool runOnModule(llvm::Module &M) override;
+};

include/revng/InlineHelpers/LinkHelpersToInline.h

@@ -0,0 +1,21 @@
+#pragma once
+
+//
+// This file is distributed under the MIT License. See LICENSE.md for details.
+//
+
+#include "llvm/IR/Module.h"
+#include "llvm/Pass.h"
+
+/// Link the bodies of every `revng_inline`-tagged helper that's still a
+/// declaration in `M`, cloning them from the per-architecture
+/// `libtcg-helpers-to-inline-<arch>.bc` shipped with revng.
+class LinkHelpersToInlinePass : public llvm::ModulePass {
+public:
+  static char ID;
+
+public:
+  LinkHelpersToInlinePass() : llvm::ModulePass(ID) {}
+
+  bool runOnModule(llvm::Module &M) override;
+};

lib/CMakeLists.txt

@@ -37,6 +37,7 @@ add_subdirectory(HeadersGeneration)
 add_subdirectory(HelperArgumentsAnalysis)
 add_subdirectory(ImportFromC)
 add_subdirectory(InitModelTypes)
+add_subdirectory(InlineHelpers)
 add_subdirectory(LocalVariables)
 add_subdirectory(LLMRename)
 add_subdirectory(Lift)

lib/EarlyFunctionAnalysis/CMakeLists.txt

@@ -35,6 +35,7 @@ target_link_libraries(
   revngEarlyFunctionAnalysis
   revngBasicAnalyses
   revngABI
+  revngInlineHelpers
   revngSupport
   revngModel
   revngPipes

lib/EarlyFunctionAnalysis/DetectABI.cpp

@@ -30,8 +30,10 @@
 #include "revng/EarlyFunctionAnalysis/DetectABI.h"
 #include "revng/EarlyFunctionAnalysis/FunctionEdgeBase.h"
 #include "revng/EarlyFunctionAnalysis/FunctionSummaryOracle.h"
+#include "revng/InlineHelpers/InlineHelpers.h"
+#include "revng/InlineHelpers/LinkHelpersToInline.h"
 #include "revng/Model/Binary.h"
-#include "revng/Model/NameBuilder.h"
+#include "revng/Model/IRHelpers.h"
 #include "revng/Model/Pass/DeduplicateCollidingNames.h"
 #include "revng/Model/Register.h"
 #include "revng/Pipeline/Pipe.h"
@@ -39,6 +41,7 @@
 #include "revng/Pipes/Kinds.h"
 #include "revng/Pipes/ModelGlobal.h"
 #include "revng/Support/BasicBlockID.h"
+#include "revng/Support/CommonOptions.h"
 #include "revng/Support/Debug.h"
 #include "revng/Support/IRBuilder.h"
 #include "revng/Support/IRHelpers.h"
@@ -82,6 +85,14 @@ static opt<ABIOpt> ABIEnforcement("abi-enforcement-level",
                                                     "found.")),
                                   init(ABIOpt::FullABIEnforcement));
 
+static opt<std::string> DumpPostInline("detect-abi-dump-post-helpers-inlining",
+                                       desc("Path of a file the whole "
+                                            "module is written to right "
+                                            "after helper inlining inside "
+                                            "`analyzeABI` (for debugging "
+                                            "and testing)"),
+                                       init(""));
+
 static Logger Log("detect-abi");
 
 struct Changes {
@@ -107,6 +118,7 @@ public:
     Manager.add(new LoadModelWrapperPass(ModelWrapper(Global->get())));
     Manager.add(new CollectFunctionsFromCalleesWrapperPass());
     Manager.add(new ControlFlowGraphCachePass(CFGs));
+    Manager.add(new LinkHelpersToInlinePass());
     Manager.add(new efa::DetectABIPass());
     Manager.add(new CollectFunctionsFromUnusedAddressesWrapperPass());
     Manager.add(new efa::DetectABIPass());
@@ -451,7 +463,7 @@ void DetectABI::analyzeABI() {
   revng_log(Log, "Running ABI analyses");
   LoggerIndent Indent(Log);
 
-  llvm::Task Task(2, "analyzeABI");
+  llvm::Task Task(3, "analyzeABI");
   std::map<MetaAddress, std::unique_ptr<OutlinedFunction>> Functions;
 
   // Create all temporary functions
@@ -463,6 +475,39 @@ void DetectABI::analyzeABI() {
     Functions[Function.Entry()] = std::move(NewFunction);
   }
 
+  // Inline `revng_inline`-tagged helper calls so the subsequent ABI dataflow
+  // analysis can observe the helpers' register reads and writes directly.
+  // `InlineHelpersPass` only inlines into `Isolated` functions, so we tag the
+  // outlined stubs as such. These stubs are temporary and are discarded with
+  // the cloned module once the analysis ends, so the tag never escapes.
+  // Helpers whose per-call critical arguments are not LLVM constants here
+  // survive un-inlined and are consumed as opaque calls by the analysis.
+  for (auto &[Entry, OutlinedFn] : Functions)
+    FunctionTags::Isolated.addTo(OutlinedFn->Function.get());
+
+  Task.advance("Inline helpers");
+  {
+    llvm::legacy::PassManager PM;
+    PM.add(new InlineHelpersPass());
+    PM.run(M);
+  }
+
+  // When `--debug-names` is set, rename each outlined stub from the
+  // metaaddress-based identifier to its source-symbol name, using the same
+  // `llvmName` scheme as the rest of the pipeline.
+  if (DebugNames) {
+    for (auto &[Entry, OutlinedFn] : Functions) {
+      llvm::Function *F = OutlinedFn->Function.get();
+      F->setName(llvmName(Binary->Functions().at(Entry)));
+    }
+  }
+
+  // Dump the whole module right after helper inlining, for debugging and
+  // testing. The outlined stubs live in the module at this point, so a single
+  // module dump captures them all.
+  if (not DumpPostInline.empty())
+    dumpModule(&M, DumpPostInline.c_str());
+
   // Push this into analyzeFunction
   OpaqueRegisterUser RegisterUser(&M);
 
@@ -482,11 +527,10 @@ void DetectABI::analyzeABI() {
   }
 
   unsigned Runs = 0;
-  model::CNameBuilder NameBuilder = *Binary;
   while (not ToAnalyze.empty()) {
     model::Function &Function = *ToAnalyze.pop();
     revng_log(Log, "Analyzing " << Function.Entry().toString());
-    FixedPointTask.advance(NameBuilder.name(Function));
+    FixedPointTask.advance(llvmName(Function));
     OutlinedFunction &OutlinedFunction = *Functions.at(Function.Entry());
     Changes Changes = analyzeFunctionABI(Function,
                                          OutlinedFunction,
@@ -1117,6 +1161,14 @@ llvm::Error DetectABI::run(Model &Model,
   revng::pipes::CFGMap CFGs("");
   ControlFlowGraphCache FMC(CFGs);
 
+  // Link helper bodies once at the start of the analysis, to avoid relinking
+  // for every `inline-helpers` call
+  {
+    llvm::legacy::PassManager PM;
+    PM.add(new LinkHelpersToInlinePass());
+    PM.run(Module);
+  }
+
   collectFunctionsFromCallees(Module, GCBI, Binary);
   efa::runDetectABI(Module, GCBI, FMC, TupleModel);
   collectFunctionsFromUnusedAddresses(Module, GCBI, Binary, FMC);

lib/FunctionIsolation/CMakeLists.txt

@@ -5,10 +5,8 @@
 revng_add_analyses_library_internal(
   revngFunctionIsolation
   EnforceABI.cpp
-  InlineHelpers.cpp
   InvokeIsolatedFunctions.cpp
   IsolateFunctions.cpp
-  PostInlineHelpersVerifyPass.cpp
   PromoteCSVs.cpp
   RemoveExceptionalCalls.cpp
   StripDebugInfoFromHelpers.cpp
@@ -21,6 +19,7 @@ target_link_libraries(
   revngABI
   revngModel
   revngEarlyFunctionAnalysis
+  revngInlineHelpers
   revngSupport
   revngPipes
   ${LLVM_LIBRARIES})

lib/InlineHelpers/CMakeLists.txt

@@ -0,0 +1,16 @@
+#
+# This file is distributed under the MIT License. See LICENSE.md for details.
+#
+
+# The inline-helpers chain (link / inline / delete), exposed both as LLVM passes
+# and as free functions. It lives in its own library so that
+# EarlyFunctionAnalysis can depend on it without creating a cycle with
+# FunctionIsolation, which depends on EarlyFunctionAnalysis.
+revng_add_analyses_library_internal(
+  revngInlineHelpers DeleteHelperBodies.cpp InlineHelpers.cpp
+  LinkHelpersToInline.cpp PostInlineHelpersVerifyPass.cpp)
+
+llvm_map_components_to_libnames(LLVM_LIBRARIES TransformUtils)
+
+target_link_libraries(revngInlineHelpers revngBasicAnalyses revngModel
+                      revngSupport ${LLVM_LIBRARIES})

lib/InlineHelpers/DeleteHelperBodies.cpp

@@ -0,0 +1,41 @@
+//
+// This file is distributed under the MIT License. See LICENSE.md for details.
+//
+
+#include "llvm/IR/Function.h"
+#include "llvm/IR/Module.h"
+#include "llvm/Pass.h"
+
+#include "revng/InlineHelpers/DeleteHelperBodies.h"
+#include "revng/Support/IRHelpers.h"
+
+#include "PostInlineHelpersVerifyPass.h"
+
+using namespace llvm;
+
+// Drop the body of every `revng_inline`-tagged function in `M`, turning each
+// one back into a declaration.
+static void deleteHelperBodies(llvm::Module &M) {
+  for (llvm::Function &F : M) {
+    if (F.getSection() == InlineHelpersSection)
+      deleteOnlyBody(F);
+  }
+}
+
+char DeleteHelperBodiesPass::ID = 0;
+
+using Register = RegisterPass<DeleteHelperBodiesPass>;
+static Register
+  X("delete-helper-bodies", "Delete Helper Bodies Pass", true, true);
+
+bool DeleteHelperBodiesPass::runOnModule(llvm::Module &M) {
+  deleteHelperBodies(M);
+
+  // Verify that no `getelementptr` instruction was dragged into an Isolated
+  // function by the inlining performed by the upstream `-inline-helpers`
+  // pass. The downstream pipeline relies on this invariant.
+  // TODO: convert this from a pass to a free-standing function
+  PostInlineHelpersVerifyPass{}.runOnModule(M);
+
+  return true;
+}

lib/InlineHelpers/InlineHelpers.cpp

@@ -7,7 +7,6 @@
 
 #include "llvm/ADT/BitVector.h"
 #include "llvm/ADT/SmallVector.h"
-#include "llvm/ADT/StringSet.h"
 #include "llvm/IR/Instructions.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/FileSystem.h"
@@ -15,16 +14,10 @@
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Transforms/Utils/Cloning.h"
 
-#include "revng/ADT/Queue.h"
-#include "revng/BasicAnalyses/GeneratedCodeBasicInfo.h"
-#include "revng/FunctionIsolation/InlineHelpers.h"
+#include "revng/InlineHelpers/InlineHelpers.h"
 #include "revng/Support/CommandLine.h"
 #include "revng/Support/Debug.h"
 #include "revng/Support/IRHelpers.h"
-#include "revng/Support/OpaqueFunctionsPool.h"
-#include "revng/Support/ResourceFinder.h"
-
-#include "PostInlineHelpersVerifyPass.h"
 
 using namespace llvm;
 
@@ -194,67 +187,9 @@ void InlineHelpers::run(Function *F) {
     dumpFunctionToDir(*F, DumpAfterDir);
 }
 
-void InlineHelpersPass::linkRequiredHelpers(llvm::Module &M) {
-  NamedMDNode *Node = M.getNamedMetadata(QemuArchitectureMD);
-  revng_assert(Node != nullptr);
-  revng_assert(Node->getNumOperands() > 0);
-
-  // Multiple linkings could have lead to this node having more than one
-  // operand, if so check that they are all the same.
-  llvm::StringSet Strings;
-  for (size_t I = 0; I < Node->getNumOperands(); I++) {
-    revng_assert(Node->getOperand(0)->getNumOperands() == 1);
-    const MDOperand &StringNode = Node->getOperand(0)->getOperand(0);
-    Strings.insert(cast<MDString>(StringNode)->getString());
-  }
-
-  revng_assert(Strings.size() == 1,
-               "QEMU helpers of multiple architectures were linked inside the "
-               "module");
-
-  // Collect the names of every `revng_inline` helper referenced in `M`;
-  // they all lack a body in `M` and need to be cloned from the
-  // per-architecture to-inline bitcode.
-  SmallVector<StringRef, 32> HelpersToInline;
-  for (llvm::Function &F : M) {
-    if (F.isDeclaration() and F.getSection() == InlineHelpersSection)
-      HelpersToInline.push_back(F.getName());
-  }
-
-  // Skip the expensive bitcode parse + clone + link if there's nothing to do.
-  if (HelpersToInline.empty())
-    return;
-
-  // Given the architecture MD, retrieve the correct libtcg module (if not
-  // already in the `HelpersModules` map) and link it into the main module
-  StringRef ArchName = Strings.begin()->first();
-  if (HelpersModules.count(ArchName) == 0) {
-    const std::string LibHelpersName = ("/share/revng/"
-                                        "libtcg-helpers-to-inline-"
-                                        + ArchName + ".bc")
-                                         .str();
-    auto OptionalHelpers = revng::ResourceFinder.findFile(LibHelpersName);
-    revng_assert(OptionalHelpers.has_value(), "Cannot find libtcg helpers");
-    HelpersModules[ArchName] = parseIR(M.getContext(), OptionalHelpers.value());
-  }
-
-  llvm::Module &HelpersModule = *HelpersModules[ArchName];
-  std::set<const llvm::Function *> ToClone;
-  for (StringRef Name : HelpersToInline) {
-    llvm::Function *HelperFunction = HelpersModule.getFunction(Name);
-    revng_assert(HelperFunction != nullptr
-                   and not HelperFunction->isDeclaration(),
-                 "revng_inline helper missing body in to-inline.bc");
-    ToClone.insert(HelperFunction);
-  }
-
-  auto ClonedHelpersModule = ::cloneFiltered(HelpersModule, ToClone);
-  linkModules(std::move(ClonedHelpersModule), M);
-}
-
 bool InlineHelpersPass::runOnModule(llvm::Module &M) {
-  linkRequiredHelpers(M);
-
+  // Inline `revng_inline` helper calls into the `Isolated` functions, where the
+  // per-call critical arguments are constant at the call site.
   SmallVector<Function *, 32> Isolated;
   for (Function &F : M)
     if (FunctionTags::Isolated.isTagOf(&F))
@@ -267,18 +202,5 @@ bool InlineHelpersPass::runOnModule(llvm::Module &M) {
     IH.run(F);
   }
 
-  // Since we're done inlining helpers they are no longer needed, delete their
-  // body
-  for (Function &F : M) {
-    if (F.getSection() == InlineHelpersSection)
-      deleteOnlyBody(F);
-  }
-
-  // Verify that no `getelementptr` instruction was dragged into an Isolated
-  // function by the inlining above. The downstream pipeline relies on this
-  // invariant.
-  // TODO: convert this from a pass to a free-standing function
-  PostInlineHelpersVerifyPass{}.runOnModule(M);
-
   return true;
 }

lib/InlineHelpers/LinkHelpersToInline.cpp

@@ -0,0 +1,92 @@
+//
+// This file is distributed under the MIT License. See LICENSE.md for details.
+//
+
+#include <memory>
+#include <set>
+#include <string>
+
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/ADT/StringSet.h"
+#include "llvm/IR/Function.h"
+#include "llvm/IR/Metadata.h"
+#include "llvm/IR/Module.h"
+#include "llvm/Pass.h"
+
+#include "revng/InlineHelpers/LinkHelpersToInline.h"
+#include "revng/Support/Assert.h"
+#include "revng/Support/IRHelpers.h"
+#include "revng/Support/ResourceFinder.h"
+
+using namespace llvm;
+
+char LinkHelpersToInlinePass::ID = 0;
+
+using Register = RegisterPass<LinkHelpersToInlinePass>;
+static Register X("link-helpers-to-inline",
+                  "Link Required Helpers To Inline Pass",
+                  true,
+                  true);
+
+bool LinkHelpersToInlinePass::runOnModule(llvm::Module &M) {
+  NamedMDNode *Node = M.getNamedMetadata(QemuArchitectureMD);
+  revng_assert(Node != nullptr);
+  revng_assert(Node->getNumOperands() > 0);
+
+  // Multiple linkings could have lead to this node having more than one
+  // operand, if so check that they are all the same.
+  llvm::StringSet Strings;
+  for (size_t I = 0; I < Node->getNumOperands(); I++) {
+    revng_assert(Node->getOperand(0)->getNumOperands() == 1);
+    const MDOperand &StringNode = Node->getOperand(0)->getOperand(0);
+    Strings.insert(cast<MDString>(StringNode)->getString());
+  }
+
+  revng_assert(Strings.size() == 1,
+               "QEMU helpers of multiple architectures were linked inside the "
+               "module");
+
+  // Collect the names of revng_inline helpers that lack a body in `M` and
+  // therefore need to be cloned from the per-architecture to-inline bitcode.
+  SmallVector<StringRef, 32> MissingBodies;
+  for (llvm::Function &F : M) {
+    if (F.isDeclaration() and F.getSection() == InlineHelpersSection) {
+      MissingBodies.push_back(F.getName());
+    }
+  }
+
+  // Nothing to link — every referenced revng_inline helper already has a
+  // body in `M`. Skip the expensive bitcode parse + clone + link in this case.
+  if (MissingBodies.empty()) {
+    return true;
+  }
+
+  // Given the architecture MD, retrieve the correct libtcg module and link
+  // it into the main module.
+  StringRef ArchName = Strings.begin()->first();
+  const std::string LibHelpersName = ("/share/revng/"
+                                      "libtcg-helpers-to-inline-"
+                                      + ArchName + ".bc")
+                                       .str();
+  auto OptionalHelpers = revng::ResourceFinder.findFile(LibHelpersName);
+  revng_assert(OptionalHelpers.has_value(), "Cannot find libtcg helpers");
+  std::unique_ptr<llvm::Module>
+    HelpersModule = parseIR(M.getContext(), OptionalHelpers.value());
+
+  std::set<const llvm::Function *> ToClone;
+  for (StringRef Name : MissingBodies) {
+    llvm::Function *HelperFunction = HelpersModule->getFunction(Name);
+    revng_assert(HelperFunction != nullptr
+                   and not HelperFunction->isDeclaration(),
+                 "revng_inline helper missing body in to-inline.bc; the "
+                 "static-policy / slim-down chain produced an "
+                 "inconsistent state");
+    ToClone.insert(HelperFunction);
+  }
+
+  auto ClonedHelpersModule = ::cloneFiltered(*HelpersModule, ToClone);
+  linkModules(std::move(ClonedHelpersModule), M);
+
+  return true;
+}

python/CMakeLists.txt

@@ -351,6 +351,7 @@ set(REVNG_CLI_COMMANDS_MODULE_FILES
     revng/internal/cli/_commands/test_docs/__init__.py
     revng/internal/cli/_commands/test_docs/doctest_runner.py
     revng/internal/cli/_commands/trace_run.py
+    revng/internal/cli/_commands/merge-irs.py
     revng/internal/cli/_commands/model_compare.py
     revng/internal/cli/_commands/model_migrate.py
     revng/internal/cli/_commands/model_export_sqlite.py

python/revng/internal/cli/_commands/merge-irs.py

@@ -0,0 +1,96 @@
+#
+# This file is distributed under the MIT License. See LICENSE.md for details.
+#
+
+import tarfile
+from argparse import ArgumentParser, RawDescriptionHelpFormatter
+from base64 import b64decode
+from contextlib import suppress
+from pathlib import Path
+from shutil import copyfileobj
+from tempfile import TemporaryDirectory, TemporaryFile
+from typing import IO
+
+import yaml
+
+from revng.internal.cli.commands_registry import Command, CommandsRegistry, Options
+from revng.internal.cli.support import build_command_with_loads, file_wrapper, try_run
+from revng.support import TarDictionary
+
+
+def read_single_file(input_: IO[bytes]) -> list[bytes]:
+    # If we encounter a non-seekable file (e.g. stdin) first copy it to a
+    # temporary file and then re-run this function
+    if not input_.seekable():
+        with TemporaryFile() as temp_file:
+            copyfileobj(input_, temp_file)
+            temp_file.seek(0)
+            return read_single_file(temp_file)
+
+    # If here, the file is seekable, test the various formats by trying to read
+    # them and moving onto the next format if the previous throws an error
+
+    # Test for YAML
+    with suppress(yaml.YAMLError):
+        data = yaml.load(input_, Loader=yaml.CSafeLoader)
+        return [b64decode(v) for v in data.values()]
+
+    # Test for tar
+    input_.seek(0)
+    with suppress(tarfile.ReadError):
+        return list(TarDictionary(input_).values())
+
+    # No good format found, assume it's a single plain file
+    input_.seek(0)
+    return [input_.read()]
+
+
+class MergeLLVMIRCommand(Command):
+    def __init__(self):
+        super().__init__(("merge", "llvm"), "Merge multiple LLVM IR files into a single one")
+
+    def register_arguments(self, parser: ArgumentParser):
+        parser.formatter_class = RawDescriptionHelpFormatter
+        parser.description = """Links multiple LLVM IR files into a single one.
+This program works differently depending on the number of inputs:
+* If one input is passed it is interpreted as either:
+  * a YAML dict with base64-encoded values
+  * a tar where each file is a bitcode file
+  * a single LLVM module
+* If no input is passed, treat stdin as a single input
+* If multiple input files are passed, treat all of them as bitcode files
+"""
+        parser.add_argument("-o", "--output", help="Where to output the file, stdout if omitted")
+        parser.add_argument("inputs", nargs="*", help="Input file(s), omit to use stdin")
+
+    def run(self, options: Options):
+        args = options.parsed_args
+        if len(args.inputs) >= 2:
+            return self.run_merge(args.inputs, options)
+
+        input_file = args.inputs[0] if len(args.inputs) == 1 else "-"
+        with file_wrapper(input_file, "rb") as input_:
+            data = read_single_file(input_)
+
+        with TemporaryDirectory(prefix="tmp.revng-merge-llvm.") as temp_dir:
+            temp_dir_path = Path(temp_dir)
+            input_paths = []
+            for index, value in enumerate(data):
+                element_path = temp_dir_path / str(index)
+                element_path.write_bytes(value)
+                input_paths.append(str(element_path.resolve()))
+
+            return self.run_merge(input_paths, options)
+
+    def run_merge(self, inputs: list[str], options: Options):
+        with file_wrapper(options.parsed_args.output, "wb") as output_file:
+            return try_run(
+                build_command_with_loads("merge-llvm", inputs, options),
+                options,
+                stdout=output_file,
+            )
+
+
+def setup(commands_registry: CommandsRegistry):
+    commands_registry.define_namespace(("merge",), "Tools to merge function-wise object together")
+    commands_registry.register_command(MergeLLVMIRCommand())

python/revng/internal/cli/support.py

@@ -2,6 +2,8 @@
 # This file is distributed under the MIT License. See LICENSE.md for details.
 #
 
+from __future__ import annotations
+
 import os
 import shlex
 import signal
@@ -10,8 +12,8 @@ from contextlib import contextmanager
 from dataclasses import dataclass
 from subprocess import Popen
 from tempfile import NamedTemporaryFile
-from typing import Any, Callable, Dict, Iterable, List, Literal, Mapping, NoReturn, Optional
-from typing import Tuple, Union
+from typing import TYPE_CHECKING, Any, Callable, Dict, Iterable, List, Literal, Mapping, NoReturn
+from typing import Optional, Tuple, Union
 
 import yaml
 
@@ -19,6 +21,9 @@ from revng.internal.support.collect import collect_libraries
 from revng.internal.support.elf import is_executable
 from revng.support import TarDictionary, get_command
 
+if TYPE_CHECKING:
+    from _typeshed import FileDescriptorLike
+
 OptionalEnv = Optional[Mapping[str, str]]
 
 
@@ -74,7 +79,7 @@ def _run_common(
     return command, environment
 
 
-def popen(command, options: Options, environment: OptionalEnv = None, **kwargs) -> int | Popen:
+def _popen(command, options: Options, environment: OptionalEnv = None, **kwargs) -> int | Popen:
     command, environment = _run_common(command, options, environment)
 
     if options.dry_run:
@@ -83,15 +88,36 @@ def popen(command, options: Options, environment: OptionalEnv = None, **kwargs)
     return Popen(command, env=environment, **kwargs)
 
 
-def try_run(command, options: Options, environment: OptionalEnv = None) -> int:
+def popen(
+    command,
+    options: Options,
+    environment: OptionalEnv = None,
+    stdin: FileDescriptorLike | None = None,
+    stdout: FileDescriptorLike | None = None,
+    stderr: FileDescriptorLike | None = None,
+) -> int | Popen:
+    return _popen(command, options, environment, stdin=stdin, stdout=stdout, stderr=stderr)
+
+
+def try_run(
+    command,
+    options: Options,
+    environment: OptionalEnv = None,
+    stdin: FileDescriptorLike | None = None,
+    stdout: FileDescriptorLike | None = None,
+    stderr: FileDescriptorLike | None = None,
+) -> int:
     try:
         signal.signal(signal.SIGINT, signal.SIG_IGN)
-        process = popen(
+        process = _popen(
             command,
             options,
             environment,
             preexec_fn=lambda: signal.signal(signal.SIGINT, signal.SIG_DFL),
             close_fds=False,
+            stdin=stdin,
+            stdout=stdout,
+            stderr=stderr,
         )
         if isinstance(process, int):
             return process
@@ -100,8 +126,15 @@ def try_run(command, options: Options, environment: OptionalEnv = None) -> int:
         signal.signal(signal.SIGINT, signal.SIG_DFL)
 
 
-def run(command, options: Options, environment: OptionalEnv = None):
-    result = try_run(command, options, environment)
+def run(
+    command,
+    options: Options,
+    environment: OptionalEnv = None,
+    stdin: FileDescriptorLike | None = None,
+    stdout: FileDescriptorLike | None = None,
+    stderr: FileDescriptorLike | None = None,
+):
+    result = try_run(command, options, environment, stdin, stdout, stderr)
     if result != 0:
         sys.exit(result)
 

python/revng/internal/pipeline.yml

@@ -178,7 +178,9 @@ branches:
         configuration:
           passes:
             - mem2reg
+            - link-helpers-to-inline
             - inline-helpers
+            - delete-helper-bodies
       - pipe: attach-debug-info
         arguments: [cfg-map, llvm-functions]
       - pipe: promote-csvs
@@ -362,6 +364,11 @@ branches:
             - dce
       - savepoint: segregate-stack-accesses
         containers: [llvm-functions]
+        artifacts:
+          - name: segregate-stack-accesses
+            container: llvm-functions
+            category: debug
+            filename: stack_accesses_segregated.bc
       - pipe: pure-llvm-passes-pipe
         arguments: [llvm-functions]
         configuration:

share/revng/pipelines/revng-pipelines.yml

@@ -170,7 +170,9 @@ Branches:
               #       `cc_compute_c` and `cc_compute_all`.
               - promote-csvs
               - mem2reg
+              - link-helpers-to-inline
               - inline-helpers
+              - delete-helper-bodies
           - Type: attach-debug-info-to-abi-enforced
             UsedContainers: [cfg.yml.tar.gz, functions.bc.zstd]
           - Type: llvm-pipe

share/revng/test/configuration/revng/detect-stack-size.yml

@@ -6,13 +6,14 @@ commands:
   #
   # Run detect-stack-size and test against ground truth
   #
-  # TODO: this crashes in the new pipeline, investigate
   - type: revng.test-detect-stack-size
     from:
       - type: revng-qa.compiled-with-debug-info
         filter: for-detect-stack-size
     suffix: /
     command: |-
-      revng analyze --resume "$OUTPUT" import-binary "$INPUT" -o /dev/null;
-      revng analyze --resume "$OUTPUT" detect-abi "$INPUT" -o /dev/null;
-      revng analyze --resume "$OUTPUT" detect-stack-size "$INPUT" | revng model compare "${SOURCE}.model.yml" -;
+      cd "$OUTPUT";
+      revng2 project init --no-initial-auto-analysis "$INPUT";
+      revng2 project analyze parse-binary -o /dev/null;
+      revng2 project analyze detect-abi -o /dev/null;
+      revng2 project analyze detect-stack-size | revng model compare "${SOURCE}.model.yml" -;

share/revng/test/configuration/revng/floating-point-helpers-inlining.yml

@@ -0,0 +1,49 @@
+#
+# This file is distributed under the MIT License. See LICENSE.md for details.
+#
+
+# End-to-end tests that verify the emergence of the softfloat calls, on the same
+# `for-floating-point-helpers-inlining` ground truth, at two points of the
+# pipeline:
+#   * `revng.test-floating-point-helpers-inlining-in-enforce-abi`:
+#     in the `enforce-abi` artifact;
+#   * `revng.test-floating-point-helpers-inlining-in-detect-abi`:
+#     right after helper inlining inside `DetectABI::analyzeABI()`.
+# We pass the `--debug-names` flag to easily anchor the `CHECK` directives for
+# multiple function names.
+# The CHECK-64 prefix is used for 64-bit-only tests.
+
+commands:
+  - type: revng.test-floating-point-helpers-inlining-in-enforce-abi
+    from:
+      - type: revng-qa.compiled-with-debug-info
+        # i386 is excluded until x87 FPU support lands.
+        filter: for-floating-point-helpers-inlining and !i386
+    # `revng merge llvm` merges the per-function IR modules emitted by the
+    # `--tar` artifact into a single module, so we can `FileCheck` them all at
+    # once.
+    # The `sed` below strips the `local_` prefix for a coherent FileCheck file.
+    command: |-
+      PREFIXES=CHECK ;
+      [ "$POINTER_SIZE" = "64" ] && PREFIXES=$$PREFIXES,CHECK-64 ;
+      revng2 quick artifact enforce-abi "$INPUT" --tar -- --debug-names |
+        revng merge llvm |
+        sed 's,@local_,@,g' |
+        FileCheck --check-prefixes=$$PREFIXES "${SOURCE}".filecheck
+
+  - type: revng.test-floating-point-helpers-inlining-in-detect-abi
+    from:
+      - type: revng-qa.compiled-with-debug-info
+        # i386 is excluded until x87 FPU support lands.
+        filter: for-floating-point-helpers-inlining and !i386
+    # The `sed` below strips the `local_` prefix for a coherent FileCheck file.
+    command: |-
+      DUMP="$$(temp)" ;
+      revng2 quick analyze --analyses=parse-binary,detect-abi "$INPUT"
+        -- --debug-names
+        --detect-abi-dump-post-helpers-inlining="$$DUMP"
+        > /dev/null ;
+      PREFIXES=CHECK ;
+      [ "$POINTER_SIZE" = "64" ] && PREFIXES=$$PREFIXES,CHECK-64 ;
+      sed 's,@local_,@,g' "$$DUMP" |
+      FileCheck --check-prefixes=$$PREFIXES "${SOURCE}".filecheck

share/revng/test/configuration/revng/floating-point.yml

@@ -1,29 +0,0 @@
-#
-# This file is distributed under the MIT License. See LICENSE.md for details.
-#
-
-# End-to-end tests which verify the emergence of the softfloat calls in the
-# `enforce-abi` artifact.
-# We pass the `--debug-names` flag to easily anchor the `CHECK` directives for
-# multiple function names.
-# We also strip the `local_` prefix to have a more coherent FileCheck file.
-# Since we use the `revng2` pipeline, we need to extract all the generated IRs
-# for each isolated function and to concatenated their content to easily match
-# the CHECK directives.
-# The CHECK-64 prefix is used for 64-bit arch only tests.
-
-commands:
-  - type: revng.floating-point
-    from:
-      - type: revng-qa.compiled-with-debug-info
-        filter: for-floating-point and !i386
-    command: |-
-      PROJECT="$$(temp -d)" ;
-      revng2 -C "$$PROJECT" project init "$INPUT" -- --debug-names ;
-      revng2 -C "$$PROJECT" project artifact enforce-abi --format yaml -o "$$PROJECT/out.yml" -- --debug-names ;
-      PREFIXES=CHECK ;
-      file "$INPUT" | grep -q "ELF 64-bit" && PREFIXES=$$PREFIXES,CHECK-64 ;
-      yq -r 'to_entries[].value' "$$PROJECT/out.yml"
-        | while IFS= read -r b64 ; do printf '%s' "$$b64" | base64 -d | revng opt -S /dev/stdin -o - ; done
-        | sed 's,@local_,@,g'
-        | FileCheck --check-prefixes=$$PREFIXES "${SOURCE}".filecheck

share/revng/test/configuration/revng/segregate-stack-accesses.yml

@@ -3,29 +3,31 @@
 #
 
 commands:
-  # TODO: this crashes with the new pipeline, investigate
   - type: revng.test-segregate-stack-accesses
     from:
       - type: revng-qa.compiled-with-debug-info
         filter: for-segregate-stack-accesses
     suffix: /
     command: |-
+      cd "$OUTPUT";
       export REVNG_OPTIONS="$${REVNG_OPTIONS:-} --debug-names";
 
-      revng analyze --resume "$OUTPUT" import-binary "$INPUT" -o /dev/null;
+      revng2 project init --no-initial-auto-analysis "$INPUT";
+      revng2 project analyze parse-binary -o /dev/null;
 
       revng model override-by-name
-        "$OUTPUT/context/model.yml"
+        model.yml
         ${SOURCE}.override.$QEMU_NAME.yml
-        > "$OUTPUT/context/model-tmp.yml";
-      mv "$OUTPUT/context/model-tmp.yml" "$OUTPUT/context/model.yml";
+        > model-tmp.yml;
+      mv model-tmp.yml model.yml;
+      rm -r .cache;
 
-      revng analyze --resume "$OUTPUT" detect-abi "$INPUT" -o /dev/null;
+      revng2 project analyze detect-abi -o /dev/null;
+      revng2 project analyze detect-stack-size -o /dev/null;
 
-      revng analyze --resume "$OUTPUT" detect-stack-size "$INPUT" -o /dev/null;
-
-      revng artifact --resume "$OUTPUT" segregate-stack-accesses "$INPUT" |
+      revng2 project artifact segregate-stack-accesses --tar |
+        revng merge llvm |
         revng opt -remove-llvmassume-calls -instcombine-noarrays -sroa-noarrays -drop-redundant-ptrint-casts -dce -type-shrinking -dce -drop-redundant-ptrint-casts -early-cse -instsimplify -dce -S |
         FileCheck --check-prefix=CHECK-$QEMU_NAME ${SOURCE}.filecheck.ll;
 
-      revng artifact --resume "$OUTPUT" emit-c "$INPUT" -o /dev/null;
+      revng2 project artifact emit-c -o /dev/null;

share/revng/test/configuration/revng/simplify-switch.yml

@@ -11,15 +11,6 @@ commands:
       - type: revng-qa.compiled-with-debug-info
         filter: for-simplify-switch
     command: |-
-      WORKDIR=$$(temp -d);
-      revng2 quick artifact simplify-switch "$INPUT"
-        --tar -o "$$WORKDIR/simplify-switch.tar"
-        -- --abi-enforcement-level=no;
-
-      tar -tf "$$WORKDIR/simplify-switch.tar" 2>/dev/null |
-      while IFS= read -r ENTRY; do
-        tar -xf "$$WORKDIR/simplify-switch.tar" "$$ENTRY" --to-stdout 2>/dev/null |
-          revng opt -S >> "$$WORKDIR/simplify-switch.ll";
-      done;
-
-      FileCheck "${SOURCE}.filecheck" < "$$WORKDIR/simplify-switch.ll";
+      revng2 quick artifact simplify-switch "$INPUT" --tar -- --abi-enforcement-level=no |
+        revng merge llvm |
+        FileCheck "${SOURCE}.filecheck";

tests/filecheck/llvm-passes/inline-helpers.ll

@@ -8,7 +8,7 @@
 ; where every listed argument is a compile-time constant. A helper
 ; with a zero policy must always be inlined.
 ;
-; RUN: %root/bin/revng opt -inline-helpers %s -S -o - | FileCheck %s
+; RUN: %root/bin/revng opt -link-helpers-to-inline -inline-helpers -delete-helper-bodies %s -S -o - | FileCheck %s
 
 !revng.qemu_architecture = !{!0}
 !0 = !{!"x86_64"}

tools/CMakeLists.txt

@@ -5,6 +5,7 @@
 add_subdirectory(clift-opt)
 add_subdirectory(internal)
 add_subdirectory(lddtree)
+add_subdirectory(merge-llvm)
 add_subdirectory(model)
 add_subdirectory(pipeline)
 add_subdirectory(pypeline)

tools/merge-llvm/CMakeLists.txt

@@ -0,0 +1,7 @@
+#
+# This file is distributed under the MIT License. See LICENSE.md for details.
+#
+
+revng_add_executable(merge-llvm Main.cpp)
+
+target_link_libraries(merge-llvm revngSupport ${LLVM_LIBRARIES})

tools/merge-llvm/Main.cpp

@@ -0,0 +1,56 @@
+//
+// This file is distributed under the MIT License. See LICENSE.md for details.
+//
+
+#include "llvm/IR/Module.h"
+#include "llvm/IRReader/IRReader.h"
+#include "llvm/Support/Error.h"
+#include "llvm/Support/PluginLoader.h"
+#include "llvm/Support/SourceMgr.h"
+
+#include "revng/Support/CommandLine.h"
+#include "revng/Support/Error.h"
+#include "revng/Support/IRHelpers.h"
+#include "revng/Support/InitRevng.h"
+#include "revng/Support/Tar.h"
+
+static llvm::cl::list<std::string> Arguments(llvm::cl::Positional,
+                                             llvm::cl::ZeroOrMore,
+                                             llvm::cl::desc("FILES"),
+                                             llvm::cl::cat(MainCategory));
+
+static llvm::ExitOnError AbortOnError;
+
+int main(int argc, char *argv[]) {
+  revng::InitRevng X(argc, argv, "", { &MainCategory });
+
+  llvm::LLVMContext Context;
+  std::unique_ptr<llvm::Module> OutputModule;
+
+  for (llvm::StringRef Filename : Arguments) {
+    llvm::SMDiagnostic Err;
+    auto Module = llvm::parseIRFile(Filename, Err, Context);
+    revng_assert(Module != nullptr);
+
+    if (OutputModule != nullptr) {
+      // Make the existing module be linked to the new module, this allows
+      // preserving the order of definitions as the order of the supplied files
+      std::swap(Module, OutputModule);
+      linkFunctionModules(std::move(Module), OutputModule);
+    } else {
+      OutputModule = std::move(Module);
+    }
+  }
+
+  revng_assert(OutputModule != nullptr);
+  revng::verify(OutputModule.get());
+  {
+    std::error_code EC;
+    llvm::raw_fd_ostream OS("-", EC);
+    revng_assert(not EC);
+
+    OutputModule->print(OS, nullptr);
+  }
+
+  return EXIT_SUCCESS;
+}