Merge branch 'feature/pypeline-implement-first-half'

include/revng/ADT/SetOperations.h

@@ -4,6 +4,8 @@
 // This file is distributed under the MIT License. See LICENSE.md for details.
 //
 
+#include "llvm/ADT/SetOperations.h"
+
 template<class S1Ty, class S2Ty>
 bool intersects(const S1Ty &S1, const S2Ty &S2) {
   S1Ty Intersection = llvm::set_intersection(S1, S2);

include/revng/EarlyFunctionAnalysis/AttachDebugInfo.h

@@ -0,0 +1,36 @@
+#pragma once
+
+//
+// This file is distributed under the MIT License. See LICENSE.md for details.
+//
+
+#include "revng/EarlyFunctionAnalysis/CollectCFG.h"
+#include "revng/PipeboxCommon/LLVMContainer.h"
+
+namespace revng::pypeline::piperuns {
+
+class AttachDebugInfo {
+private:
+  const model::Binary &Binary;
+  const CFGMap &CFG;
+  LLVMFunctionContainer &ModuleContainer;
+
+public:
+  static constexpr llvm::StringRef Name = "AttachDebugInfo";
+  using Arguments = TypeList<
+    PipeRunArgument<const CFGMap, "CFG", "Function control-flow data">,
+    PipeRunArgument<LLVMFunctionContainer,
+                    "Module",
+                    "function LLVM module(s)">>;
+
+  AttachDebugInfo(const class Model &Model,
+                  llvm::StringRef Config,
+                  llvm::StringRef DynamicConfig,
+                  const CFGMap &CFG,
+                  LLVMFunctionContainer &ModuleContainer) :
+    Binary(*Model.get().get()), CFG(CFG), ModuleContainer(ModuleContainer){};
+
+  void runOnFunction(const model::Function &TheFunction);
+};
+
+} // namespace revng::pypeline::piperuns

include/revng/EarlyFunctionAnalysis/CollectCFG.h

@@ -0,0 +1,69 @@
+#pragma once
+
+//
+// This file is distributed under the MIT License. See LICENSE.md for details.
+//
+
+#include "revng/EarlyFunctionAnalysis/CFGAnalyzer.h"
+#include "revng/EarlyFunctionAnalysis/ControlFlowGraph.h"
+#include "revng/Pipebox/TupleTreeContainer.h"
+#include "revng/PipeboxCommon/Common.h"
+#include "revng/PipeboxCommon/LLVMContainer.h"
+
+namespace revng::pypeline {
+
+using CFGMap = TupleTreeContainer<efa::ControlFlowGraph,
+                                  Kinds::Function,
+                                  "CFGMap">;
+
+namespace piperuns {
+
+namespace detail {
+
+struct GCBIRun {
+  GCBIRun(GeneratedCodeBasicInfo &GCBI, llvm::Module &Module) {
+    GCBI.run(Module);
+  }
+};
+
+} // namespace detail
+
+class CollectCFG {
+private:
+  const class Model &Model;
+  CFGMap &Output;
+
+  GeneratedCodeBasicInfo GCBI;
+  detail::GCBIRun GCBIRun;
+  efa::FunctionSummaryOracle Oracle;
+  efa::CFGAnalyzer Analyzer;
+
+public:
+  static constexpr llvm::StringRef Name = "CollectCFG";
+  using Arguments = TypeList<PipeRunArgument<LLVMRootContainer,
+                                             "Input",
+                                             "LLVM module to analyze to "
+                                             "produce the CFG",
+                                             // The root container is
+                                             // manipulated to create the
+                                             // CFGMap, hence the need to
+                                             // declare Access::Read and a
+                                             // non-const argument.
+                                             Access::Read>,
+                             PipeRunArgument<CFGMap,
+                                             "Output",
+                                             "The produced CFG for each "
+                                             "function",
+                                             Access::Write>>;
+
+  CollectCFG(const class Model &Model,
+             llvm::StringRef Config,
+             llvm::StringRef DynamicConfig,
+             LLVMRootContainer &Input,
+             CFGMap &Output);
+  void runOnFunction(const model::Function &TheFunction);
+};
+
+} // namespace piperuns
+
+} // namespace revng::pypeline

include/revng/FunctionIsolation/EnforceABI.h

@@ -0,0 +1,41 @@
+#pragma once
+
+//
+// This file is distributed under the MIT License. See LICENSE.md for details.
+//
+
+#include "revng/BasicAnalyses/GeneratedCodeBasicInfo.h"
+#include "revng/EarlyFunctionAnalysis/CollectCFG.h"
+#include "revng/Model/NameBuilder.h"
+#include "revng/PipeboxCommon/LLVMContainer.h"
+
+class EnforceABI;
+
+namespace revng::pypeline::piperuns {
+
+class EnforceABI {
+private:
+  const model::Binary &Binary;
+  LLVMFunctionContainer &Output;
+  const CFGMap &CFG;
+  model::CNameBuilder NameBuilder;
+
+public:
+  static constexpr llvm::StringRef Name = "EnforceABI";
+  using Arguments = TypeList<
+    PipeRunArgument<const CFGMap, "CFG", "The per-function CFG data">,
+    PipeRunArgument<LLVMFunctionContainer,
+                    "Module",
+                    "The LLVM Module(s) to run on">>;
+
+  EnforceABI(const class Model &Model,
+             llvm::StringRef Config,
+             llvm::StringRef DynamicConfig,
+             const CFGMap &CFG,
+             LLVMFunctionContainer &Output) :
+    Binary(*Model.get().get()), Output(Output), CFG(CFG), NameBuilder(Binary){};
+
+  void runOnFunction(const model::Function &TheFunction);
+};
+
+} // namespace revng::pypeline::piperuns

include/revng/FunctionIsolation/InvokeIsolatedFunctions.h

@@ -0,0 +1,30 @@
+#pragma once
+
+//
+// This file is distributed under the MIT License. See LICENSE.md for details.
+//
+
+#include "revng/PipeboxCommon/LLVMContainer.h"
+
+namespace revng::pypeline::piperuns {
+
+class InvokeIsolatedFunctions {
+public:
+  static constexpr llvm::StringRef Name = "InvokeIsolatedFunctions";
+  using Arguments = TypeList<
+    PipeArgument<"RootModule", "Root module containing the root function">,
+    PipeArgument<"FunctionModules",
+                 "LLVM Modules containing isolated functions">,
+    PipeArgument<"Output",
+                 "Output LLVM Module with root, functions and dispatcher",
+                 Access::Write>>;
+
+  static void run(const class Model &Model,
+                  llvm::StringRef Config,
+                  llvm::StringRef DynamicConfig,
+                  const LLVMRootContainer &Root,
+                  const LLVMFunctionContainer &Functions,
+                  LLVMRootContainer &Output);
+};
+
+} // namespace revng::pypeline::piperuns

include/revng/FunctionIsolation/IsolateFunctions.h

@@ -9,6 +9,7 @@
 #include "llvm/Pass.h"
 
 #include "revng/BasicAnalyses/GeneratedCodeBasicInfo.h"
+#include "revng/EarlyFunctionAnalysis/CollectCFG.h"
 
 class IsolateFunctions : public llvm::ModulePass {
 public:
@@ -21,3 +22,48 @@ public:
 
   void getAnalysisUsage(llvm::AnalysisUsage &AU) const override;
 };
+
+class IsolateFunctionsImpl;
+
+namespace revng::pypeline::piperuns {
+
+class Isolate {
+private:
+  LLVMRootContainer &Root;
+  LLVMFunctionContainer &Output;
+  GeneratedCodeBasicInfo GCBI;
+  // unique_ptr to the implementation. This is a temporary measure until the
+  // old pipeline is dropped and the body of the `Impl` class can be inlined in
+  // this one.
+  std::unique_ptr<IsolateFunctionsImpl> Impl;
+  std::vector<std::tuple<MetaAddress, llvm::Function *>> IsolatedFunctions;
+
+public:
+  static constexpr llvm::StringRef Name = "Isolate";
+  using Arguments = TypeList<
+    PipeRunArgument<const CFGMap, "CFG", "Function control flow data">,
+    PipeRunArgument<LLVMRootContainer,
+                    "Input",
+                    "Input LLVM module to be isolated",
+                    // The root container is first modified in-place to be
+                    // isolated, then, as part of the destructor, the individual
+                    // functions are split and put in their respective module in
+                    // the LLVMFunctionContainer.
+                    Access::Read>,
+    PipeRunArgument<LLVMFunctionContainer,
+                    "Output",
+                    "Output LLVM modules with isolated functions",
+                    Access::Write>>;
+
+  Isolate(const class Model &Model,
+          llvm::StringRef Config,
+          llvm::StringRef DynamicConfig,
+          const CFGMap &CFG,
+          LLVMRootContainer &Root,
+          LLVMFunctionContainer &Output);
+  ~Isolate();
+
+  void runOnFunction(const model::Function &TheFunction);
+};
+
+} // namespace revng::pypeline::piperuns

include/revng/FunctionIsolation/PromoteCSVs.h

@@ -4,6 +4,30 @@
 // This file is distributed under the MIT License. See LICENSE.md for details.
 //
 
-#include "llvm/Pass.h"
-
 #include "revng/BasicAnalyses/GeneratedCodeBasicInfo.h"
+#include "revng/PipeboxCommon/LLVMContainer.h"
+
+namespace revng::pypeline::piperuns {
+
+class PromoteCSVs {
+private:
+  const model::Binary &Binary;
+  LLVMFunctionContainer &ModuleContainer;
+
+public:
+  static constexpr llvm::StringRef Name = "PromoteCSVs";
+  using Arguments = TypeList<PipeRunArgument<LLVMFunctionContainer,
+                                             "Module",
+                                             "The LLVM Module(s) where the CSV "
+                                             "will be promoted">>;
+
+  PromoteCSVs(const class Model &Model,
+              llvm::StringRef Config,
+              llvm::StringRef DynamicConfig,
+              LLVMFunctionContainer &ModuleContainer) :
+    Binary(*Model.get().get()), ModuleContainer(ModuleContainer) {}
+
+  void runOnFunction(const model::Function &TheFunction);
+};
+
+} // namespace revng::pypeline::piperuns

include/revng/HeadersGeneration/ModelToHeaderPipe.h

@@ -0,0 +1,34 @@
+#pragma once
+
+//
+// This file is distributed under the MIT License. See LICENSE.md for details.
+//
+
+#include "llvm/ADT/StringRef.h"
+
+#include "revng/ADT/TypeList.h"
+#include "revng/PipeboxCommon/Common.h"
+#include "revng/PipeboxCommon/Model.h"
+#include "revng/PipeboxCommon/RawContainer.h"
+
+namespace revng::pypeline {
+
+using CBytesContainer = BytesContainer<"CBytesContainer", "text/x.c+ptml">;
+
+namespace piperuns {
+
+class ModelToHeader {
+public:
+  static constexpr llvm::StringRef Name = "ModelToHeader";
+  using Arguments = TypeList<
+    PipeArgument<"Buffer", "The output C header of the model", Access::Write>>;
+
+  static void run(const Model &TheModel,
+                  llvm::StringRef StaticConfig,
+                  llvm::StringRef DynamicConfig,
+                  CBytesContainer &Buffer);
+};
+
+} // namespace piperuns
+
+} // namespace revng::pypeline

include/revng/HeadersGeneration/ModelTypeDefinitionPipe.h

@@ -0,0 +1,44 @@
+#pragma once
+
+//
+// This file is distributed under the MIT License. See LICENSE.md for details.
+//
+
+#include "revng/ADT/TypeList.h"
+#include "revng/PipeboxCommon/Common.h"
+#include "revng/PipeboxCommon/Model.h"
+#include "revng/PipeboxCommon/RawContainer.h"
+
+namespace revng::pypeline {
+
+using PTMLCTypeContainer = TypeDefinitionToBytesContainer<"PTMLCTypeContainer",
+                                                          "text/x.c+ptml">;
+
+namespace piperuns {
+
+class GenerateModelTypeDefinition {
+private:
+  const Model &Model;
+  PTMLCTypeContainer &Output;
+
+public:
+  static constexpr llvm::StringRef Name = "GenerateModelTypeDefinition";
+  using Arguments = TypeList<PipeRunArgument<PTMLCTypeContainer,
+                                             "Output",
+                                             "The output C headers of each "
+                                             "Type Definition",
+                                             Access::Write>>;
+
+  GenerateModelTypeDefinition(const class Model &Model,
+                              llvm::StringRef Config,
+                              llvm::StringRef DynamicConfig,
+                              PTMLCTypeContainer &Output) :
+    Model(Model), Output(Output) {}
+
+  void runOnTypeDefinition(const UpcastablePointer<model::TypeDefinition>
+                             &TypeDefinition);
+};
+
+} // namespace piperuns
+
+} // namespace revng::pypeline

include/revng/Lift/Lift.h

@@ -8,6 +8,9 @@
 #include "llvm/Pass.h"
 
 #include "revng/Lift/LoadBinaryPass.h"
+#include "revng/PipeboxCommon/BinariesContainer.h"
+#include "revng/PipeboxCommon/LLVMContainer.h"
+#include "revng/PipeboxCommon/Model.h"
 #include "revng/Pipeline/ExecutionContext.h"
 #include "revng/Pipes/FunctionPass.h"
 #include "revng/Support/Debug.h"
@@ -172,3 +175,31 @@ public:
 
   bool runOnModule(llvm::Module &M) override;
 };
+
+namespace detail {
+
+llvm::Error liftCheckPrecondition(const model::Binary &Model);
+
+}
+
+namespace revng::pypeline::piperuns {
+
+class Lift {
+public:
+  static constexpr llvm::StringRef Name = "Lift";
+  using Arguments = TypeList<PipeArgument<"Input", "Input binaries to lift">,
+                             PipeArgument<"Output",
+                                          "LLVM Module containing the lifted "
+                                          "binaries",
+                                          Access::Write>>;
+
+public:
+  static llvm::Error checkPrecondition(const class Model &Model);
+  static void run(const class Model &Model,
+                  llvm::StringRef Config,
+                  llvm::StringRef DynamicConfig,
+                  const BinariesContainer &Binary,
+                  LLVMRootContainer &ModuleContainer);
+};
+
+} // namespace revng::pypeline::piperuns

include/revng/Lift/LinkSupportPipe.h

@@ -8,6 +8,9 @@
 
 #include "llvm/ADT/ArrayRef.h"
 
+#include "revng/PipeboxCommon/Common.h"
+#include "revng/PipeboxCommon/LLVMContainer.h"
+#include "revng/PipeboxCommon/Model.h"
 #include "revng/Pipeline/ContainerSet.h"
 #include "revng/Pipeline/Contract.h"
 #include "revng/Pipeline/LLVMContainer.h"
@@ -32,3 +35,20 @@ public:
 };
 
 } // namespace revng::pipes
+
+namespace revng::pypeline::piperuns {
+
+class LinkSupport {
+public:
+  static constexpr llvm::StringRef Name = "LinkSupport";
+  using Arguments = TypeList<
+    PipeArgument<"Module", "Module to link support into">>;
+
+public:
+  static void run(const class Model &Model,
+                  llvm::StringRef Config,
+                  llvm::StringRef DynamicConfig,
+                  LLVMRootContainer &ModuleContainer);
+};
+
+} // namespace revng::pypeline::piperuns

include/revng/Model/RawBinaryView.h

@@ -74,6 +74,10 @@ public:
                          unsigned LoadSize,
                          bool IsLittleEndian) const;
 
+  // Factored-out checkPrecondition, should be added to each pipe that uses
+  // this class. Eventually this will be moved into Binary::verify.
+  static llvm::Error checkPrecondition(const model::Binary &Binary);
+
 private:
   std::pair<const model::Segment *, uint64_t>
   findOffsetInSegment(MetaAddress Address, uint64_t Size) const;

include/revng/Pipebox/LLVMPipe.h

@@ -0,0 +1,105 @@
+#pragma once
+
+//
+// This file is distributed under the MIT License. See LICENSE.md for details.
+//
+
+#include "llvm/IR/LegacyPassManager.h"
+#include "llvm/PassInfo.h"
+#include "llvm/Support/Progress.h"
+
+#include "revng/PipeboxCommon/Common.h"
+#include "revng/PipeboxCommon/LLVMContainer.h"
+#include "revng/PipeboxCommon/Model.h"
+
+namespace revng::pypeline::pipes {
+
+namespace detail {
+
+/// Base class for both PureLLVMPassesRootPipe and PureLLVMPassesPipe, which
+/// factors out some common logic. Both pipes run pure LLVM passes and do not
+/// read the model, hence the fact that they are plain pipes and not piperuns.
+class PureLLVMPassesPipeBase {
+public:
+  struct Configuration {
+    std::vector<std::string> Passes;
+    static Configuration parse(llvm::StringRef Input);
+  };
+
+private:
+  std::vector<const llvm::PassInfo *> PassInfos;
+
+protected:
+  std::string TaskName;
+
+public:
+  const std::string StaticConfiguration;
+  PureLLVMPassesPipeBase(llvm::StringRef StaticConfiguration);
+
+protected:
+  llvm::legacy::PassManager makePassManager() {
+    llvm::legacy::PassManager Manager;
+    for (const llvm::PassInfo *PassInfo : PassInfos)
+      Manager.add(PassInfo->createPass());
+    return Manager;
+  }
+};
+
+} // namespace detail
+
+class PureLLVMPassesRootPipe : public detail::PureLLVMPassesPipeBase {
+public:
+  static constexpr llvm::StringRef Name = "PureLLVMPassesRootPipe";
+  using Arguments = TypeList<
+    PipeArgument<"Module", "LLVM Module to apply the LLVM passes to">>;
+
+public:
+  revng::pypeline::ObjectDependencies
+  run(const Model &TheModel,
+      const revng::pypeline::Request &Incoming,
+      const revng::pypeline::Request &Outgoing,
+      llvm::StringRef Configuration,
+      LLVMRootContainer &Container) {
+    if (Outgoing[0].size() == 0)
+      return {};
+
+    revng_assert(Outgoing[0].size() == 1);
+    revng_assert(Outgoing[0][0]->kind() == Kinds::Binary);
+
+    llvm::Task T(1, TaskName);
+    T.advance("run on LLVMRootContainer", true);
+    llvm::legacy::PassManager Manager = makePassManager();
+    Manager.run(Container.getModule());
+
+    return {};
+  }
+};
+
+class PureLLVMPassesPipe : public detail::PureLLVMPassesPipeBase {
+public:
+  static constexpr llvm::StringRef Name = "PureLLVMPassesPipe";
+  using Arguments = TypeList<
+    PipeArgument<"Module", "LLVM Modules to apply the LLVM passes to">>;
+
+public:
+  revng::pypeline::ObjectDependencies
+  run(const Model &TheModel,
+      const revng::pypeline::Request &Incoming,
+      const revng::pypeline::Request &Outgoing,
+      llvm::StringRef Configuration,
+      LLVMFunctionContainer &Container) {
+
+    llvm::Task T(Outgoing[0].size(), TaskName);
+    llvm::legacy::PassManager Manager = makePassManager();
+    for (const ObjectID *Object : Outgoing[0]) {
+      revng_assert(Object->kind() == Kinds::Function);
+      const MetaAddress &Key = std::get<MetaAddress>(Object->key());
+      T.advance(Key.toString(), true);
+      Manager.run(Container.getModule(*Object));
+    }
+
+    return {};
+  }
+};
+
+} // namespace revng::pypeline::pipes

include/revng/Pipebox/TupleTreeContainer.h

@@ -0,0 +1,56 @@
+#pragma once
+
+//
+// This file is distributed under the MIT License. See LICENSE.md for details.
+//
+
+#include "revng/PipeboxCommon/Common.h"
+#include "revng/PipeboxCommon/ObjectID.h"
+
+namespace revng::pypeline {
+
+template<TupleTreeCompatible T, Kind TheKind, ConstexprString TheName>
+class TupleTreeContainer {
+public:
+  static constexpr llvm::StringRef Name = TheName;
+  static constexpr Kind Kind = TheKind;
+  static constexpr llvm::StringRef MimeType = "text/x.yaml";
+
+private:
+  std::map<ObjectID, TupleTree<T>> Map;
+
+public:
+  std::set<ObjectID> objects() const {
+    return std::views::keys(Map) | revng::to<std::set<ObjectID>>();
+  }
+
+  void
+  deserialize(const std::map<const ObjectID *, llvm::ArrayRef<char>> Data) {
+    for (const auto &[Key, Value] : Data) {
+      revng_assert(Key->kind() == Kind);
+      llvm::StringRef String{ Value.data(), Value.size() };
+      Map[*Key] = llvm::cantFail(TupleTree<T>::fromString(String));
+    }
+  }
+
+  std::map<ObjectID, Buffer>
+  serialize(const std::vector<const ObjectID *> Objects) const {
+    std::map<ObjectID, Buffer> Result;
+    for (const ObjectID *Object : Objects) {
+      llvm::raw_svector_ostream OS(Result[*Object].data());
+      Map.at(*Object).serialize(OS);
+    }
+    return Result;
+  }
+
+  bool verify() const { return true; }
+
+public:
+  bool contains(const ObjectID &Key) const { return Map.contains(Key); }
+  TupleTree<T> &getElement(const ObjectID &Key) { return Map[Key]; }
+  const TupleTree<T> &getElement(const ObjectID &Key) const {
+    return Map.at(Key);
+  }
+};
+
+} // namespace revng::pypeline

include/revng/PipeboxCommon/BinariesContainer.h

@@ -24,6 +24,7 @@ class BinariesContainer {
 public:
   static constexpr llvm::StringRef Name = "BinariesContainer";
   static constexpr Kind Kind = Kinds::Binary;
+  static constexpr llvm::StringRef MimeType = "application/x-tar";
 
 private:
   struct File {
@@ -85,6 +86,8 @@ public:
   bool verify() const { return true; }
 
 public:
+  size_t size() const { return Files.size(); }
+
   llvm::ArrayRef<char> getFile(size_t Index) const {
     revng_assert(Index < Files.size());
     return Files[Index].Contents.data();

include/revng/PipeboxCommon/Common.h

@@ -25,14 +25,42 @@ using ModelPath = std::string;
 using ObjectDependencies = std::vector<
   std::vector<std::pair<ObjectID, ModelPath>>>;
 
-template<ConstexprString N, ConstexprString HT>
-struct PipeArgumentDocumentation {
-  static constexpr llvm::StringRef Name = N;
-  static constexpr llvm::StringRef HelpText = HT;
+/// Defines the access of a container when declaring a Pipe{,Run}Argument. In
+/// PipeRuns there needs to be exactly one container with either Write or
+/// ReadWrite as that will be the one where the model dependencies will be
+/// tracked upon.
+enum class Access {
+  /// The requested container will only be read (request objects as a
+  /// dependency). Note that specifying this allows the container to be used
+  /// without `const`, this is intended but should really be used as a
+  /// last-resort in situations where the container remains conceptually const
+  /// but cannot be for performance reasons.
+  Read,
+  /// The requested container will be written (don't request objects as a
+  /// dependency, the pipe will produce them). This is required when a pipe is
+  /// the first to write to a container.
+  Write,
+  /// The requested container will be overwritten in-place (e.g. LLVM Pipe).
+  /// This still requests objects as a dependency.
+  ReadWrite,
+  /// Automatically detect the access based on const-ness
+  /// * const -> Read
+  /// * non-const -> ReadWrite
+  Auto,
 };
 
-template<typename T, ConstexprString N, ConstexprString HT>
-struct PipeArgument : public PipeArgumentDocumentation<N, HT> {
+template<ConstexprString N, ConstexprString HT, Access A = Access::Auto>
+struct PipeArgument {
+  static constexpr llvm::StringRef Name = N;
+  static constexpr llvm::StringRef HelpText = HT;
+  static constexpr Access Access = A;
+};
+
+template<typename T,
+         ConstexprString N,
+         ConstexprString HT,
+         Access A = Access::Auto>
+struct PipeRunArgument : public PipeArgument<N, HT, A> {
   using Type = T;
 };
 

include/revng/PipeboxCommon/Concepts.h

@@ -28,6 +28,7 @@ concept IsContainer = requires(T &A, const T &AConst) {
   requires HasName<T>;
   { T() } -> std::same_as<T>;
   { T::Kind } -> std::same_as<const Kind &>;
+  { T::MimeType } -> std::same_as<const llvm::StringRef &>;
   { AConst.objects() } -> std::same_as<std::set<ObjectID>>;
   { AConst.verify() } -> std::same_as<bool>;
   {
@@ -127,29 +128,29 @@ struct PipeRunTraits<T> {
 };
 
 template<typename T>
-concept IsPipeArgumentDocumentation = requires {
+concept IsPipeArgument = requires {
+  { T::Access } -> std::same_as<const revng::pypeline::Access &>;
   { T::Name } -> std::same_as<const llvm::StringRef &>;
   { T::HelpText } -> std::same_as<const llvm::StringRef &>;
 };
 
 template<SpecializationOf<TypeList> List>
-inline constexpr bool checkDocumentation() {
+inline constexpr bool checkArguments() {
   constexpr bool Size = std::tuple_size_v<List>;
   return compile_time::repeatAnd<Size>([]<size_t I>() {
-    return IsPipeArgumentDocumentation<std::tuple_element_t<I, List>>;
+    return IsPipeArgument<std::tuple_element_t<I, List>>;
   });
 }
 
 template<typename T>
-inline constexpr size_t
-  DocSize = std::tuple_size_v<typename T::ArgumentsDocumentation>;
+inline constexpr size_t ArgSize = std::tuple_size_v<typename T::Arguments>;
 
 } // namespace detail
 
 template<typename T>
-concept HasArgumentsDocumenation = requires {
-  requires StrictSpecializationOf<typename T::ArgumentsDocumentation, TypeList>;
-  requires detail::checkDocumentation<typename T::ArgumentsDocumentation>();
+concept HasArguments = requires {
+  requires StrictSpecializationOf<typename T::Arguments, TypeList>;
+  requires detail::checkArguments<typename T::Arguments>();
 };
 
 template<typename T>
@@ -160,6 +161,12 @@ concept IsPipe = requires(T &A, llvm::StringRef StaticConfig) {
   requires HasName<T>;
   { T(StaticConfig) } -> std::same_as<T>;
   { A.StaticConfiguration } -> std::same_as<const std::string &>;
-  requires HasArgumentsDocumenation<T>;
-  requires PipeRunTraits<T>::ContainerCount == detail::DocSize<T>;
+  requires HasArguments<T>;
+  requires PipeRunTraits<T>::ContainerCount == detail::ArgSize<T>;
+};
+
+/// Optional method that a pipe can implement
+template<typename T>
+concept HasCheckPrecondition = requires(const T &A, const Model &Model) {
+  { A.checkPrecondition(Model) } -> std::same_as<llvm::Error>;
 };

include/revng/PipeboxCommon/Helpers/PipeRunPipes/Base.h

@@ -12,28 +12,52 @@
 
 namespace detail {
 
-template<StrictSpecializationOf<TypeList> T>
+template<typename ContainerType>
+constexpr bool isReadOnly(const revng::pypeline::Access &Access) {
+  using AccessEnum = revng::pypeline::Access;
+  return Access == AccessEnum::Read
+         or (Access == AccessEnum::Auto and std::is_const_v<ContainerType>);
+}
+
+template<StrictSpecializationOf<TypeList> T, typename PipeRunT>
 constexpr size_t writableContainersCount() {
   size_t Result = 0;
   forEach<T>([&Result]<typename A, size_t I>() {
-    if constexpr (not std::is_const_v<A>)
+    using Argument = std::tuple_element_t<I, typename PipeRunT::Arguments>;
+    if constexpr (not isReadOnly<A>(Argument::Access))
       Result += 1;
   });
   return Result;
 }
 
-template<StrictSpecializationOf<TypeList> T>
-  requires(writableContainersCount<T>() == 1)
+template<StrictSpecializationOf<TypeList> T, typename PipeRunT>
+  requires(writableContainersCount<T, PipeRunT>() == 1)
 constexpr size_t writableContainerIndex() {
   int Result = -1;
   forEach<T>([&Result]<typename A, size_t I>() {
-    if constexpr (not std::is_const_v<A>) {
+    using Argument = std::tuple_element_t<I, typename PipeRunT::Arguments>;
+    if constexpr (not isReadOnly<A>(Argument::Access)) {
       Result = I;
     }
   });
   return Result;
 }
 
+template<typename T>
+concept HasPipeRunCheckPrecondition = requires(const Model &Model) {
+  { T::checkPrecondition(Model) } -> std::same_as<llvm::Error>;
+};
+
+template<typename T>
+struct CheckPreconditionMixin {};
+
+template<HasPipeRunCheckPrecondition T>
+struct CheckPreconditionMixin<T> {
+  llvm::Error checkPrecondition(const Model &Model) const {
+    return T::checkPrecondition(Model);
+  }
+};
+
 } // namespace detail
 
 template<typename T>
@@ -43,7 +67,7 @@ concept SingleOutputPipeBaseCompatible = requires {
 };
 
 template<SingleOutputPipeBaseCompatible T>
-class SingleOutputPipeBase {
+class SingleOutputPipeBase : public detail::CheckPreconditionMixin<T> {
 public:
   static constexpr llvm::StringRef Name = T::Name;
   using ContainerTypes = PipeRunContainerTypes<T>;
@@ -56,7 +80,7 @@ public:
 protected:
   static constexpr size_t ContainerCount = std::tuple_size_v<ContainerTypes>;
   static constexpr size_t
-    OutputContainerIndex = detail::writableContainerIndex<ContainerTypes>();
+    OutputContainerIndex = detail::writableContainerIndex<ContainerTypes, T>();
   using OutputContainerType = std::tuple_element_t<OutputContainerIndex,
                                                    ContainerTypes>;
 };

include/revng/PipeboxCommon/Helpers/PipeRunPipes/FunctionPipe.h

@@ -4,6 +4,8 @@
 // This file is distributed under the MIT License. See LICENSE.md for details.
 //
 
+#include "llvm/Support/Progress.h"
+
 #include "revng/PipeboxCommon/Helpers/PipeRunPipes/Base.h"
 #include "revng/PipeboxCommon/Helpers/PipeRunPipes/Helpers.h"
 
@@ -26,7 +28,7 @@ private:
   static_assert(Base::OutputContainerType::Kind == Kinds::Function);
 
 public:
-  using ArgumentsDocumentation = T::Arguments;
+  using Arguments = T::Arguments;
 
 public:
   template<typename... Args>
@@ -38,17 +40,26 @@ public:
       llvm::StringRef Configuration,
       Args &...Containers) {
     ObjectDependenciesHelper ODH(Model, Outgoing, this->ContainerCount);
+
+    llvm::Task T1(3, "Running " + this->Name);
+    T1.advance("prologue", true);
+
     T Instance(Model, this->StaticConfiguration, Configuration, Containers...);
     const model::Binary &Binary = *Model.get().get();
+    auto &RequestedFunctions = Outgoing.at(this->OutputContainerIndex);
 
-    for (const ObjectID *Object : Outgoing.at(this->OutputContainerIndex)) {
-      auto Committer = ODH.getCommitterFor(*Object, this->OutputContainerIndex);
-
+    T1.advance("run on functions", true);
+    llvm::Task T2(RequestedFunctions.size(), "Running pipe on functions");
+    for (const ObjectID *Object : RequestedFunctions) {
       const MetaAddress &Entry = std::get<MetaAddress>(Object->key());
+      T2.advance(Entry.toString(), true);
+
+      auto Committer = ODH.getCommitterFor(*Object, this->OutputContainerIndex);
       const model::Function &Function = Binary.Functions().at(Entry);
       Instance.runOnFunction(Function);
     }
 
+    T1.advance("epilogue");
     return ODH.takeDependencies();
   }
 };

include/revng/PipeboxCommon/Helpers/PipeRunPipes/Helpers.h

@@ -135,23 +135,23 @@ private:
 namespace detail {
 
 template<typename T>
-concept IsPipeArgument = requires {
-  requires IsPipeArgumentDocumentation<T>;
+concept IsPipeRunArgument = requires {
+  requires IsPipeArgument<T>;
   requires IsContainerArgument<typename T::Type>;
 };
 
 template<SpecializationOf<TypeList> List>
-inline constexpr bool checkArguments() {
+inline constexpr bool checkPipeRunArguments() {
   constexpr bool Size = std::tuple_size_v<List>;
   return compile_time::repeatAnd<Size>([]<size_t I>() {
-    return IsPipeArgument<std::tuple_element_t<I, List>>;
+    return IsPipeRunArgument<std::tuple_element_t<I, List>>;
   });
 }
 
 template<typename T>
-concept HasArguments = requires {
+concept HasPipeRunArguments = requires {
   requires StrictSpecializationOf<typename T::Arguments, TypeList>;
-  requires checkArguments<typename T::Arguments>();
+  requires checkPipeRunArguments<typename T::Arguments>();
 };
 
 template<typename T>
@@ -164,13 +164,13 @@ template<typename T>
 concept IsSingleObjectPipeRun = requires {
   requires HasName<T>;
   requires detail::HasStaticRun<T>;
-  requires not detail::HasArguments<T>;
+  requires not detail::HasPipeRunArguments<T>;
 };
 
 template<typename T>
 concept IsMultipleObjectsPipeRun = requires {
   requires HasName<T>;
-  requires detail::HasArguments<T>;
+  requires detail::HasPipeRunArguments<T>;
   requires not detail::HasStaticRun<T>;
 };
 

include/revng/PipeboxCommon/Helpers/PipeRunPipes/LLVMFunctionPassPipe.h

@@ -6,6 +6,7 @@
 
 #include "llvm/IR/LegacyPassManager.h"
 #include "llvm/Pass.h"
+#include "llvm/Support/Progress.h"
 
 #include "revng/Model/FunctionTags.h"
 #include "revng/PipeboxCommon/Helpers/PipeRunPipes/Base.h"
@@ -25,6 +26,13 @@ inline constexpr llvm::StringRef PassNamePrefix = "LLVMFunctionPassPipe "
 inline constexpr llvm::StringRef PassArgumentPrefix = "llvm-function-pass-pipe-"
                                                       "for-";
 
+template<StrictSpecializationOf<TypeList> TL>
+constexpr bool checkAnalyses() {
+  return compile_time::repeatAnd<std::tuple_size_v<TL>>([]<size_t I>() {
+    return std::is_base_of_v<llvm::Pass, std::tuple_element_t<I, TL>>;
+  });
+}
+
 } // namespace detail
 
 template<typename T>
@@ -32,6 +40,7 @@ concept IsLLVMFunctionPassPipe = requires(T &PipeRun,
                                           const model::Function &Function,
                                           llvm::Function &LLVMFunction) {
   requires IsMultipleObjectsPipeRun<T>;
+  requires detail::checkAnalyses<typename T::Analyses>();
   requires hasConstructor<
     T,
     // The Structure of the constructor is:
@@ -42,19 +51,23 @@ concept IsLLVMFunctionPassPipe = requires(T &PipeRun,
   { PipeRun.runOnFunction(Function, LLVMFunction) } -> std::same_as<void>;
 };
 
-template<typename T>
+template<IsLLVMFunctionPassPipe T>
 class LLVMFunctionPassPipe : public SingleOutputPipeBase<T> {
 private:
   using Base = SingleOutputPipeBase<T>;
-  using LLVMRootContainer = revng::pypeline::LLVMRootContainer;
-  static_assert(std::is_same_v<typename Base::OutputContainerType,
-                               LLVMRootContainer>);
+  static_assert(anyOf<typename Base::OutputContainerType,
+                      revng::pypeline::LLVMRootContainer,
+                      revng::pypeline::LLVMFunctionContainer>());
+  static constexpr bool
+    SingleModule = not std::is_same_v<typename Base::OutputContainerType,
+                                      revng::pypeline::LLVMFunctionContainer>;
   using ContainerTypesRef = detail::TupleWithRef<typename Base::ContainerTypes>;
 
   class Pass : public llvm::ModulePass {
   private:
     ObjectDependenciesHelper &ODH;
     const revng::pypeline::Request &Outgoing;
+    std::optional<T> PipeRun;
 
     const Model &Model;
     llvm::StringRef StaticConfiguration;
@@ -79,7 +92,17 @@ private:
       Model(Model),
       StaticConfiguration(StaticConfiguration),
       Configuration(Configuration),
-      Containers(Containers) {}
+      Containers(Containers) {
+
+      compile_time::callWithIndexSequence<
+        Base::ContainerCount>([&]<size_t... I>() {
+        PipeRun.emplace(*this,
+                        Model,
+                        StaticConfiguration,
+                        Configuration,
+                        std::get<I>(Containers)...);
+      });
+    }
 
     void getAnalysisUsage(llvm::AnalysisUsage &AU) const override {
       forEach<typename T::Analyses>([&AU]<typename A, size_t I>() {
@@ -88,37 +111,64 @@ private:
     }
 
     bool runOnModule(llvm::Module &Module) override {
-      std::map<MetaAddress, llvm::Function *> AddressToFunction;
-      for (llvm::Function &Function : Module.functions()) {
-        if (not FunctionTags::Isolated.isTagOf(&Function))
-          continue;
+      if constexpr (SingleModule)
+        return runOnSingleModule(Module);
+      else
+        return runOnFunctionModule(Module);
+    }
 
-        AddressToFunction.emplace(getMetaAddressOfIsolatedFunction(Function),
-                                  &Function);
+    bool runOnSingleModule(llvm::Module &Module) {
+      std::map<MetaAddress, llvm::Function *> AddressToFunction;
+      for (llvm::Function *Function : getFunctions(Module)) {
+        AddressToFunction.emplace(getMetaAddressOfIsolatedFunction(*Function),
+                                  Function);
       }
 
-      T Instance = compile_time::callWithIndexSequence<
-        Base::ContainerCount>([&]<size_t... I>() {
-        return T{ *this,
-                  Model,
-                  this->StaticConfiguration,
-                  Configuration,
-                  std::get<I>(Containers)... };
-      });
-
       const model::Binary &Binary = *Model.get().get();
-      for (const ObjectID *Object : Outgoing.at(Base::OutputContainerIndex)) {
+      auto &RequestedFunctions = Outgoing.at(Base::OutputContainerIndex);
+      llvm::Task T1(RequestedFunctions.size(), "Running pipe on functions");
+      for (const ObjectID *Object : RequestedFunctions) {
+        const MetaAddress &Entry = std::get<MetaAddress>(Object->key());
+        T1.advance(Entry.toString(), true);
+
         auto Committer = ODH.getCommitterFor(*Object,
                                              Base::OutputContainerIndex);
-
-        const MetaAddress &Entry = std::get<MetaAddress>(Object->key());
         const model::Function &Function = Binary.Functions().at(Entry);
         llvm::Function *LLVMFunction = AddressToFunction.at(Function.Entry());
-        Instance.runOnFunction(Function, *LLVMFunction);
+        PipeRun->runOnFunction(Function, *LLVMFunction);
       }
 
       return true;
     }
+
+    bool runOnFunctionModule(llvm::Module &Module) {
+      std::set<llvm::Function *> Functions = getFunctions(Module);
+      revng_assert(Functions.size() == 1);
+      llvm::Function &LLVMFunction = **Functions.begin();
+      MetaAddress Address = getMetaAddressOfIsolatedFunction(LLVMFunction);
+      ObjectID Object(Address);
+      const model::Binary &Binary = *Model.get().get();
+
+      {
+        auto Committer = ODH.getCommitterFor(Object,
+                                             Base::OutputContainerIndex);
+        const model::Function &Function = Binary.Functions().at(Address);
+        PipeRun->runOnFunction(Function, LLVMFunction);
+      }
+
+      return true;
+    }
+
+  private:
+    static std::set<llvm::Function *> getFunctions(llvm::Module &Module) {
+      std::set<llvm::Function *> Result;
+      for (llvm::Function &Function : Module.functions()) {
+        if (FunctionTags::Isolated.isTagOf(&Function)
+            and not Function.isDeclaration())
+          Result.insert(&Function);
+      }
+      return Result;
+    }
   };
 
 private:
@@ -131,7 +181,7 @@ private:
                                                           true };
 
 public:
-  using ArgumentsDocumentation = T::Arguments;
+  using Arguments = T::Arguments;
 
 public:
   template<typename... Args>
@@ -149,6 +199,8 @@ public:
       Manager.add(new A());
     });
 
+    llvm::Task T1(3, "Running " + this->Name);
+    T1.advance("prologue", true);
     ContainerTypesRef ContainersRef(Containers...);
     Manager.add(new Pass(ODH,
                          Outgoing,
@@ -156,10 +208,23 @@ public:
                          this->StaticConfiguration,
                          Configuration,
                          ContainersRef));
-    LLVMRootContainer
-      &ModuleContainer = std::get<Base::OutputContainerIndex>(ContainersRef);
-    Manager.run(ModuleContainer.getModule());
+    auto &ModuleContainer = std::get<Base::OutputContainerIndex>(ContainersRef);
 
+    T1.advance("run on functions", true);
+    if constexpr (SingleModule) {
+      Manager.run(ModuleContainer.getModule());
+    } else {
+      auto &RequestedFunctions = Outgoing[this->OutputContainerIndex];
+      llvm::Task T2(RequestedFunctions.size(), "Running pipe on functions");
+      for (const ObjectID *Object : RequestedFunctions) {
+        MetaAddress Address = std::get<MetaAddress>(Object->key());
+        T2.advance(Address.toString(), true);
+
+        Manager.run(ModuleContainer.getModule(Address));
+      }
+    }
+
+    T1.advance("epilogue", true);
     return ODH.takeDependencies();
   }
 };

include/revng/PipeboxCommon/Helpers/PipeRunPipes/SingleOutputPipe.h

@@ -4,6 +4,8 @@
 // This file is distributed under the MIT License. See LICENSE.md for details.
 //
 
+#include "llvm/Support/Progress.h"
+
 #include "revng/PipeboxCommon/Concepts.h"
 #include "revng/PipeboxCommon/Helpers/PipeRunPipes/Base.h"
 #include "revng/PipeboxCommon/Helpers/PipeRunPipes/Helpers.h"
@@ -12,7 +14,7 @@
 template<typename T>
 concept IsSingleOutputPipeRun = requires {
   requires IsSingleObjectPipeRun<T>;
-  requires HasArgumentsDocumenation<T>;
+  requires HasArguments<T>;
   requires SpecializationOf<PipeRunContainerTypes<T>, TypeList>;
 };
 
@@ -22,7 +24,7 @@ private:
   using Base = SingleOutputPipeBase<T>;
 
 public:
-  using ArgumentsDocumentation = T::ArgumentsDocumentation;
+  using Arguments = T::Arguments;
 
 public:
   template<typename... Args>
@@ -33,9 +35,14 @@ public:
       const revng::pypeline::Request &Outgoing,
       llvm::StringRef Configuration,
       Args &...Containers) {
-    revng_assert(Outgoing.at(this->OutputContainerIndex).size() == 1);
-
     ObjectDependenciesHelper ODH(Model, Outgoing, this->ContainerCount);
+    auto &RequestedOutputs = Outgoing.at(this->OutputContainerIndex);
+    if (RequestedOutputs.size() == 0)
+      return ODH.takeDependencies();
+
+    revng_assert(RequestedOutputs.size() == 1);
+    llvm::Task T1(1, "Running " + this->Name);
+    T1.advance("Running 'run'", true);
     T::run(Model, this->StaticConfiguration, Configuration, Containers...);
     ODH.commitUniqueTarget(this->OutputContainerIndex);
     return ODH.takeDependencies();

include/revng/PipeboxCommon/Helpers/PipeRunPipes/TypeDefinitionPipe.h

@@ -4,6 +4,8 @@
 // This file is distributed under the MIT License. See LICENSE.md for details.
 //
 
+#include "llvm/Support/Progress.h"
+
 #include "revng/PipeboxCommon/Helpers/PipeRunPipes/Base.h"
 #include "revng/PipeboxCommon/Helpers/PipeRunPipes/Helpers.h"
 
@@ -28,7 +30,7 @@ private:
   static_assert(Base::OutputContainerType::Kind == Kinds::TypeDefinition);
 
 public:
-  using ArgumentsDocumentation = T::Arguments;
+  using Arguments = T::Arguments;
 
 public:
   template<typename... Args>
@@ -40,18 +42,29 @@ public:
       llvm::StringRef Configuration,
       Args &...Containers) {
     ObjectDependenciesHelper ODH(Model, Outgoing, this->ContainerCount);
+
+    llvm::Task T1(3, "Running " + this->Name);
+    T1.advance("prologue");
+
     T Instance(Model, this->StaticConfiguration, Configuration, Containers...);
     const model::Binary &Binary = *Model.get().get();
+    auto &RequestedTypeDefinitions = Outgoing.at(this->OutputContainerIndex);
 
-    for (const ObjectID *Object : Outgoing.at(this->OutputContainerIndex)) {
-      auto Committer = ODH.getCommitterFor(*Object, this->OutputContainerIndex);
-
+    T1.advance("run on type definitions", true);
+    llvm::Task T2(RequestedTypeDefinitions.size(),
+                  "Running pipe on type definitions");
+    for (const ObjectID *Object : RequestedTypeDefinitions) {
       using TDK = model::TypeDefinition::Key;
       const TDK &Entry = std::get<TDK>(Object->key());
+      T2.advance(toString(Entry), true);
+
+      auto Committer = ODH.getCommitterFor(*Object, this->OutputContainerIndex);
       const UpcastablePointer<model::TypeDefinition>
         &TypeDefinition = Binary.TypeDefinitions().at(Entry);
       Instance.runOnTypeDefinition(TypeDefinition);
     }
+
+    T1.advance("epilogue");
     return ODH.takeDependencies();
   }
 };

include/revng/PipeboxCommon/Helpers/Python/Helpers.h

@@ -12,6 +12,7 @@
 
 #include "revng/PipeboxCommon/Common.h"
 #include "revng/PipeboxCommon/Helpers/Helpers.h"
+#include "revng/PipeboxCommon/Model.h"
 #include "revng/PipeboxCommon/ObjectID.h"
 
 namespace revng::pypeline::helpers {
@@ -71,6 +72,22 @@ public:
   const Py_buffer *operator->() { return &Buffer; }
 };
 
+inline Model &convertReadOnlyModel(nanobind::object &TheModel) {
+  nanobind::object ReadOnlyModel = importObject("revng.pypeline.model."
+                                                "ReadOnlyModel");
+  revng_assert(nanobind::isinstance(TheModel, ReadOnlyModel));
+  nanobind::object ActualModel = nanobind::getattr(TheModel, "downcast")();
+  return *nanobind::cast<Model *>(ActualModel);
+}
+
+template<typename T, typename... ArgsT>
+inline nanobind::capsule makeCapsule(ArgsT &&...Args) {
+  return nanobind::capsule(new T(std::forward<ArgsT>(Args)...),
+                           [](void *Ptr) noexcept {
+                             delete static_cast<T *>(Ptr);
+                           });
+}
+
 } // namespace python
 
 } // namespace revng::pypeline::helpers

include/revng/PipeboxCommon/Helpers/Python/RunPipe.h

@@ -21,14 +21,9 @@ inline ObjectDependencies runPipe(T &Handle,
                                   Request Incoming,
                                   Request Outgoing,
                                   llvm::StringRef Configuration) {
-  nanobind::object ReadOnlyModel = importObject("revng.pypeline.model."
-                                                "ReadOnlyModel");
-  revng_assert(nanobind::isinstance(TheModel, ReadOnlyModel));
-  nanobind::object ActualModel = nanobind::getattr(TheModel, "downcast")();
-  const Model *CppModel = nanobind::cast<Model *>(ActualModel);
-
+  const Model &CppModel = convertReadOnlyModel(TheModel);
   return revng::pypeline::helpers::runPipe(Handle,
-                                           *CppModel,
+                                           CppModel,
                                            Incoming,
                                            Outgoing,
                                            Configuration,

include/revng/PipeboxCommon/Helpers/Python/SignatureHelper.h

@@ -62,8 +62,7 @@ private:
     forEach<CT>([&Result,
                  &TaskArgument,
                  &TaskArgumentAccess]<typename A, size_t I>() {
-      using Argument = std::tuple_element_t<I,
-                                            typename T::ArgumentsDocumentation>;
+      using Argument = std::tuple_element_t<I, typename T::Arguments>;
       // Create a Kwargs dictionary, this will be passed to the constructor of
       // TaskArgument
       nanobind::dict Kwargs;
@@ -75,11 +74,8 @@ private:
       // nanobind::type<T> returns a reference, so we need to borrow it and
       // increase the reference count
       Kwargs["container_type"] = nanobind::borrow(nanobind::type<A>());
-      // If the argument is const then the access is READ, otherwise READ_WRITE
-      if constexpr (std::is_const_v<A>)
-        Kwargs["access"] = nanobind::getattr(TaskArgumentAccess, "READ");
-      else
-        Kwargs["access"] = nanobind::getattr(TaskArgumentAccess, "READ_WRITE");
+      Kwargs["access"] = nanobind::getattr(TaskArgumentAccess,
+                                           getAccess<A>(Argument::Access));
 
       // kwargs_proxy is a special nanobind class that allows passing a
       // nanobind::dict as kwargs, this is equivalent to doing
@@ -92,6 +88,26 @@ private:
     return nanobind::tuple(Result);
   }
 
+  /// Given a container type A and a pipe's Access, determine the correct member
+  /// of the `TaskArgumentAccess` python enum.
+  template<typename A>
+  static const char *getAccess(const Access &Access) {
+    switch (Access) {
+    case Access::Read:
+      return "READ";
+    case Access::Write:
+      return "WRITE";
+    case Access::ReadWrite:
+      return "READ_WRITE";
+    case Access::Auto:
+      // If the argument is const then the access is READ, otherwise READ_WRITE
+      if constexpr (std::is_const_v<A>)
+        return "READ";
+      else
+        return "READ_WRITE";
+    }
+  }
+
   /// Given an analysis type T, compute its signature
   static nanobind::object parseAnalysisSignature() {
     nanobind::list Result;

include/revng/PipeboxCommon/Helpers/Registrars.h

@@ -24,6 +24,8 @@
 #include "revng/PipeboxCommon/Helpers/Python/RunPipe.h"
 #include "revng/PipeboxCommon/Helpers/Python/SignatureHelper.h"
 
+inline Logger PypelineRegisterLogger("pypeline-register");
+
 template<IsAnalysis T>
 struct RegisterAnalysis {
   RegisterAnalysis() {
@@ -33,7 +35,8 @@ struct RegisterAnalysis {
     // Python
     python::Registry.registerModuleInitializer([](nanobind::module_ &M,
                                                   python::BaseClasses &BC) {
-      nanobind::class_<T>(M, T::Name.data(), BC.BaseAnalysis)
+      std::string Name = T::Name.str();
+      nanobind::class_<T>(M, Name.c_str(), BC.BaseAnalysis)
         .def_ro_static("name", &T::Name)
         .def(nanobind::init<>())
         .def_static("signature",
@@ -66,8 +69,10 @@ struct RegisterContainer {
     // Python
     python::Registry.registerModuleInitializer([](nanobind::module_ &M,
                                                   python::BaseClasses &BC) {
-      nanobind::class_<T>(M, T::Name.data(), BC.BaseContainer)
+      std::string Name = T::Name.str();
+      nanobind::class_<T>(M, Name.c_str(), BC.BaseContainer)
         .def_ro_static("kind", &T::Kind)
+        .def_static("mime_type", []() { return T::MimeType; })
         .def(nanobind::init<>())
         .def("objects", &python::ContainerIO<T>::objects)
         .def("verify", &T::verify)
@@ -84,17 +89,41 @@ struct RegisterContainer {
   }
 };
 
+namespace detail {
+
+template<typename T>
+void checkPipeArgumentAccess() {
+  if (not PypelineRegisterLogger.isEnabled())
+    return;
+
+  using CT = PipeRunTraits<T>::ContainerTypes;
+  using AccessEnum = revng::pypeline::Access;
+  forEach<CT>([]<typename A, size_t I>() {
+    using Argument = std::tuple_element_t<I, typename T::Arguments>;
+    if (Argument::Access == AccessEnum::Read and not std::is_const_v<A>) {
+      revng_log(PypelineRegisterLogger,
+                T::Name << " has the " << I
+                        << "th argument with READ access but marked non-const");
+    }
+  });
+}
+
+} // namespace detail
+
 template<IsPipe T>
 struct RegisterPipe {
   RegisterPipe() {
     using namespace nanobind::literals;
     using namespace revng::pypeline::helpers;
 
+    detail::checkPipeArgumentAccess<T>();
+
     // Python
     python::Registry.registerModuleInitializer([](nanobind::module_ &M,
                                                   python::BaseClasses &BC) {
-      nanobind::class_<T>(M, T::Name.data(), BC.BasePipe)
-        .def_ro_static("name", &T::Name)
+      std::string Name = T::Name.str();
+      auto PipeClass = nanobind::class_<T>(M, Name.c_str(), BC.BasePipe);
+      PipeClass.def_ro_static("name", &T::Name)
         .def_static("signature",
                     &python::SignatureHelper<T>::getSignature,
                     nanobind::sig("def signature() -> "
@@ -111,6 +140,14 @@ struct RegisterPipe {
              "incoming"_a,
              "outgoing"_a,
              "configuration"_a);
+
+      if constexpr (HasCheckPrecondition<T>) {
+        PipeClass
+          .def("check_precondition", [](T &Handle, nanobind::object TheModel) {
+            const Model &CppModel = python::convertReadOnlyModel(TheModel);
+            return Handle.checkPrecondition(CppModel);
+          });
+      }
     });
 
     // Native

include/revng/PipeboxCommon/LLVMContainer.h

@@ -8,12 +8,10 @@
 #include "llvm/Bitcode/BitcodeWriter.h"
 #include "llvm/IR/LLVMContext.h"
 #include "llvm/IR/Module.h"
-#include "llvm/IR/Verifier.h"
 
 #include "revng/PipeboxCommon/Common.h"
 #include "revng/PipeboxCommon/ObjectID.h"
 #include "revng/Support/IRHelpers.h"
-#include "revng/Support/ZstdStream.h"
 
 namespace revng::pypeline {
 
@@ -21,6 +19,7 @@ class LLVMRootContainer {
 public:
   static constexpr llvm::StringRef Name = "LLVMRootContainer";
   static constexpr Kind Kind = Kinds::Binary;
+  static constexpr llvm::StringRef MimeType = "application/x.llvm.bc";
 
 private:
   llvm::LLVMContext Context;
@@ -45,14 +44,9 @@ public:
       return;
 
     revng_assert(Data.size() == 1);
-    for (const auto &[Key, Value] : Data) {
-      revng_assert(Key->kind() == Kind);
-
-      llvm::SmallVector<char> DecompressedData = zstdDecompress(Value);
-      llvm::MemoryBufferRef Ref{
-        { DecompressedData.data(), DecompressedData.size() }, "input"
-      };
-
+    for (const auto &[Object, Buffer] : Data) {
+      revng_assert(Object->kind() == Kind);
+      llvm::MemoryBufferRef Ref{ { Buffer.data(), Buffer.size() }, "input" };
       Module = llvm::cantFail(llvm::parseBitcodeFile(Ref, Context));
     }
   }
@@ -64,10 +58,8 @@ public:
 
     revng_assert(Objects.size() == 1 and Objects[0]->kind() == Kind);
     std::map<ObjectID, Buffer> Result;
-    llvm::raw_svector_ostream OS(Result[*Objects[0]].data());
-    ZstdCompressedOstream CompressedOS(OS, 3);
-    llvm::WriteBitcodeToFile(*Module, CompressedOS);
-    CompressedOS.flush();
+    writeBitcode(*Module, Result[*Objects[0]].data());
+
     return Result;
   }
 
@@ -79,6 +71,73 @@ public:
 public:
   const llvm::Module &getModule() const { return *Module; }
   llvm::Module &getModule() { return *Module; }
+
+  void assign(std::unique_ptr<llvm::Module> &&NewModule) {
+    if (&Context == &NewModule->getContext())
+      Module = std::move(NewModule);
+    else
+      Module = cloneIntoContext(*NewModule, Context);
+  }
+};
+
+class LLVMFunctionContainer {
+public:
+  static constexpr llvm::StringRef Name = "LLVMFunctionContainer";
+  static constexpr Kind Kind = Kinds::Function;
+  static constexpr llvm::StringRef MimeType = "application/x.llvm.bc";
+
+private:
+  llvm::LLVMContext Context;
+  std::map<ObjectID, std::unique_ptr<llvm::Module>> Modules;
+
+public:
+  LLVMFunctionContainer() {}
+
+public:
+  std::set<ObjectID> objects() const {
+    return std::views::keys(Modules) | revng::to<std::set<ObjectID>>();
+  }
+
+  void
+  deserialize(const std::map<const ObjectID *, llvm::ArrayRef<char>> Data) {
+    for (auto const &[Object, Buffer] : Data) {
+      llvm::MemoryBufferRef BufferRef({ Buffer.data(), Buffer.size() },
+                                      "newBuffer");
+      Modules[*Object] = llvm::cantFail(llvm::parseBitcodeFile(BufferRef,
+                                                               Context));
+    }
+  }
+
+  std::map<ObjectID, Buffer>
+  serialize(const std::vector<const ObjectID *> Objects) const {
+    std::map<ObjectID, Buffer> Result;
+    for (const ObjectID *Object : Objects)
+      writeBitcode(*Modules.at(*Object), Result[*Object].data());
+    return Result;
+  }
+
+  bool verify() const {
+    for (auto const &[_, Module] : Modules)
+      revng::forceVerify(&*Module);
+    return true;
+  }
+
+public:
+  llvm::LLVMContext &getContext() { return Context; }
+  const llvm::LLVMContext &getContext() const { return Context; }
+
+  const llvm::Module &getModule(const ObjectID &ID) const {
+    return *Modules.at(ID);
+  }
+
+  llvm::Module &getModule(const ObjectID &ID) { return *Modules.at(ID); }
+
+  void assign(const ObjectID &ID, std::unique_ptr<llvm::Module> &&NewModule) {
+    if (&Context == &NewModule->getContext())
+      Modules[ID] = std::move(NewModule);
+    else
+      Modules[ID] = cloneIntoContext(*NewModule, Context);
+  }
 };
 
 } // namespace revng::pypeline

include/revng/PipeboxCommon/RawContainer.h

@@ -11,11 +11,12 @@ namespace revng::pypeline {
 
 namespace detail {
 
-template<Kind TheKind, ConstexprString TheName>
+template<Kind TheKind, ConstexprString TheName, ConstexprString TheMime>
 class RawContainer {
 public:
   static constexpr llvm::StringRef Name = TheName;
   static constexpr Kind Kind = TheKind;
+  static constexpr llvm::StringRef MimeType = TheMime;
 
 private:
   std::map<ObjectID, Buffer> Map;
@@ -49,7 +50,7 @@ public:
 public:
   bool contains(const ObjectID &Key) const { return Map.contains(Key); }
 
-  std::unique_ptr<llvm::raw_ostream> getOStream(const ObjectID &Key) {
+  std::unique_ptr<llvm::raw_pwrite_stream> getOStream(const ObjectID &Key) {
     revng_assert(Key.kind() == Kind);
     revng_assert(not Map.contains(Key));
     return std::make_unique<llvm::raw_svector_ostream>(Map[Key].data());
@@ -63,19 +64,20 @@ public:
   }
 };
 
-template<Kind K, ConstexprString S>
-using RC = RawContainer<K, S>;
+template<Kind K, ConstexprString S, ConstexprString S2>
+using RC = RawContainer<K, S, S2>;
+
+constexpr auto TD = Kinds::TypeDefinition;
 
 } // namespace detail
 
-using BytesContainer = detail::RC<Kinds::Binary, "BytesContainer">;
+template<ConstexprString Name, ConstexprString Mime>
+using BytesContainer = detail::RC<Kinds::Binary, Name, Mime>;
 
-using FunctionToBytesContainer = detail::RC<Kinds::Function,
-                                            "FunctionToBytesContaine"
-                                            "r">;
+template<ConstexprString Name, ConstexprString Mime>
+using FunctionToBytesContainer = detail::RC<Kinds::Function, Name, Mime>;
 
-using TypeDefinitionToBytesContainer = detail::RC<Kinds::TypeDefinition,
-                                                  "TypeDefinitionToBytesContain"
-                                                  "er">;
+template<ConstexprString Name, ConstexprString Mime>
+using TypeDefinitionToBytesContainer = detail::RC<detail::TD, Name, Mime>;
 
 } // namespace revng::pypeline

include/revng/Pipes/FunctionPass.h

@@ -29,6 +29,7 @@ protected:
 
 public:
   FunctionPassImpl(llvm::ModulePass &Pass) : Pass(&Pass) {}
+  FunctionPassImpl() : Pass(nullptr){};
 
   virtual ~FunctionPassImpl() = default;
 
@@ -43,6 +44,7 @@ public:
 public:
   template<typename T, typename... ArgType>
   T &getAnalysis(ArgType &&...Arg) {
+    revng_check(Pass != nullptr);
     return Pass->getAnalysis<T>(std::forward<ArgType>(Arg)...);
   }
 };

include/revng/Recompile/CompileModulePipe.h

@@ -8,6 +8,9 @@
 
 #include "llvm/ADT/ArrayRef.h"
 
+#include "revng/PipeboxCommon/LLVMContainer.h"
+#include "revng/PipeboxCommon/Model.h"
+#include "revng/PipeboxCommon/RawContainer.h"
 #include "revng/Pipeline/ContainerSet.h"
 #include "revng/Pipeline/Context.h"
 #include "revng/Pipeline/Contract.h"
@@ -52,3 +55,30 @@ public:
 };
 
 } // namespace revng::pipes
+
+namespace revng::pypeline {
+
+using ObjectFileContainer = BytesContainer<"ObjectFileContainer",
+                                           "application/x-object">;
+
+namespace piperuns {
+
+class CompileRootModule {
+public:
+  static constexpr llvm::StringRef Name = "CompileRootModule";
+  using Arguments = TypeList<
+    PipeArgument<"Input",
+                 "The LLVM module that will be compiled",
+                 Access::Read>,
+    PipeArgument<"Output", "The compiled object file", Access::Write>>;
+
+  static void run(const Model &TheModel,
+                  llvm::StringRef StaticConfig,
+                  llvm::StringRef DynamicConfig,
+                  LLVMRootContainer &Input,
+                  ObjectFileContainer &Output);
+};
+
+} // namespace piperuns
+
+} // namespace revng::pypeline

include/revng/Recompile/LinkForTranslationPipe.h

@@ -8,6 +8,9 @@
 
 #include "llvm/ADT/ArrayRef.h"
 
+#include "revng/PipeboxCommon/BinariesContainer.h"
+#include "revng/PipeboxCommon/Model.h"
+#include "revng/PipeboxCommon/RawContainer.h"
 #include "revng/Pipeline/ContainerSet.h"
 #include "revng/Pipeline/Context.h"
 #include "revng/Pipeline/Contract.h"
@@ -16,6 +19,7 @@
 #include "revng/Pipeline/Target.h"
 #include "revng/Pipes/FileContainer.h"
 #include "revng/Pipes/Kinds.h"
+#include "revng/Recompile/CompileModulePipe.h"
 
 namespace revng::pipes {
 
@@ -44,3 +48,32 @@ public:
 };
 
 } // namespace revng::pipes
+
+namespace revng::pypeline {
+
+using TranslatedContainer = BytesContainer<"TranslatedContainer",
+                                           "application/x-executable">;
+
+namespace piperuns {
+
+class LinkForTranslation {
+public:
+  static constexpr llvm::StringRef Name = "LinkForTranslation";
+  using Arguments = TypeList<
+    PipeArgument<"Binaries", "The input binaries">,
+    PipeArgument<"ObjectFile", "The complied object file">,
+    PipeArgument<"Output", "The output executable", Access::Write>>;
+
+  static llvm::Error checkPrecondition(const class Model &Model);
+
+  static void run(const Model &TheModel,
+                  llvm::StringRef StaticConfig,
+                  llvm::StringRef DynamicConfig,
+                  const BinariesContainer &Binaries,
+                  const ObjectFileContainer &ObjectFile,
+                  TranslatedContainer &Output);
+};
+
+} // namespace piperuns
+
+} // namespace revng::pypeline

include/revng/Support/Error.h

@@ -33,16 +33,17 @@ inline T cantFail(llvm::ErrorOr<T> Obj) {
 template<RangeOf<llvm::Error> T>
 inline llvm::Error joinErrors(T &Container) {
   auto Size = std::ranges::distance(Container);
-  revng_check(Size > 0);
+  if (Size == 0)
+    return llvm::Error::success();
 
   auto Iter = Container.begin();
   llvm::Error Result{ std::move(*Iter) };
-  if (Size == 1) {
+  if (Size == 1)
     return Result;
-  }
-  for (Iter++; Iter < Container.end(); Iter++) {
+
+  for (Iter++; Iter < Container.end(); Iter++)
     Result = llvm::joinErrors(std::move(Result), std::move(*Iter));
-  }
+
   return Result;
 }
 

include/revng/Support/IRHelpers.h

@@ -1546,3 +1546,18 @@ inline void linkModules(std::unique_ptr<llvm::Module> &&Source,
                         llvm::Module &Destination) {
   linkModules(std::move(Source), Destination, std::nullopt);
 }
+
+/// Clone the module, keeping only the functions specified in \p ToClone .
+/// This requires the source module to be writable in order to guarantee that
+/// the metadata attached to the cloned functions is also preserved.
+std::unique_ptr<llvm::Module>
+cloneFiltered(llvm::Module &Module, std::set<const llvm::Function *> &ToClone);
+
+/// Serialize a module to bitcode, this is preferable over
+/// llvm::WriteBitcodeToFile as it does not create a redundant temporary buffer
+void writeBitcode(const llvm::Module &Module,
+                  llvm::SmallVectorImpl<char> &Output);
+
+/// Copy a module to a new LLVMContext.
+std::unique_ptr<llvm::Module> cloneIntoContext(const llvm::Module &Module,
+                                               llvm::LLVMContext &NewContext);

include/revng/Support/InitRevng.h

@@ -19,14 +19,24 @@ namespace revng {
 /// By default this performs the regular LLVM initialization steps.
 /// This is required in order to initialize the stack trace printers on signal.
 class InitRevng : public llvm::InitLLVM {
+private:
+  static inline bool Initialized = false;
+
 public:
+  InitRevng(int &Argc, auto **&Argv, const char *Overview) :
+    InitRevng(Argc, Argv, Overview, {}) {}
+
   InitRevng(int &Argc,
             auto **&Argv,
             const char *Overview,
             llvm::ArrayRef<const llvm::cl::OptionCategory *> CategoriesToHide) :
     InitLLVM(Argc, Argv, true) {
 
+    revng_assert(not Initialized);
+    Initialized = true;
+
     OnQuit->install();
+    initializeLLVMLibraries();
 
     llvm::setBugReportMsg("PLEASE submit a bug report to "
                           "https://github.com/revng/revng and include the "
@@ -63,6 +73,9 @@ public:
   }
 
   ~InitRevng() { OnQuit->quit(); }
+
+private:
+  void initializeLLVMLibraries();
 };
 
 } // namespace revng

include/revng/Yield/Assembly/DisassemblyHelper.h

@@ -33,7 +33,18 @@ public:
                               const efa::ControlFlowGraph &Metadata,
                               const RawBinaryView &BinaryView,
                               const model::Binary &Binary,
-                              const model::AssemblyNameBuilder &NameBuilder);
+                              const model::AssemblyNameBuilder &NameBuilder) {
+    yield::Function Result;
+    disassemble(Function, Metadata, BinaryView, Binary, NameBuilder, Result);
+    return Result;
+  }
+
+  void disassemble(const model::Function &Function,
+                   const efa::ControlFlowGraph &Metadata,
+                   const RawBinaryView &BinaryView,
+                   const model::Binary &Binary,
+                   const model::AssemblyNameBuilder &NameBuilder,
+                   yield::Function &ResultFunction);
 
 private:
   LLVMDisassemblerInterface &

include/revng/Yield/HexDump.h

@@ -0,0 +1,44 @@
+#pragma once
+
+//
+// This file is distributed under the MIT License. See LICENSE.md for details.
+//
+
+#include "revng/EarlyFunctionAnalysis/CollectCFG.h"
+#include "revng/PipeboxCommon/BinariesContainer.h"
+#include "revng/PipeboxCommon/LLVMContainer.h"
+#include "revng/PipeboxCommon/RawContainer.h"
+
+namespace revng::pypeline {
+
+using HexDumpContainer = BytesContainer<"HexDumpContainer",
+                                        "text/x.hexdump+ptml">;
+
+namespace piperuns {
+
+class HexDump {
+public:
+  static constexpr llvm::StringRef Name = "HexDump";
+  using Arguments = TypeList<
+    PipeArgument<"Binary", "The binaries to create the hexdump out of">,
+    PipeArgument<"Module", "The LLVM Module(s) with lifted functions">,
+    PipeArgument<"CFG", "The per-function CFG data">,
+    PipeArgument<"Output", "The hexdump of the input binaries", Access::Write>>;
+
+public:
+  static llvm::Error checkPrecondition(const class Model &Model) {
+    return RawBinaryView::checkPrecondition(*Model.get().get());
+  }
+
+  static void run(const class Model &Model,
+                  llvm::StringRef Config,
+                  llvm::StringRef DynamicConfig,
+                  const BinariesContainer &BinaryContainer,
+                  const LLVMFunctionContainer &ModuleContainer,
+                  const CFGMap &CFG,
+                  HexDumpContainer &Output);
+};
+
+} // namespace piperuns
+
+} // namespace revng::pypeline

include/revng/Yield/Pipes/ProcessAssembly.h

@@ -11,12 +11,18 @@
 #include "llvm/Support/raw_ostream.h"
 
 #include "revng/EarlyFunctionAnalysis/CFGStringMap.h"
+#include "revng/EarlyFunctionAnalysis/CollectCFG.h"
+#include "revng/Pipebox/TupleTreeContainer.h"
+#include "revng/PipeboxCommon/BinariesContainer.h"
 #include "revng/Pipeline/Contract.h"
 #include "revng/Pipeline/Target.h"
 #include "revng/Pipes/FileContainer.h"
 #include "revng/Pipes/StringMap.h"
+#include "revng/Yield/Function.h"
 #include "revng/Yield/Pipes/YieldControlFlow.h"
 
+class DissassemblyHelper;
+
 namespace revng::pipes {
 
 class ProcessAssembly {
@@ -44,3 +50,51 @@ public:
 };
 
 } // namespace revng::pipes
+
+namespace revng::pypeline {
+
+using AssemblyInternalContainer = TupleTreeContainer<yield::Function,
+                                                     Kinds::Function,
+                                                     "AssemblyInternalContaine"
+                                                     "r">;
+
+namespace piperuns {
+
+class ProcessAssembly {
+private:
+  const model::Binary &Binary;
+  const CFGMap &CFG;
+  AssemblyInternalContainer &Output;
+
+  std::unique_ptr<DissassemblyHelper> Helper;
+  std::unique_ptr<RawBinaryView> BinaryView;
+  model::AssemblyNameBuilder NameBuilder;
+
+public:
+  static constexpr llvm::StringRef Name = "ProcessAssembly";
+  using Arguments = TypeList<
+    PipeRunArgument<const BinariesContainer, "Binaries", "The input binaries">,
+    PipeRunArgument<const CFGMap, "CFG", "Per-function CFG data">,
+    PipeRunArgument<AssemblyInternalContainer,
+                    "Output",
+                    "Internal data for disassembly",
+                    Access::Write>>;
+
+  ProcessAssembly(const class Model &Model,
+                  llvm::StringRef Config,
+                  llvm::StringRef DynamicConfig,
+                  const BinariesContainer &BinariesContainer,
+                  const CFGMap &CFG,
+                  AssemblyInternalContainer &Output);
+  ~ProcessAssembly();
+
+  static llvm::Error checkPrecondition(const class Model &Model) {
+    return RawBinaryView::checkPrecondition(*Model.get().get());
+  }
+
+  void runOnFunction(const model::Function &TheFunction);
+};
+
+} // namespace piperuns
+
+} // namespace revng::pypeline

include/revng/Yield/Pipes/YieldAssembly.h

@@ -10,8 +10,11 @@
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/Support/raw_ostream.h"
 
+#include "revng/PTML/Tag.h"
+#include "revng/PipeboxCommon/RawContainer.h"
 #include "revng/Pipeline/Contract.h"
 #include "revng/Pipes/StringMap.h"
+#include "revng/Yield/Pipes/ProcessAssembly.h"
 #include "revng/Yield/Pipes/YieldControlFlow.h"
 
 namespace revng::pipes {
@@ -36,3 +39,46 @@ public:
 };
 
 } // namespace revng::pipes
+
+namespace revng::pypeline {
+
+using AssemblyContainer = FunctionToBytesContainer<"AssemblyContainer",
+                                                   "text/x.asm+ptml">;
+
+namespace piperuns {
+
+class YieldAssembly {
+private:
+  const model::Binary &Model;
+  const AssemblyInternalContainer &Input;
+  AssemblyContainer &Output;
+
+  model::CNameBuilder NameBuilder;
+  ptml::MarkupBuilder B;
+
+public:
+  static constexpr llvm::StringRef Name = "YieldAssembly";
+  using Arguments = TypeList<PipeRunArgument<const AssemblyInternalContainer,
+                                             "Input",
+                                             "The internal disassembly data">,
+                             PipeRunArgument<AssemblyContainer,
+                                             "Output",
+                                             "Per-function disassembly",
+                                             Access::Write>>;
+
+  YieldAssembly(const class Model &Model,
+                llvm::StringRef Config,
+                llvm::StringRef DynamicConfig,
+                const AssemblyInternalContainer &Input,
+                AssemblyContainer &Output) :
+    Model(*Model.get().get()),
+    Input(Input),
+    Output(Output),
+    NameBuilder(*Model.get().get()) {}
+
+  void runOnFunction(const model::Function &TheFunction);
+};
+
+} // namespace piperuns
+
+} // namespace revng::pypeline

lib/EarlyFunctionAnalysis/AttachDebugInfo.cpp

@@ -30,6 +30,7 @@
 #include "llvm/IR/Metadata.h"
 
 #include "revng/BasicAnalyses/GeneratedCodeBasicInfo.h"
+#include "revng/EarlyFunctionAnalysis/AttachDebugInfo.h"
 #include "revng/EarlyFunctionAnalysis/ControlFlowGraphCache.h"
 #include "revng/Model/FunctionTags.h"
 #include "revng/Model/LoadModelPass.h"
@@ -47,16 +48,43 @@ using namespace llvm;
 static Logger Log("attach-debug-info");
 
 class AttachDebugInfo : public pipeline::FunctionPassImpl {
+private:
+  using CFG = efa::ControlFlowGraph;
+  using CFGGetterType = std::function<const CFG &(const MetaAddress &)>;
+
 private:
   llvm::Module &M;
   DIBuilder DIB;
   DICompileUnit *CU = nullptr;
 
+  GeneratedCodeBasicInfo &GCBI;
+  CFGGetterType CFGGetter;
+
 public:
   AttachDebugInfo(llvm::ModulePass &Pass,
                   const model::Binary &Binary,
                   llvm::Module &M) :
-    pipeline::FunctionPassImpl(Pass), M(M), DIB(M) {}
+    pipeline::FunctionPassImpl(Pass),
+    M(M),
+    DIB(M),
+    GCBI(getAnalysis<GeneratedCodeBasicInfoWrapperPass>().getGCBI()) {
+    ControlFlowGraphCache &Cache = getAnalysis<ControlFlowGraphCachePass>()
+                                     .get();
+    CFGGetter =
+      [&Cache](const MetaAddress &Address) -> const efa::ControlFlowGraph & {
+      return Cache.getControlFlowGraph(Address);
+    };
+  }
+
+  AttachDebugInfo(const model::Binary &Binary,
+                  llvm::Module &M,
+                  GeneratedCodeBasicInfo &GCBI,
+                  CFGGetterType CFGGetter) :
+    pipeline::FunctionPassImpl(),
+    M(M),
+    DIB(M),
+    GCBI(GCBI),
+    CFGGetter(CFGGetter) {}
 
   static void getAnalysisUsage(llvm::AnalysisUsage &AU) {
     AU.setPreservesAll();
@@ -231,10 +259,6 @@ bool AttachDebugInfo::prologue() {
 
 bool AttachDebugInfo::runOnFunction(const model::Function &ModelFunction,
                                     llvm::Function &F) {
-  ControlFlowGraphCache *Cache = &getAnalysis<ControlFlowGraphCachePass>()
-                                    .get();
-  auto &GCBI = getAnalysis<GeneratedCodeBasicInfoWrapperPass>().getGCBI();
-
   // Skip functions with debug-info.
   if (F.getSubprogram())
     return true;
@@ -242,7 +266,7 @@ bool AttachDebugInfo::runOnFunction(const model::Function &ModelFunction,
   // Skip declarations
   revng_assert(not F.isDeclaration());
 
-  auto FM = Cache->getControlFlowGraph(&F);
+  auto FM = CFGGetter(ModelFunction.Entry());
   revng_log(Log,
             "Metadata for Function " << F.getName() << ":"
                                      << FM.Entry().toString());
@@ -321,3 +345,36 @@ struct AttachDebugInfoToABIEnforcedPipe {
 };
 
 static pipeline::RegisterPipe<AttachDebugInfoToABIEnforcedPipe> Y2;
+
+namespace revng::pypeline::piperuns {
+
+void AttachDebugInfo::runOnFunction(const model::Function &TheFunction) {
+  const MetaAddress &Address = TheFunction.Entry();
+  llvm::Module &Module = ModuleContainer.getModule(ObjectID(Address));
+
+  GeneratedCodeBasicInfo GCBI(Binary);
+  GCBI.run(Module);
+
+  auto CFGGetter =
+    [*this](const MetaAddress &Address) -> const efa::ControlFlowGraph & {
+    return *CFG.getElement(ObjectID(Address));
+  };
+
+  llvm::Function *LLVMFunction = nullptr;
+  for (llvm::Function &Function : Module) {
+    if (FunctionTags::Isolated.isTagOf(&Function)
+        and not Function.isDeclaration()) {
+      revng_assert(LLVMFunction == nullptr);
+      LLVMFunction = &Function;
+    }
+  }
+  revng_assert(LLVMFunction != nullptr);
+
+  // TODO: inline the body of prologue and epilogue here
+  ::AttachDebugInfo Impl(Binary, Module, GCBI, CFGGetter);
+  Impl.prologue();
+  Impl.runOnFunction(TheFunction, *LLVMFunction);
+  Impl.epilogue();
+}
+
+} // namespace revng::pypeline::piperuns

lib/EarlyFunctionAnalysis/CollectCFG.cpp

@@ -7,6 +7,7 @@
 #include "revng/BasicAnalyses/GeneratedCodeBasicInfo.h"
 #include "revng/EarlyFunctionAnalysis/CFGAnalyzer.h"
 #include "revng/EarlyFunctionAnalysis/CFGStringMap.h"
+#include "revng/EarlyFunctionAnalysis/CollectCFG.h"
 #include "revng/EarlyFunctionAnalysis/ControlFlowGraph.h"
 #include "revng/EarlyFunctionAnalysis/FunctionSummaryOracle.h"
 #include "revng/Model/Binary.h"
@@ -89,3 +90,50 @@ static pipeline::RegisterPipe<CollectCFGPipe> X;
 } // namespace revng::pipes
 
 static pipeline::RegisterDefaultConstructibleContainer<revng::pipes::CFGMap> X2;
+
+namespace revng::pypeline::piperuns {
+
+using FSO = efa::FunctionSummaryOracle;
+
+CollectCFG::CollectCFG(const class Model &Model,
+                       llvm::StringRef Config,
+                       llvm::StringRef DynamicConfig,
+                       LLVMRootContainer &Input,
+                       CFGMap &Output) :
+  Model(Model),
+  Output(Output),
+  GCBI(*Model.get().get()),
+  GCBIRun(GCBI, Input.getModule()),
+  Oracle(FSO::importBasicPrototypeData(Input.getModule(),
+                                       GCBI,
+                                       *Model.get().get())),
+  Analyzer(Input.getModule(), GCBI, Model.get(), Oracle) {
+}
+
+void CollectCFG::runOnFunction(const model::Function &Function) {
+  MetaAddress EntryAddress = Function.Entry();
+  const model::Binary &Binary = *Model.get().get();
+
+  // Recover the control-flow graph of the function
+  TupleTree<efa::ControlFlowGraph> New;
+  New->Entry() = EntryAddress;
+  New->Blocks() = std::move(Analyzer.analyze(EntryAddress).CFG);
+
+  if (DebugNames) {
+    auto Function = Binary.Functions().at(EntryAddress);
+    New->Name() = Function.Name();
+  }
+
+  if (New->Blocks().size() > 0)
+    revng_assert(New->Blocks().contains(BasicBlockID(New->Entry())));
+
+  // Run final steps on the CFG
+  New->simplify(Binary);
+
+  if (New->Blocks().size() > 0)
+    revng_assert(New->Blocks().contains(BasicBlockID(New->Entry())));
+
+  Output.getElement(ObjectID(EntryAddress)) = std::move(New);
+}
+
+} // namespace revng::pypeline::piperuns

lib/FunctionIsolation/EnforceABI.cpp

@@ -26,6 +26,7 @@
 #include "revng/BasicAnalyses/GeneratedCodeBasicInfo.h"
 #include "revng/EarlyFunctionAnalysis/CallEdge.h"
 #include "revng/EarlyFunctionAnalysis/ControlFlowGraphCache.h"
+#include "revng/FunctionIsolation/EnforceABI.h"
 #include "revng/FunctionIsolation/StructInitializers.h"
 #include "revng/Model/FunctionTags.h"
 #include "revng/Model/IRHelpers.h"
@@ -48,6 +49,8 @@ using namespace llvm;
 class EnforceABI final : public pipeline::FunctionPassImpl {
 private:
   using UsedRegisters = abi::FunctionType::UsedRegisters;
+  using CFG = efa::ControlFlowGraph;
+  using CFGGetterType = std::function<const CFG &(const MetaAddress &)>;
 
 private:
   const model::Binary &Binary;
@@ -56,8 +59,7 @@ private:
   std::vector<Function *> OldFunctions;
   Function *FunctionDispatcher = nullptr;
   StructInitializers Initializers;
-  ControlFlowGraphCache &Cache;
-  pipeline::LoadExecutionContextPass &LECP;
+  CFGGetterType CFGGetter;
   GeneratedCodeBasicInfo &GCBI;
 
 public:
@@ -68,9 +70,25 @@ public:
     Binary(Binary),
     M(M),
     Initializers(&M),
-    Cache(getAnalysis<ControlFlowGraphCachePass>().get()),
-    LECP(getAnalysis<pipeline::LoadExecutionContextPass>()),
-    GCBI(getAnalysis<GeneratedCodeBasicInfoWrapperPass>().getGCBI()) {}
+    GCBI(getAnalysis<GeneratedCodeBasicInfoWrapperPass>().getGCBI()) {
+    ControlFlowGraphCache &Cache = getAnalysis<ControlFlowGraphCachePass>()
+                                     .get();
+    CFGGetter =
+      [&Cache](const MetaAddress &Address) -> const efa::ControlFlowGraph & {
+      return Cache.getControlFlowGraph(Address);
+    };
+  }
+
+  EnforceABI(const model::Binary &Binary,
+             llvm::Module &M,
+             GeneratedCodeBasicInfo &GCBI,
+             CFGGetterType CFGGetter) :
+    pipeline::FunctionPassImpl(),
+    Binary(Binary),
+    M(M),
+    Initializers(&M),
+    CFGGetter(CFGGetter),
+    GCBI(GCBI) {}
 
   ~EnforceABI() final = default;
 
@@ -336,7 +354,8 @@ void EnforceABI::handleRegularFunctionCall(const MetaAddress &CallerAddress,
 
   // Identify the corresponding call site in the model
   Function *CallerFunction = Call->getParent()->getParent();
-  const efa::ControlFlowGraph &FM = Cache.getControlFlowGraph(CallerFunction);
+  const efa::ControlFlowGraph
+    &FM = CFGGetter(getMetaAddressOfIsolatedFunction(*CallerFunction));
 
   const efa::BasicBlock *CallerBlock = FM.findBlock(GCBI, Call->getParent());
   revng_assert(CallerBlock != nullptr);
@@ -539,3 +558,26 @@ struct EnforceABIPipe {
 };
 
 static pipeline::RegisterPipe<EnforceABIPipe> Y;
+
+namespace revng::pypeline::piperuns {
+
+void EnforceABI::runOnFunction(const model::Function &TheFunction) {
+  llvm::Module &Module = Output.getModule(ObjectID(TheFunction.Entry()));
+  llvm::Function *LLVMFunction = Module.getFunction(NameBuilder
+                                                      .llvmName(TheFunction));
+
+  GeneratedCodeBasicInfo GCBI(Binary);
+  GCBI.run(Module);
+
+  auto CFGGetter =
+    [this](const MetaAddress &Address) -> const efa::ControlFlowGraph & {
+    return *CFG.getElement(ObjectID(Address));
+  };
+
+  ::EnforceABI Impl(Binary, Module, GCBI, CFGGetter);
+  Impl.prologue();
+  Impl.runOnFunction(TheFunction, *LLVMFunction);
+  Impl.epilogue();
+}
+
+} // namespace revng::pypeline::piperuns

lib/FunctionIsolation/InvokeIsolatedFunctions.cpp

@@ -14,6 +14,7 @@
 #include "revng/ABI/FunctionType/Layout.h"
 #include "revng/BasicAnalyses/GeneratedCodeBasicInfo.h"
 #include "revng/EarlyFunctionAnalysis/ControlFlowGraphCache.h"
+#include "revng/FunctionIsolation/InvokeIsolatedFunctions.h"
 #include "revng/Model/IRHelpers.h"
 #include "revng/Model/NameBuilder.h"
 #include "revng/Pipeline/AllRegistries.h"
@@ -28,15 +29,14 @@ using namespace llvm;
 
 class InvokeIsolatedFunctionsImpl {
 private:
-  using FunctionInfo = tuple<const model::Function *,
-                             BasicBlock *,
-                             const Function *>;
+  using FunctionInfo = tuple<const model::Function *, BasicBlock *, Function *>;
   using FunctionMap = std::map<model::Function::Key, FunctionInfo>;
 
 private:
   const model::Binary &Binary;
   Function &RootFunction;
   Module &RootModule;
+  const Module *FunctionModule;
   LLVMContext &Context;
   GeneratedCodeBasicInfo GCBI;
   FunctionMap Map;
@@ -44,10 +44,11 @@ private:
 public:
   InvokeIsolatedFunctionsImpl(const model::Binary &Binary,
                               llvm::Module &RootModule,
-                              const llvm::Module &FunctionModule) :
+                              const llvm::Module *FunctionModule) :
     Binary(Binary),
     RootFunction(*RootModule.getFunction("root")),
     RootModule(RootModule),
+    FunctionModule(FunctionModule),
     Context(RootModule.getContext()),
     GCBI(Binary) {
 
@@ -55,7 +56,7 @@ public:
 
     model::CNameBuilder NameBuilder = Binary;
     for (const model::Function &Function : Binary.Functions()) {
-      auto *F = FunctionModule.getFunction(NameBuilder.llvmName(Function));
+      auto *F = FunctionModule->getFunction(NameBuilder.llvmName(Function));
       revng_assert(F != nullptr);
       Map[Function.key()] = { &Function, nullptr, F };
     }
@@ -172,11 +173,15 @@ public:
         }
       }
 
-      llvm::Function *
+      llvm::Function *NewDeclaration = nullptr;
+      if (&RootModule != FunctionModule) {
         NewDeclaration = llvm::Function::Create(F->getFunctionType(),
                                                 llvm::Function::ExternalLinkage,
                                                 F->getName(),
                                                 RootModule);
+      } else {
+        NewDeclaration = F;
+      }
 
       // Emit the invoke instruction, propagating debug info
       auto *NewInvoke = Builder.CreateInvoke(NewDeclaration,
@@ -194,6 +199,56 @@ public:
   }
 };
 
+static void populateFunctionDispatcher(const model::Binary &Binary,
+                                       llvm::Module &Module) {
+  GeneratedCodeBasicInfo GCBI(Binary);
+  GCBI.run(Module);
+
+  llvm::LLVMContext &Context = Module.getContext();
+  llvm::Function *FunctionDispatcher = getIRHelper("function_dispatcher",
+                                                   Module);
+  BasicBlock *Dispatcher = BasicBlock::Create(Context,
+                                              "function_dispatcher",
+                                              FunctionDispatcher,
+                                              nullptr);
+
+  BasicBlock *Unexpected = BasicBlock::Create(Context,
+                                              "unexpectedpc",
+                                              FunctionDispatcher,
+                                              nullptr);
+  revng::NonDebugInfoCheckingIRBuilder UnreachableBuilder(Unexpected);
+  UnreachableBuilder.CreateUnreachable();
+  setBlockType(Unexpected->getTerminator(), BlockType::UnexpectedPCBlock);
+
+  // TODO: the checks should be enabled conditionally based on the user.
+  revng::NonDebugInfoCheckingIRBuilder Builder(Context);
+
+  // Create all the entries of the dispatcher
+  ProgramCounterHandler::DispatcherTargets Targets;
+  for (llvm::Function &F : Module.functions()) {
+    if (not FunctionTags::Isolated.isTagOf(&F))
+      continue;
+
+    MetaAddress Address = getMetaAddressOfIsolatedFunction(F);
+    BasicBlock *Trampoline = BasicBlock::Create(Context,
+                                                F.getName() + "_trampoline",
+                                                FunctionDispatcher,
+                                                nullptr);
+    Targets.emplace_back(Address, Trampoline);
+
+    Builder.SetInsertPoint(Trampoline);
+    Builder.CreateCall(&F);
+    Builder.CreateRetVoid();
+  }
+
+  // Create switch
+  Builder.SetInsertPoint(Dispatcher);
+  GCBI.programCounterHandler()->buildDispatcher(Targets,
+                                                Builder,
+                                                Unexpected,
+                                                {});
+}
+
 struct InvokeIsolatedPipe {
   static constexpr auto Name = "invoke-isolated-functions";
 
@@ -220,9 +275,11 @@ public:
     // Clone the container
     OutputRootContainer.cloneFrom(InputRootContainer);
 
+    populateFunctionDispatcher(*revng::getModelFromContext(EC),
+                               FunctionContainer.getModule());
     InvokeIsolatedFunctionsImpl Impl(*revng::getModelFromContext(EC),
                                      OutputRootContainer.getModule(),
-                                     FunctionContainer.getModule());
+                                     &FunctionContainer.getModule());
     Impl.run();
 
     const llvm::Module &FunctionModule = FunctionContainer.getModule();
@@ -235,3 +292,35 @@ public:
 };
 
 static pipeline::RegisterPipe<InvokeIsolatedPipe> Y;
+
+namespace revng::pypeline::piperuns {
+
+void InvokeIsolatedFunctions::run(const class Model &Model,
+                                  llvm::StringRef Config,
+                                  llvm::StringRef DynamicConfig,
+                                  const LLVMRootContainer &Root,
+                                  const LLVMFunctionContainer &Functions,
+                                  LLVMRootContainer &Output) {
+  const model::Binary &Binary = *Model.get().get();
+
+  // Clone the container
+  llvm::LLVMContext &OutputContext = Output.getModule().getContext();
+  std::unique_ptr<llvm::Module>
+    OutputModule = cloneIntoContext(Root.getModule(), OutputContext);
+
+  // Merge all the functions into a single Module
+  llvm::Linker Linker(*OutputModule);
+  for (auto &Object : Functions.objects()) {
+    const llvm::Module &Module = Functions.getModule(Object);
+    Linker.linkInModule(cloneIntoContext(Module, OutputContext),
+                        llvm::Linker::OverrideFromSrc);
+  }
+
+  populateFunctionDispatcher(Binary, *OutputModule);
+  InvokeIsolatedFunctionsImpl Impl(Binary, *OutputModule, OutputModule.get());
+  Impl.run();
+
+  Output.assign(std::move(OutputModule));
+}
+
+} // namespace revng::pypeline::piperuns

lib/FunctionIsolation/IsolateFunctions.cpp

@@ -194,6 +194,8 @@ public:
 class IsolateFunctionsImpl {
 private:
   using SuccessorsContainer = std::map<const efa::FunctionEdgeBase *, int>;
+  using CFG = efa::ControlFlowGraph;
+  using CFGGetterType = std::function<const CFG &(const MetaAddress &)>;
 
 private:
   Function *RootFunction = nullptr;
@@ -206,24 +208,21 @@ private:
   std::map<MetaAddress, Function *> IsolatedFunctionsMap;
   std::map<StringRef, Function *> DynamicFunctionsMap;
 
-  ControlFlowGraphCache *Cache = nullptr;
+  CFGGetterType CFGGetter;
   FunctionType *IsolatedFunctionType = nullptr;
-  pipeline::LoadExecutionContextPass &LECP;
 
 public:
   IsolateFunctionsImpl(Function *RootFunction,
                        GeneratedCodeBasicInfo &GCBI,
                        const model::Binary &Binary,
-                       ControlFlowGraphCache &Cache,
-                       pipeline::LoadExecutionContextPass &LECP) :
+                       CFGGetterType CFGGetter) :
     RootFunction(RootFunction),
     TheModule(RootFunction->getParent()),
     Context(TheModule->getContext()),
     GCBI(GCBI),
     Binary(Binary),
     NameBuilder(Binary),
-    Cache(&Cache),
-    LECP(LECP) {
+    CFGGetter(CFGGetter) {
     IsolatedFunctionType = createFunctionType<void>(Context);
   }
 
@@ -250,7 +249,9 @@ public:
   auto &gcbi() const { return GCBI; }
 
 public:
-  void run();
+  void prologue();
+  llvm::Function *runOnFunction(const MetaAddress &Address);
+  void epilogue();
 
   void emitAbort(revng::IRBuilder &Builder,
                  const Twine &Reason,
@@ -280,54 +281,11 @@ public:
   }
 
 private:
-  /// Populate the function_dispatcher, needed to handle the indirect calls
-  void populateFunctionDispatcher();
-
   void handleUnexpectedPCCloned(efa::OutlinedFunction &Outlined);
   void handleAnyPCJumps(efa::OutlinedFunction &Outlined,
                         const efa::ControlFlowGraph &FM);
 };
 
-void IFI::populateFunctionDispatcher() {
-
-  BasicBlock *Dispatcher = BasicBlock::Create(Context,
-                                              "function_dispatcher",
-                                              FunctionDispatcher,
-                                              nullptr);
-
-  BasicBlock *Unexpected = BasicBlock::Create(Context,
-                                              "unexpectedpc",
-                                              FunctionDispatcher,
-                                              nullptr);
-  const DebugLoc &Dbg = GCBI.unexpectedPC()->getTerminator()->getDebugLoc();
-  emitUnreachable(Unexpected, "An unexpected function has been called", Dbg);
-  setBlockType(Unexpected->getTerminator(), BlockType::UnexpectedPCBlock);
-
-  // TODO: the checks should be enabled conditionally based on the user.
-  revng::NonDebugInfoCheckingIRBuilder Builder(Context);
-
-  // Create all the entries of the dispatcher
-  ProgramCounterHandler::DispatcherTargets Targets;
-  for (auto &[Address, F] : IsolatedFunctionsMap) {
-    BasicBlock *Trampoline = BasicBlock::Create(Context,
-                                                F->getName() + "_trampoline",
-                                                FunctionDispatcher,
-                                                nullptr);
-    Targets.emplace_back(Address, Trampoline);
-
-    Builder.SetInsertPoint(Trampoline);
-    Builder.CreateCall(F);
-    Builder.CreateRetVoid();
-  }
-
-  // Create switch
-  Builder.SetInsertPoint(Dispatcher);
-  GCBI.programCounterHandler()->buildDispatcher(Targets,
-                                                Builder,
-                                                Unexpected,
-                                                {});
-}
-
 template<typename T, typename F>
 static bool
 allOrNone(const T &Range, const F &Predicate, bool Default = false) {
@@ -533,9 +491,7 @@ public:
   }
 };
 
-void IsolateFunctionsImpl::run() {
-  Task T(6, "IsolateFunctions");
-
+void IsolateFunctionsImpl::prologue() {
   auto SimpleFunctionType = createFunctionType<void>(Context);
   FunctionDispatcher = createIRHelper("function_dispatcher",
                                       *TheModule,
@@ -548,7 +504,6 @@ void IsolateFunctionsImpl::run() {
   //
   // TODO: we can (and should) push processing of dynamic functions into the
   //       loop emitting individual local functions, and make it lazy
-  T.advance("Create dynamic functions", true);
   Task DynamicFunctionsTask(Binary.ImportedDynamicFunctions().size(),
                             "Dynamic functions creation");
   for (const model::DynamicFunction &Function :
@@ -585,66 +540,46 @@ void IsolateFunctionsImpl::run() {
 
     DynamicFunctionsMap[Name] = NewFunction;
   }
+}
 
-  using namespace efa;
-  using llvm::BasicBlock;
+llvm::Function *IsolateFunctionsImpl::runOnFunction(const MetaAddress &Entry) {
 
-  T.advance("Create dynamic functions", true);
+  revng_assert(Entry.isValid());
+  const efa::ControlFlowGraph &FM = CFGGetter(Entry);
 
-  // Obtain the set of requested targets
-  pipeline::ExecutionContext &Context = *LECP.get();
-  const pipeline::TargetsList &RequestedTargets = LECP.getRequestedTargets();
+  // Get or create the llvm::Function
+  Function *F = getLocalFunction(Entry);
 
-  Task IsolateTask(RequestedTargets.size(), "Isolating functions");
-  for (const pipeline::Target &Target : RequestedTargets) {
-    IsolateTask.advance(Target.toString(), true);
-    Context.getContext().pushReadFields();
+  // Decorate the function as appropriate
+  F->addFnAttr(Attribute::NullPointerIsValid);
+  F->addFnAttr(Attribute::NoMerge);
+  IsolatedFunctionsMap[Entry] = F;
+  revng_assert(F != nullptr);
 
-    auto Entry = MetaAddress::fromString(Target.getPathComponents()[0]);
-    revng_assert(Entry.isValid());
-    const efa::ControlFlowGraph &FM = Cache->getControlFlowGraph(Entry);
+  // Outline the function (later on we'll steal its body and move it into F)
+  CallIsolatedFunction CallHandler(*this, FM);
+  FunctionOutliner Outliner(*TheModule, Binary, GCBI);
+  efa::OutlinedFunction Outlined = Outliner.outline(Entry, &CallHandler);
 
-    // Get or create the llvm::Function
-    Function *F = getLocalFunction(Entry);
+  handleUnexpectedPCCloned(Outlined);
 
-    // Decorate the function as appropriate
-    F->addFnAttr(Attribute::NullPointerIsValid);
-    F->addFnAttr(Attribute::NoMerge);
-    IsolatedFunctionsMap[Entry] = F;
-    revng_assert(F != nullptr);
+  handleAnyPCJumps(Outlined, FM);
 
-    // Outline the function (later on we'll steal its body and move it into F)
-    CallIsolatedFunction CallHandler(*this, FM);
-    FunctionOutliner Outliner(*TheModule, Binary, GCBI);
-    OutlinedFunction Outlined = Outliner.outline(Entry, &CallHandler);
+  if (Outlined.Function)
+    for (BasicBlock &BB : *Outlined.Function)
+      revng_assert(BB.getTerminator() != nullptr);
 
-    handleUnexpectedPCCloned(Outlined);
+  // Steal the function body and let the outlined function be destroyed
+  moveBlocksInto(*Outlined.Function, *F);
 
-    handleAnyPCJumps(Outlined, FM);
-
-    if (Outlined.Function)
-      for (BasicBlock &BB : *Outlined.Function)
-        revng_assert(BB.getTerminator() != nullptr);
-
-    // Steal the function body and let the outlined function be destroyed
-    moveBlocksInto(*Outlined.Function, *F);
-
-    // Commit the produced target
-    Context.commit(Target, LECP.getContainerName());
-
-    Context.getContext().popReadFields();
-  }
+  return F;
+}
 
+void IsolateFunctionsImpl::epilogue() {
+  llvm::Task T(3, "Isolate: epilogue");
   T.advance("Verify module", true);
   revng::verify(TheModule);
 
-  // Create the functions and basic blocks needed for the correct execution of
-  // the exception handling mechanism
-
-  // Populate the function_dispatcher
-  T.advance("Populate function_dispatcher", true);
-  populateFunctionDispatcher();
-
   // Cleanup root
   T.advance("Cleanup", true);
   EliminateUnreachableBlocks(*RootFunction, nullptr, false);
@@ -753,17 +688,100 @@ bool IF::runOnModule(Module &TheModule) {
   const auto &ModelWrapper = getAnalysis<LoadModelWrapperPass>().get();
   const model::Binary &Binary = *ModelWrapper.getReadOnlyModel();
 
+  auto &LECP = getAnalysis<pipeline::LoadExecutionContextPass>();
+  pipeline::ExecutionContext &Context = *LECP.get();
+  const pipeline::TargetsList &RequestedTargets = LECP.getRequestedTargets();
+
+  auto &CFGC = getAnalysis<ControlFlowGraphCachePass>().get();
+  auto CFGGetter =
+    [&CFGC](const MetaAddress &Address) -> const efa::ControlFlowGraph & {
+    return CFGC.getControlFlowGraph(Address);
+  };
+
+  llvm::Task MainTask(3, "Isolate functions");
+  MainTask.advance("Isolate: prologue");
+
   // Create an object of type IsolateFunctionsImpl and run the pass
-  IFI Impl(TheModule.getFunction("root"),
-           GCBI,
-           Binary,
-           getAnalysis<ControlFlowGraphCachePass>().get(),
-           getAnalysis<pipeline::LoadExecutionContextPass>());
-  Impl.run();
+  IFI Impl(TheModule.getFunction("root"), GCBI, Binary, CFGGetter);
+
+  Impl.prologue();
+
+  MainTask.advance("Isolate: run on functions");
+  Task IsolateTask(RequestedTargets.size(), "Isolating functions");
+  for (const pipeline::Target &Target : RequestedTargets) {
+    Context.getContext().pushReadFields();
+
+    auto Entry = MetaAddress::fromString(Target.getPathComponents()[0]);
+    IsolateTask.advance(Entry.toString(), true);
+    Impl.runOnFunction(Entry);
+
+    // Commit the produced target
+    Context.commit(Target, LECP.getContainerName());
+    Context.getContext().popReadFields();
+  }
+
+  MainTask.advance("Isolate: epilogue");
+  Impl.epilogue();
 
   return false;
 }
 
+namespace revng::pypeline::piperuns {
+
+Isolate::Isolate(const class Model &Model,
+                 llvm::StringRef Config,
+                 llvm::StringRef DynamicConfig,
+                 const CFGMap &CFG,
+                 LLVMRootContainer &Root,
+                 LLVMFunctionContainer &Output) :
+  Root(Root), Output(Output), GCBI(*Model.get().get()) {
+  llvm::Module &Module = Root.getModule();
+  GCBI.run(Module);
+
+  auto CFGGetter =
+    [&CFG](const MetaAddress &Address) -> const efa::ControlFlowGraph & {
+    return *CFG.getElement(ObjectID(Address));
+  };
+
+  // TODO: inline Impl
+  Impl = std::make_unique<IFI>(Module.getFunction("root"),
+                               GCBI,
+                               *Model.get().get(),
+                               CFGGetter);
+  Impl->prologue();
+}
+
+void Isolate::runOnFunction(const model::Function &TheFunction) {
+  llvm::Function *Function = Impl->runOnFunction(TheFunction.Entry());
+  IsolatedFunctions.push_back({ TheFunction.Entry(), Function });
+}
+
+Isolate::~Isolate() {
+  Impl->epilogue();
+
+  auto ClonedModule = cloneIntoContext(Root.getModule(), Output.getContext());
+  std::set<const llvm::Function *> ExternalFunctions;
+  for (llvm::Function &ModuleFunction : ClonedModule->functions()) {
+    if (not FunctionTags::Root.isTagOf(&ModuleFunction)
+        and not FunctionTags::Isolated.isTagOf(&ModuleFunction)) {
+      ExternalFunctions.insert(&ModuleFunction);
+    }
+  }
+
+  llvm::Task T(IsolatedFunctions.size(),
+               "Splitting functions into individual modules");
+  for (auto &[Address, Function] : IsolatedFunctions) {
+    T.advance(Address.toString(), true);
+    std::set<const llvm::Function *> ToClone;
+    ToClone.insert(ClonedModule->getFunction(Function->getName()));
+    ToClone.insert(ExternalFunctions.begin(), ExternalFunctions.end());
+
+    Output.assign(ObjectID(Address), ::cloneFiltered(*ClonedModule, ToClone));
+  }
+}
+
+} // namespace revng::pypeline::piperuns
+
 void IsolateFunctions::getAnalysisUsage(llvm::AnalysisUsage &AU) const {
   AU.setPreservesAll();
   AU.addRequired<GeneratedCodeBasicInfoWrapperPass>();

lib/FunctionIsolation/PromoteCSVs.cpp

@@ -66,12 +66,17 @@ private:
   SetVector<GlobalVariable *> CSVs;
   model::Architecture::Values Architecture;
   const model::NamingConfiguration &Configuration;
+  GeneratedCodeBasicInfo &GCBI;
+  llvm::Module &Module;
 
 public:
-  PromoteCSVs(ModulePass &Pass, const model::Binary &Binary, Module &M);
+  PromoteCSVs(ModulePass &Pass, const model::Binary &Binary, llvm::Module &M);
+  PromoteCSVs(const model::Binary &Binary,
+              llvm::Module &M,
+              GeneratedCodeBasicInfo &GCBI);
 
 public:
-  bool prologue() final { return false; }
+  bool prologue();
 
   bool runOnFunction(const model::Function &ModelFunction,
                      llvm::Function &Function) final;
@@ -98,20 +103,35 @@ private:
 
 PromoteCSVs::PromoteCSVs(ModulePass &Pass,
                          const model::Binary &Binary,
-                         Module &M) :
+                         llvm::Module &M) :
   pipeline::FunctionPassImpl(Pass),
   Initializers(&M),
   CSVInitializers(&M, false),
   Architecture(Binary.Architecture()),
-  Configuration(Binary.Configuration().Naming()) {
+  Configuration(Binary.Configuration().Naming()),
+  GCBI(getAnalysis<GeneratedCodeBasicInfoWrapperPass>().getGCBI()),
+  Module(M) {
+}
 
+PromoteCSVs::PromoteCSVs(const model::Binary &Binary,
+                         llvm::Module &M,
+                         GeneratedCodeBasicInfo &GCBI) :
+  pipeline::FunctionPassImpl(),
+  Initializers(&M),
+  CSVInitializers(&M, false),
+  Architecture(Binary.Architecture()),
+  Configuration(Binary.Configuration().Naming()),
+  GCBI(GCBI),
+  Module(M) {
+}
+
+bool PromoteCSVs::prologue() {
   CSVInitializers.setMemoryEffects(MemoryEffects::readOnly());
   CSVInitializers.addFnAttribute(Attribute::NoUnwind);
   CSVInitializers.addFnAttribute(Attribute::WillReturn);
   CSVInitializers.setTags({ &FunctionTags::OpaqueCSVValue });
 
   // Record existing initializers
-  auto &GCBI = getAnalysis<GeneratedCodeBasicInfoWrapperPass>().getGCBI();
   const auto &PCCSVs = GCBI.programCounterHandler()->pcCSVs();
   const auto &R = llvm::concat<GlobalVariable *const>(GCBI.csvs(), PCCSVs);
   SmallVector<GlobalVariable *> CSVsToSort{ R.begin(), R.end() };
@@ -121,11 +141,13 @@ PromoteCSVs::PromoteCSVs(ModulePass &Pass,
       continue;
 
     CSVs.insert(CSV);
-    if (auto *F = M.getFunction(Configuration.OpaqueCSVValuePrefix()
-                                + CSV->getName().str()))
+    if (auto *F = Module.getFunction(Configuration.OpaqueCSVValuePrefix()
+                                     + CSV->getName().str()))
       if (FunctionTags::OpaqueCSVValue.isTagOf(F))
         CSVInitializers.record(CSV->getName(), F);
   }
+
+  return false;
 }
 
 // TODO: assign alias information
@@ -610,3 +632,29 @@ struct PromoteCSVsPipe {
 };
 
 static pipeline::RegisterLLVMPass<PromoteCSVsPipe> Y;
+
+namespace revng::pypeline::piperuns {
+
+void PromoteCSVs::runOnFunction(const model::Function &TheFunction) {
+  llvm::Module &Module = ModuleContainer
+                           .getModule(ObjectID(TheFunction.Entry()));
+  GeneratedCodeBasicInfo GCBI(Binary);
+  GCBI.run(Module);
+
+  ::PromoteCSVs Impl(Binary, Module, GCBI);
+  Impl.prologue();
+
+  llvm::Function *LLVMFunction = nullptr;
+  for (llvm::Function &Function : Module.functions()) {
+    if (FunctionTags::Isolated.isTagOf(&Function)
+        and not Function.isDeclaration()) {
+      revng_assert(LLVMFunction == nullptr);
+      LLVMFunction = &Function;
+    }
+  }
+
+  Impl.runOnFunction(TheFunction, *LLVMFunction);
+  Impl.epilogue();
+}
+
+} // namespace revng::pypeline::piperuns

lib/HeadersGeneration/ModelToHeaderPipe.cpp

@@ -3,6 +3,7 @@
 //
 
 #include "revng/HeadersGeneration/ConfigurationHelpers.h"
+#include "revng/HeadersGeneration/ModelToHeaderPipe.h"
 #include "revng/HeadersGeneration/Options.h"
 #include "revng/HeadersGeneration/PTMLHeaderBuilder.h"
 #include "revng/Pipeline/AllRegistries.h"
@@ -76,3 +77,24 @@ static pipeline::RegisterDefaultConstructibleContainer<ModelHeaderFileContainer>
 } // namespace revng::pipes
 
 static pipeline::RegisterPipe<revng::pipes::ModelToHeader> Y;
+
+namespace revng::pypeline::piperuns {
+
+void ModelToHeader::run(const Model &TheModel,
+                        llvm::StringRef StaticConfig,
+                        llvm::StringRef DynamicConfig,
+                        CBytesContainer &Buffer) {
+  const model::Binary &Binary = *TheModel.get().get();
+  std::unique_ptr<llvm::raw_ostream> Out = Buffer.getOStream(ObjectID());
+  ptml::ModelCBuilder
+    B(*Out,
+      Binary,
+      /* EnableTaglessMode = */ false,
+      { .EnableStackFrameInlining = revng::options::EnableStackFrameInlining,
+        .EnablePrintingOfTheMaximumEnumValue = true,
+        .ExplicitTargetPointerSize = getExplicitPointerSize(Binary) });
+  ptml::HeaderBuilder(B).printModelHeader();
+  Out->flush();
+}
+
+} // namespace revng::pypeline::piperuns

lib/HeadersGeneration/ModelTypeDefinitionPipe.cpp

@@ -3,6 +3,7 @@
 //
 
 #include "revng/Backend/DecompileFunction.h"
+#include "revng/HeadersGeneration/ModelTypeDefinitionPipe.h"
 #include "revng/HeadersGeneration/PTMLHeaderBuilder.h"
 #include "revng/Model/Binary.h"
 #include "revng/Pipeline/AllRegistries.h"
@@ -60,4 +61,22 @@ static RegisterDefaultConstructibleContainer<TypeDefinitionStringMap> F2;
 
 } // end namespace revng::pipes
 
+namespace revng::pypeline::piperuns {
+
+using TD = UpcastablePointer<model::TypeDefinition>;
+void GenerateModelTypeDefinition::runOnTypeDefinition(const TD
+                                                        &TypeDefinition) {
+  auto OS = Output.getOStream(ObjectID(TypeDefinition->key()));
+  ptml::ModelCBuilder B(*OS,
+                        *Model.get().get(),
+                        true,
+                        { .EnablePrintingOfTheMaximumEnumValue = true,
+                          .EnableExplicitPadding = false });
+
+  upcast(TypeDefinition,
+         [&B]<typename T>(const T &Upcasted) { B.printDefinition(Upcasted); });
+}
+
+} // namespace revng::pypeline::piperuns
+
 static pipeline::RegisterPipe<revng::pipes::GenerateModelTypeDefinition> X2;

lib/Lift/Lift.cpp

@@ -11,6 +11,7 @@
 #include "revng/Support/ResourceFinder.h"
 
 #include "CodeGenerator.h"
+#include "PostLiftVerifyPass.h"
 
 using namespace llvm::cl;
 
@@ -100,3 +101,61 @@ bool LiftPass::runOnModule(llvm::Module &M) {
 
   return false;
 }
+
+namespace revng::pypeline::piperuns {
+
+void Lift::run(const class Model &TheModel,
+               llvm::StringRef Config,
+               llvm::StringRef DynamicConfig,
+               const BinariesContainer &Binary,
+               LLVMRootContainer &ModuleContainer) {
+  llvm::Task T(6, "Lift");
+  const TupleTree<model::Binary> &Model = TheModel.get();
+
+  T.advance("findFiles", false);
+  const auto Paths = findExternalFilePaths(Model->Architecture());
+
+  // Look for the library in the system's paths
+  T.advance("Load libtcg", false);
+  auto TheLibTcg = LibTcg::get(Model->Architecture());
+
+  // Get access to raw binary data
+  revng_assert(Binary.size() == 1);
+  llvm::ArrayRef<char> File = Binary.getFile(0);
+  RawBinaryView RawBinary(*Model, { File.data(), File.size() });
+  llvm::Module &Module = ModuleContainer.getModule();
+
+  T.advance("Construct CodeGenerator", false);
+  CodeGenerator Generator(RawBinary,
+                          &Module,
+                          Model,
+                          Paths.LibHelpers,
+                          Paths.EarlyLinked,
+                          model::Architecture::x86_64);
+
+  std::optional<uint64_t> EntryPointAddressOptional;
+  if (EntryPointAddress.getNumOccurrences() != 0)
+    EntryPointAddressOptional = EntryPointAddress;
+  T.advance("Translate", true);
+
+  Generator.translate(TheLibTcg, EntryPointAddressOptional);
+
+  T.advance("Sort Module", true);
+  sortModule(Module);
+
+  T.advance("Verify Module", true);
+  // TODO: convert this from a pass to a free-standing function
+  PostLiftVerifyPass{}.runOnModule(Module);
+
+  // TODO: substitute with strip-dead-debug-info once the old pipeline
+  //       is dropped
+  pruneDICompileUnits(Module);
+}
+
+llvm::Error Lift::checkPrecondition(const class Model &Model) {
+  const model::Binary &Binary = *Model.get().get();
+  return revng::joinErrors(::detail::liftCheckPrecondition(Binary),
+                           RawBinaryView::checkPrecondition(Binary));
+}
+
+} // namespace revng::pypeline::piperuns

lib/Lift/LiftPipe.cpp

@@ -132,9 +132,9 @@ Lift::invalidate(const BinaryFileContainer &SourceBinary,
   return {};
 }
 
-llvm::Error Lift::checkPrecondition(const pipeline::Context &Context) const {
-  const auto &Model = *getModelFromContext(Context);
+namespace detail {
 
+llvm::Error liftCheckPrecondition(const model::Binary &Model) {
   if (Model.Architecture() == model::Architecture::Invalid) {
     return revng::createError("Cannot lift binary with architecture invalid.");
   }
@@ -148,6 +148,13 @@ llvm::Error Lift::checkPrecondition(const pipeline::Context &Context) const {
   return llvm::Error::success();
 }
 
+} // namespace detail
+
+llvm::Error Lift::checkPrecondition(const pipeline::Context &Context) const {
+  const auto &Model = *getModelFromContext(Context);
+  return ::detail::liftCheckPrecondition(Model);
+}
+
 static_assert(pipeline::HasInvalidate<Lift>);
 
 static RegisterPipe<Lift> E1;

lib/Lift/LinkSupportPipe.cpp

@@ -59,11 +59,10 @@ static llvm::StringRef getSupportName(model::Architecture::Values V) {
   return "Invalid";
 }
 
-static std::string getSupportPath(const Context &Context) {
-  const auto &Model = getModelFromContext(Context);
+static std::string getSupportPath(const model::Binary &Model) {
   const char *SupportConfig = Tracing ? "trace" : "normal";
 
-  auto ArchName = getSupportName(Model->Architecture()).str();
+  auto ArchName = getSupportName(Model.Architecture()).str();
   std::string SupportSearchPath = ("/share/revng/support-" + ArchName + "-"
                                    + SupportConfig + ".ll");
 
@@ -80,7 +79,7 @@ void revng::pipes::LinkSupport::run(ExecutionContext &EC,
   if (ModuleContainer.enumerate().empty())
     return;
 
-  std::string SupportPath = getSupportPath(EC.getContext());
+  std::string SupportPath = getSupportPath(*getModelFromContext(EC));
 
   llvm::SMDiagnostic Err;
   auto Module = llvm::parseIRFile(SupportPath,
@@ -97,3 +96,23 @@ void revng::pipes::LinkSupport::run(ExecutionContext &EC,
 }
 
 static pipeline::RegisterPipe<LinkSupport> E4;
+
+namespace revng::pypeline::piperuns {
+
+void LinkSupport::run(const class Model &Model,
+                      llvm::StringRef Config,
+                      llvm::StringRef DynamicConfig,
+                      LLVMRootContainer &ModuleContainer) {
+  std::string SupportPath = getSupportPath(*Model.get().get());
+
+  llvm::SMDiagnostic Err;
+  llvm::LLVMContext &Context = ModuleContainer.getModule().getContext();
+  auto Module = llvm::parseIRFile(SupportPath, Err, Context);
+  revng_assert(Module != nullptr);
+
+  bool Failed = llvm::Linker::linkModules(ModuleContainer.getModule(),
+                                          std::move(Module));
+  revng_assert(not Failed);
+}
+
+} // namespace revng::pypeline::piperuns

lib/Model/RawBinaryView.cpp

@@ -190,3 +190,19 @@ RawBinaryView::findOffsetInSegment(MetaAddress Address, uint64_t Size) const {
 
   return { nullptr, 0 };
 }
+
+llvm::Error RawBinaryView::checkPrecondition(const model::Binary &Binary) {
+  llvm::SmallVector<llvm::Error, 0> Errors;
+  if (Binary.Binaries().size() != 1)
+    Errors.push_back(revng::createError("Binaries should contain exactly 1 "
+                                        "entry"));
+
+  for (const model::Segment &Segment : Binary.Segments()) {
+    if (not Segment.Binary().isValid())
+      Errors.push_back(revng::createError("Segment " + toString(Segment.key())
+                                          + " does not reference a valid "
+                                            "entry in Binaries"));
+  }
+
+  return revng::joinErrors(Errors);
+}

lib/Pipebox/CMakeLists.txt

@@ -2,10 +2,20 @@
 # This file is distributed under the MIT License. See LICENSE.md for details.
 #
 
-revng_add_library_internal(revngPipebox SHARED Pipebox.cpp)
+revng_add_library_internal(revngPipebox SHARED Pipebox.cpp LLVMPipe.cpp)
 
 llvm_map_components_to_libnames(LLVM_LIBRARIES Support Core)
-target_link_libraries(revngPipebox ${Python_LIBRARIES} ${LLVM_LIBRARIES}
-                      revngModelToHeader nanobind revngSupport)
+target_link_libraries(
+  revngPipebox
+  ${Python_LIBRARIES}
+  ${LLVM_LIBRARIES}
+  revngModelToHeader
+  revngLift
+  revngFunctionIsolation
+  revngRecompile
+  revngYield
+  revngYieldPipes
+  nanobind
+  revngSupport)
 
 target_include_directories(revngPipebox PUBLIC ${Python_INCLUDE_DIRS})

lib/Pipebox/LLVMPipe.cpp

@@ -0,0 +1,43 @@
+//
+// This file is distributed under the MIT License. See LICENSE.md for details.
+//
+
+#include "llvm/PassRegistry.h"
+
+#include "revng/Pipebox/LLVMPipe.h"
+
+namespace rpp = revng::pypeline::pipes;
+using Configuration = rpp::detail::PureLLVMPassesPipeBase::Configuration;
+
+template<>
+struct llvm::yaml::MappingTraits<Configuration> {
+  static void mapping(IO &IO, Configuration &Fields) {
+    IO.mapRequired("Passes", Fields.Passes);
+  }
+};
+
+Configuration Configuration::parse(llvm::StringRef Input) {
+  return llvm::cantFail(fromString<Configuration>(Input));
+}
+
+namespace revng::pypeline::pipes::detail {
+
+PureLLVMPassesPipeBase::PureLLVMPassesPipeBase(llvm::StringRef
+                                                 StaticConfiguration) :
+  StaticConfiguration(StaticConfiguration) {
+
+  Configuration Configuration = Configuration::parse(StaticConfiguration);
+  llvm::PassRegistry &Registry = *llvm::PassRegistry::getPassRegistry();
+  for (llvm::StringRef PassName : Configuration.Passes) {
+    const llvm::PassInfo *PassInfo = Registry.getPassInfo(PassName);
+    revng_assert(PassInfo != nullptr,
+                 ("llvm-pipe: Requested pass " + PassName
+                  + " was not found in the PassRegistry")
+                   .str()
+                   .c_str());
+    PassInfos.push_back(PassInfo);
+  }
+  TaskName = "PureLLVMPasses(" + llvm::join(Configuration.Passes, ",") + ")";
+}
+
+} // namespace revng::pypeline::pipes::detail

lib/Pipebox/Pipebox.cpp

@@ -2,9 +2,25 @@
 // This file is distributed under the MIT License. See LICENSE.md for details.
 //
 
+#include "revng/EarlyFunctionAnalysis/AttachDebugInfo.h"
+#include "revng/EarlyFunctionAnalysis/CollectCFG.h"
+#include "revng/FunctionIsolation/EnforceABI.h"
+#include "revng/FunctionIsolation/InvokeIsolatedFunctions.h"
+#include "revng/FunctionIsolation/IsolateFunctions.h"
+#include "revng/FunctionIsolation/PromoteCSVs.h"
+#include "revng/HeadersGeneration/ModelToHeaderPipe.h"
+#include "revng/HeadersGeneration/ModelTypeDefinitionPipe.h"
+#include "revng/Lift/Lift.h"
+#include "revng/Lift/LinkSupportPipe.h"
+#include "revng/Pipebox/LLVMPipe.h"
 #include "revng/PipeboxCommon/BinariesContainer.h"
 #include "revng/PipeboxCommon/Helpers/Registrars.h"
 #include "revng/PipeboxCommon/RawContainer.h"
+#include "revng/Recompile/CompileModulePipe.h"
+#include "revng/Recompile/LinkForTranslationPipe.h"
+#include "revng/Yield/HexDump.h"
+#include "revng/Yield/Pipes/ProcessAssembly.h"
+#include "revng/Yield/Pipes/YieldAssembly.h"
 
 using namespace revng::pypeline;
 
@@ -12,8 +28,40 @@ using namespace revng::pypeline;
 // Containers
 //
 
-static RegisterContainer<BytesContainer> C1;
-static RegisterContainer<FunctionToBytesContainer> C2;
-static RegisterContainer<TypeDefinitionToBytesContainer> C3;
-static RegisterContainer<LLVMRootContainer> C4;
+static RegisterContainer<LLVMRootContainer> C1;
+static RegisterContainer<LLVMFunctionContainer> C3;
+static RegisterContainer<CBytesContainer> C4;
 static RegisterContainer<BinariesContainer> C5;
+static RegisterContainer<PTMLCTypeContainer> C6;
+static RegisterContainer<CFGMap> C7;
+static RegisterContainer<HexDumpContainer> C8;
+static RegisterContainer<AssemblyInternalContainer> C9;
+static RegisterContainer<AssemblyContainer> C10;
+static RegisterContainer<ObjectFileContainer> C11;
+static RegisterContainer<TranslatedContainer> C12;
+
+//
+// Pipes
+//
+
+using namespace revng::pypeline::pipes;
+using namespace revng::pypeline::piperuns;
+namespace piperuns = revng::pypeline::piperuns;
+
+static RegisterSingleOutputPipe<Lift> P1;
+static RegisterPipe<PureLLVMPassesRootPipe> P2;
+static RegisterPipe<PureLLVMPassesPipe> P3;
+static RegisterSingleOutputPipe<ModelToHeader> P4;
+static RegisterTypeDefinitionPipe<GenerateModelTypeDefinition> P5;
+static RegisterFunctionPipe<CollectCFG> P6;
+static RegisterFunctionPipe<Isolate> P7;
+static RegisterFunctionPipe<AttachDebugInfo> P8;
+static RegisterFunctionPipe<piperuns::EnforceABI> P9;
+static RegisterFunctionPipe<PromoteCSVs> P10;
+static RegisterSingleOutputPipe<HexDump> P11;
+static RegisterFunctionPipe<ProcessAssembly> P12;
+static RegisterFunctionPipe<YieldAssembly> P13;
+static RegisterSingleOutputPipe<LinkSupport> P14;
+static RegisterSingleOutputPipe<CompileRootModule> P15;
+static RegisterSingleOutputPipe<LinkForTranslation> P16;
+static RegisterSingleOutputPipe<InvokeIsolatedFunctions> P17;

lib/PipeboxCommon/Python/_pipebox.cpp

@@ -2,34 +2,71 @@
 // This file is distributed under the MIT License. See LICENSE.md for details.
 //
 
+#include <csignal>
+
 #include "nanobind/nanobind.h"
 #include "nanobind/stl/optional.h"
 #include "nanobind/stl/set.h"
 #include "nanobind/stl/string.h"
 #include "nanobind/stl/vector.h"
 
+#include "llvm/Support/Signals.h"
+
+#include "revng/ADT/SetOperations.h"
 #include "revng/PipeboxCommon/Helpers/Python/Casters.h"
 #include "revng/PipeboxCommon/Helpers/Python/Helpers.h"
 #include "revng/PipeboxCommon/Helpers/Python/Registry.h"
 #include "revng/PipeboxCommon/Model.h"
 #include "revng/Support/InitRevng.h"
 
-NB_MODULE(_pipebox, m) {
-  {
-    // Do InitRevng, use a capsule to call the destructor when the Python module
-    // is unloaded
-    int Argc = 1;
-    const char *Argv[] = { "" };
-    const char **ArgvPtr = Argv;
-    nanobind::capsule Capsule(new revng::InitRevng(Argc, ArgvPtr, "", {}),
-                              [](void *Ptr) noexcept {
-                                delete static_cast<revng::InitRevng *>(Ptr);
-                              });
-    nanobind::setattr(m, "__init_revng__", Capsule);
-  }
+struct SignalHandler {
+  bool IsTerminating;
+  sighandler_t Handler;
+};
 
+static std::map<int, SignalHandler> SavedSignals;
+
+static void handleSignal(int SigNo) {
+  const SignalHandler &Handler = SavedSignals[SigNo];
+  if (Handler.IsTerminating)
+    llvm::sys::RunInterruptHandlers();
+  Handler.Handler(SigNo);
+}
+
+NB_MODULE(_pipebox, m) {
   using namespace revng::pypeline::helpers::python;
 
+  auto Initialize = [m](std::set<int> TerminatingSignals,
+                        std::set<int> NonterminatingSignals,
+                        std::vector<std::string> ArgVector) {
+    revng_assert(not nanobind::hasattr(m, "__init_revng__"));
+    revng_assert(not intersects(TerminatingSignals, NonterminatingSignals));
+
+    // Save the signal pointers for later
+    for (int SigNumber :
+         llvm::concat<const int>(TerminatingSignals, NonterminatingSignals)) {
+      sighandler_t Handler = signal(SigNumber, SIG_DFL);
+      if (Handler != SIG_ERR && Handler != NULL) {
+        bool IsTerminating = TerminatingSignals.contains(SigNumber);
+        SavedSignals[SigNumber] = { IsTerminating, Handler };
+      }
+    }
+
+    int Argc = ArgVector.size() + 1;
+    const char *Argv[Argc];
+    Argv[0] = "";
+    for (size_t I = 0; I < ArgVector.size(); I++)
+      Argv[I + 1] = ArgVector[I].c_str();
+
+    const char **ArgvPtr = Argv;
+    // use a capsule to call the destructor when the Python module is unloaded
+    m.attr("__init_revng__") = makeCapsule<revng::InitRevng>(Argc, ArgvPtr, "");
+
+    for (int SigNumber : std::ranges::views::keys(SavedSignals))
+      signal(SigNumber, &handleSignal);
+  };
+  m.def("initialize", Initialize);
+
   // Register the Buffer class
   nanobind::class_<revng::pypeline::Buffer>(m, "Buffer")
     .def(nanobind::init<>())
@@ -94,6 +131,9 @@ NB_MODULE(_pipebox, m) {
   nanobind::object ModelBaseClass = importObject("revng.pypeline.model.Model");
   nanobind::class_<Model>(m, "Model", ModelBaseClass)
     .def(nanobind::init<>())
+    .def_static("model_name", []() { return nanobind::str("model.yml"); })
+    .def_static("mime_type",
+                []() { return nanobind::str("application/x-yaml"); })
     .def("diff",
          [](Model &Handle, nanobind::handle_t<Model> Other) {
            return Handle.diff(*nanobind::cast<Model *>(Other));

lib/Pipeline/LLVMContainer.cpp

@@ -61,116 +61,17 @@ void pipeline::makeGlobalObjectsArray(llvm::Module &Module,
                            GlobalArrayName);
 }
 
-struct FunctionsMetadata {
-  static inline const char *MetadataName = "revng.functions-metatadata-backup";
-
-  static void dropBackup(llvm::Module &M) {
-    M.eraseNamedMetadata(M.getNamedMetadata(MetadataName));
-  }
-
-  static void backup(llvm::Module &M) {
-    using namespace llvm;
-
-    LLVMContext &Context = M.getContext();
-
-    // Create the named metadata
-    NamedMDNode *BackupNMD = M.getOrInsertNamedMetadata(MetadataName);
-    BackupNMD->clearOperands();
-
-    // Backup saving names, the metadata kind IDs might change
-    SmallVector<StringRef> MDKindNames;
-    M.getMDKindNames(MDKindNames);
-
-    for (Function &F : M) {
-      // Ignore unnamed functions
-      if (F.getName().empty())
-        continue;
-
-      // Collect metadata for the function
-      SmallVector<std::pair<unsigned, MDNode *>, 8> MDs;
-      F.getAllMetadata(MDs);
-
-      // Skip if no metadata
-      if (MDs.empty())
-        continue;
-
-      // Create an MDNode for the function's metadata
-      // Format: [FunctionName, MDKind1, MDNode1, MDKind2, MDNode2, ...]
-      SmallVector<Metadata *, 8> BackupEntry;
-      BackupEntry.push_back(MDString::get(Context, F.getName()));
-
-      for (const auto &MD : MDs) {
-        BackupEntry.push_back(MDString::get(Context, MDKindNames[MD.first]));
-        BackupEntry.push_back(MD.second);
-      }
-
-      // Append entry
-      BackupNMD->addOperand(MDNode::get(Context, BackupEntry));
-    }
-  }
-
-  static void restore(llvm::Module &M) {
-    using namespace llvm;
-
-    NamedMDNode *BackupNMD = M.getNamedMetadata(MetadataName);
-    if (!BackupNMD)
-      return;
-
-    // dumpModule(&M, "/tmp/asd3.ll");
-
-    // Iterate over all operands in the backup NamedMDNode
-    for (MDNode *N : BackupNMD->operands()) {
-      if (N->getNumOperands() == 0)
-        continue;
-
-      // Get the function name
-      StringRef FuncName = cast<MDString>(N->getOperand(0))->getString();
-
-      // Find the function by name
-      Function *F = M.getFunction(FuncName);
-      if (F == nullptr)
-        continue;
-
-      // Clear existing metadata
-      F->clearMetadata();
-
-      auto OperandCount = N->getNumOperands();
-      revng_assert(OperandCount >= 3 and OperandCount % 2 == 1);
-
-      // Restore metadata (operands come in pairs: kind name, MDNode)
-      for (unsigned I = 1; I < OperandCount; I += 2) {
-        auto *KindMD = cast<MDString>(N->getOperand(I).get());
-        MDNode *MD = cast<MDNode>(N->getOperand(I + 1));
-        F->setMetadata(KindMD->getString(), MD);
-      }
-    }
-  }
-};
-
 std::unique_ptr<ContainerBase>
 LLVMContainer::cloneFiltered(const TargetsList &Targets) const {
   using InspectorT = LLVMKind;
   auto ToClone = InspectorT::functions(Targets, *this->self());
   auto ToCloneUntracked = InspectorT::untrackedFunctions(*this->self());
 
-  const auto Filter = [&ToClone, &ToCloneUntracked](const auto &GlobalSym) {
-    if (not llvm::isa<llvm::Function>(GlobalSym))
-      return true;
+  std::set<const llvm::Function *> ToCloneAll;
+  ToCloneAll.insert(ToClone.begin(), ToClone.end());
+  ToCloneAll.insert(ToCloneUntracked.begin(), ToCloneUntracked.end());
 
-    const auto &F = llvm::cast<llvm::Function>(GlobalSym);
-    return ToClone.contains(F) or ToCloneUntracked.contains(F);
-  };
-
-  llvm::ValueToValueMapTy Map;
-
-  FunctionsMetadata::backup(*Module.get());
-
-  revng::verify(Module.get());
-  auto Cloned = llvm::CloneModule(*Module, Map, Filter);
-
-  FunctionsMetadata::restore(*Cloned.get());
-  FunctionsMetadata::dropBackup(*Module.get());
-  FunctionsMetadata::dropBackup(*Cloned.get());
+  auto Cloned = ::cloneFiltered(*this->Module, ToCloneAll);
 
   return std::make_unique<ThisType>(this->name(),
                                     this->TheContext,

lib/Pipes/CMakeLists.txt

@@ -14,32 +14,6 @@ revng_add_library_internal(
   GlobalsAnalyses.cpp
   DebugInfoHelpers.cpp)
 
-llvm_map_components_to_libnames(
-  LLVM_LIBRARIES
-  AggressiveInstCombine
-  Analysis
-  AsmParser
-  BitWriter
-  CodeGen
-  Core
-  Coroutines
-  Extensions
-  GlobalISel
-  IPO
-  InstCombine
-  Instrumentation
-  MC
-  Passes
-  Remarks
-  ScalarOpts
-  Support
-  Target
-  TransformUtils
-  Vectorize
-  X86AsmParser
-  X86CodeGen
-  X86Desc
-  X86Info)
-
+llvm_map_components_to_libnames(LLVM_LIBRARIES Core MC Support)
 target_link_libraries(revngPipes revngPipeline revngModel revngSupport
                       revngStorage ${LLVM_LIBRARIES})

lib/Pipes/Pipes.cpp

@@ -74,25 +74,7 @@ public:
 
   void registerKinds(KindsRegistry &KindDictionary) override {}
 
-  void libraryInitialization() override {
-
-    llvm::codegen::RegisterCodeGenFlags Reg;
-    llvm::InitializeNativeTarget();
-    llvm::InitializeNativeTargetAsmPrinter();
-    llvm::InitializeNativeTargetAsmParser();
-
-    auto &Registry = *llvm::PassRegistry::getPassRegistry();
-    llvm::initializeCore(Registry);
-    llvm::initializeTransformUtils(Registry);
-    llvm::initializeScalarOpts(Registry);
-    llvm::initializeVectorization(Registry);
-    llvm::initializeInstCombine(Registry);
-    llvm::initializeIPO(Registry);
-    llvm::initializeAnalysis(Registry);
-    llvm::initializeCodeGen(Registry);
-    llvm::initializeGlobalISel(Registry);
-    llvm::initializeTarget(Registry);
-  }
+  void libraryInitialization() override {}
 
   ~LLVMPipelineRegistry() override = default;
 };

lib/Recompile/CompileModulePipe.cpp

@@ -82,30 +82,19 @@ public:
   }
 };
 
-static void compileModuleRunImpl(const Context &Context,
-                                 LLVMContainer &Module,
-                                 ObjectFileContainer &TargetBinary) {
+// This function is used by both the old and the new pipeline
+static void compileModuleRunImpl(const model::Binary &Binary,
+                                 llvm::Module *M,
+                                 llvm::raw_pwrite_stream &Output) {
   using namespace revng;
-
-  auto Enumeration = Module.enumerate();
-  if (not Enumeration.contains(pipeline::Target(kinds::Root))
-      and not Enumeration.contains(pipeline::Target(kinds::IsolatedRoot)))
-    return;
-
-  if (Enumeration.contains(pipeline::Target(kinds::IsolatedRoot))
-      and not Enumeration.contains(kinds::Isolated.allTargets(Context)))
-    return;
-
   StringMap<llvm::cl::Option *> &RegOptions(getRegisteredOptions());
   getOption<bool>(RegOptions, "disable-machine-licm")->setInitialValue(true);
 
-  llvm::Module *M = &Module.getModule();
-
   M->getContext()
     .setDiagnosticHandler(std::make_unique<CustomDiagnosticHandler>());
 
   {
-    auto Architecture = getModelFromContext(Context)->Architecture();
+    auto Architecture = Binary.Architecture();
     auto ArchName = model::Architecture::getQEMUName(Architecture).str();
 
     // Note: here we use the full version of the helpers, i.e., where we all the
@@ -127,7 +116,7 @@ static void compileModuleRunImpl(const Context &Context,
     F.setSection("");
 
   OriginalAssemblyAnnotationWriter OAAW(M->getContext());
-  createSelfReferencingDebugInfo(M, Module.name(), &OAAW);
+  createSelfReferencingDebugInfo(M, "llvm-container", &OAAW);
 
   // Get the target specific parser.
   std::string Error;
@@ -186,12 +175,8 @@ static void compileModuleRunImpl(const Context &Context,
   TargetLibraryInfoImpl TLII(Triple(M->getTargetTriple()));
   PM.add(new TargetLibraryInfoWrapperPass(TLII));
 
-  std::error_code EC;
-  raw_fd_ostream OutputStream(TargetBinary.getOrCreatePath(), EC);
-  revng_assert(!EC);
-
   bool Err = Target->addPassesToEmitFile(PM,
-                                         OutputStream,
+                                         Output,
                                          nullptr,
                                          CGFT_ObjectFile,
                                          true);
@@ -200,6 +185,32 @@ static void compileModuleRunImpl(const Context &Context,
   revng::verify(M);
   PM.run(*M);
   revng::verify(M);
+}
+
+// This function is a wrapper of the previous `compileModuleRunImpl` and it's
+// used only by the old pipeline in both `CompileModule::run` and
+// `CompileIsolatedModule::run`
+static void compileModuleRunImpl(const Context &Context,
+                                 LLVMContainer &Module,
+                                 ObjectFileContainer &TargetBinary) {
+  using namespace revng;
+
+  auto Enumeration = Module.enumerate();
+  if (not Enumeration.contains(pipeline::Target(kinds::Root))
+      and not Enumeration.contains(pipeline::Target(kinds::IsolatedRoot)))
+    return;
+
+  if (Enumeration.contains(pipeline::Target(kinds::IsolatedRoot))
+      and not Enumeration.contains(kinds::Isolated.allTargets(Context)))
+    return;
+
+  std::error_code EC;
+  raw_fd_ostream OutputStream(TargetBinary.getOrCreatePath(), EC);
+  revng_assert(!EC);
+
+  compileModuleRunImpl(*getModelFromContext(Context),
+                       &Module.getModule(),
+                       OutputStream);
 
   auto Path = TargetBinary.path();
 
@@ -224,3 +235,17 @@ void CompileIsolatedModule::run(ExecutionContext &EC,
 
 static RegisterPipe<CompileModule> E2;
 static RegisterPipe<CompileIsolatedModule> E3;
+
+namespace revng::pypeline::piperuns {
+
+void CompileRootModule::run(const Model &TheModel,
+                            llvm::StringRef StaticConfig,
+                            llvm::StringRef DynamicConfig,
+                            LLVMRootContainer &Input,
+                            ObjectFileContainer &Output) {
+  compileModuleRunImpl(*TheModel.get().get(),
+                       &Input.getModule(),
+                       *Output.getOStream(ObjectID{}));
+}
+
+} // namespace revng::pypeline::piperuns

lib/Recompile/LinkForTranslationPipe.cpp

@@ -34,3 +34,50 @@ void LinkForTranslation::run(ExecutionContext &EC,
 }
 
 static RegisterPipe<LinkForTranslation> E5;
+
+namespace revng::pypeline::piperuns {
+
+llvm::Error LinkForTranslation::checkPrecondition(const class Model &Model) {
+  if (Model.get().get()->Binaries().size() != 1)
+    return revng::createError("Binaries must have exactly one element");
+  return llvm::Error::success();
+}
+
+static void writeToFile(llvm::StringRef Path, llvm::StringRef Buffer) {
+  std::error_code EC;
+  llvm::raw_fd_ostream OS(Path, EC);
+  revng_assert(not EC);
+  OS << Buffer;
+}
+
+void LinkForTranslation::run(const Model &TheModel,
+                             llvm::StringRef StaticConfig,
+                             llvm::StringRef DynamicConfig,
+                             const BinariesContainer &Binaries,
+                             const ObjectFileContainer &ObjectFile,
+                             TranslatedContainer &Output) {
+  // TODO: some of the operations in linkForTranslation should be converted to
+  //       in-memory counterparts to avoid serializing everything.
+  TemporaryFile Binary("revng-lft-binary");
+  auto BinaryArrayRef = Binaries.getFile(0);
+  writeToFile(Binary.path(), { BinaryArrayRef.begin(), BinaryArrayRef.size() });
+
+  TemporaryFile Object("revng-lft-object", "o");
+  writeToFile(Object.path(),
+              ObjectFile.getMemoryBuffer(ObjectID{})->getBuffer());
+
+  TemporaryFile TempOutput("revng-lft-output");
+
+  linkForTranslation(*TheModel.get().get(),
+                     Binary.path(),
+                     Object.path(),
+                     TempOutput.path());
+
+  auto Buffer = revng::cantFail(llvm::MemoryBuffer::getFile(TempOutput.path()));
+  {
+    auto OutputOS = Output.getOStream(ObjectID{});
+    *OutputOS << Buffer->getBuffer();
+  }
+}
+
+} // namespace revng::pypeline::piperuns

lib/Support/CMakeLists.txt

@@ -14,6 +14,7 @@ revng_add_library_internal(
   CustomizedLLVMPasses.cpp
   Debug.cpp
   ExplicitSpecializations.cpp
+  InitRevng.cpp
   IRHelpers.cpp
   LDDTree.cpp
   MetaAddress.cpp
@@ -35,6 +36,29 @@ if(NOT LibArchive_FOUND)
   message(FATAL_ERROR "libarchive not found")
 endif()
 
+llvm_map_components_to_libnames(
+  LLVM_LIBRARIES
+  Analysis
+  BitReader
+  BitWriter
+  CodeGen
+  Core
+  GlobalISel
+  InstCombine
+  IPO
+  IRReader
+  Linker
+  Object
+  Passes
+  ScalarOpts
+  Support
+  Target
+  TransformUtils
+  Vectorize
+  X86AsmParser
+  X86CodeGen
+  X86Desc
+  X86Info)
 target_link_libraries(revngSupport z zstd ${LibArchive_LIBRARIES}
                       ${LLVM_LIBRARIES})
 

lib/Support/IRHelpers.cpp

@@ -11,6 +11,8 @@
 #include "llvm/ADT/PostOrderIterator.h"
 #include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/SmallSet.h"
+#include "llvm/Bitcode/BitcodeReader.h"
+#include "llvm/Bitcode/BitcodeWriter.h"
 #include "llvm/IR/DebugInfo.h"
 #include "llvm/IR/DebugInfoMetadata.h"
 #include "llvm/IR/Dominators.h"
@@ -20,6 +22,8 @@
 #include "llvm/Linker/Linker.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/raw_os_ostream.h"
+#include "llvm/Transforms/Utils/Cloning.h"
+#include "llvm/Transforms/Utils/ValueMapper.h"
 
 #include "revng/ADT/Queue.h"
 #include "revng/ADT/RecursiveCoroutine.h"
@@ -691,3 +695,130 @@ void linkModules(std::unique_ptr<Module> &&Source,
         F->setLinkage(FinalLinkage.value_or(Linkage));
   }
 }
+
+struct FunctionsMetadata {
+  static inline const char *MetadataName = "revng.functions-metatadata-backup";
+
+  static void dropBackup(llvm::Module &M) {
+    M.eraseNamedMetadata(M.getNamedMetadata(MetadataName));
+  }
+
+  static void backup(llvm::Module &M) {
+    using namespace llvm;
+
+    LLVMContext &Context = M.getContext();
+
+    // Create the named metadata
+    NamedMDNode *BackupNMD = M.getOrInsertNamedMetadata(MetadataName);
+    BackupNMD->clearOperands();
+
+    // Backup saving names, the metadata kind IDs might change
+    SmallVector<StringRef> MDKindNames;
+    M.getMDKindNames(MDKindNames);
+
+    for (Function &F : M) {
+      // Ignore unnamed functions
+      if (F.getName().empty())
+        continue;
+
+      // Collect metadata for the function
+      SmallVector<std::pair<unsigned, MDNode *>, 8> MDs;
+      F.getAllMetadata(MDs);
+
+      // Skip if no metadata
+      if (MDs.empty())
+        continue;
+
+      // Create an MDNode for the function's metadata
+      // Format: [FunctionName, MDKind1, MDNode1, MDKind2, MDNode2, ...]
+      SmallVector<Metadata *, 8> BackupEntry;
+      BackupEntry.push_back(MDString::get(Context, F.getName()));
+
+      for (const auto &MD : MDs) {
+        BackupEntry.push_back(MDString::get(Context, MDKindNames[MD.first]));
+        BackupEntry.push_back(MD.second);
+      }
+
+      // Append entry
+      BackupNMD->addOperand(MDNode::get(Context, BackupEntry));
+    }
+  }
+
+  static void restore(llvm::Module &M) {
+    using namespace llvm;
+
+    NamedMDNode *BackupNMD = M.getNamedMetadata(MetadataName);
+    if (!BackupNMD)
+      return;
+
+    // Iterate over all operands in the backup NamedMDNode
+    for (MDNode *N : BackupNMD->operands()) {
+      if (N->getNumOperands() == 0)
+        continue;
+
+      // Get the function name
+      StringRef FuncName = cast<MDString>(N->getOperand(0))->getString();
+
+      // Find the function by name
+      Function *F = M.getFunction(FuncName);
+      if (F == nullptr)
+        continue;
+
+      // Clear existing metadata
+      F->clearMetadata();
+
+      auto OperandCount = N->getNumOperands();
+      revng_assert(OperandCount >= 3 and OperandCount % 2 == 1);
+
+      // Restore metadata (operands come in pairs: kind name, MDNode)
+      for (unsigned I = 1; I < OperandCount; I += 2) {
+        auto *KindMD = cast<MDString>(N->getOperand(I).get());
+        MDNode *MD = cast<MDNode>(N->getOperand(I + 1));
+        F->setMetadata(KindMD->getString(), MD);
+      }
+    }
+  }
+};
+
+std::unique_ptr<llvm::Module>
+cloneFiltered(llvm::Module &Module, std::set<const llvm::Function *> &ToClone) {
+  const auto Filter = [&ToClone](const auto &GlobalSym) {
+    if (not llvm::isa<llvm::Function>(GlobalSym))
+      return true;
+
+    const auto &F = llvm::cast<llvm::Function>(GlobalSym);
+    return ToClone.contains(F);
+  };
+
+  llvm::ValueToValueMapTy Map;
+
+  FunctionsMetadata::backup(Module);
+
+  revng::verify(&Module);
+  auto Cloned = llvm::CloneModule(Module, Map, Filter);
+
+  FunctionsMetadata::restore(*Cloned.get());
+  FunctionsMetadata::dropBackup(Module);
+  FunctionsMetadata::dropBackup(*Cloned.get());
+
+  return Cloned;
+}
+
+void writeBitcode(const llvm::Module &Module,
+                  llvm::SmallVectorImpl<char> &Output) {
+  llvm::BitcodeWriter Writer(Output);
+  Writer.writeModule(Module);
+  Writer.writeSymtab();
+  Writer.writeStrtab();
+}
+
+std::unique_ptr<llvm::Module> cloneIntoContext(const llvm::Module &Module,
+                                               llvm::LLVMContext &NewContext) {
+  revng_assert(&Module.getContext() != &NewContext);
+
+  llvm::SmallVector<char, 0> Buffer;
+  writeBitcode(Module, Buffer);
+
+  llvm::MemoryBufferRef BufferRef{ { Buffer.data(), Buffer.size() }, "input" };
+  return llvm::cantFail(llvm::parseBitcodeFile(BufferRef, NewContext));
+}

lib/Support/InitRevng.cpp

@@ -0,0 +1,33 @@
+/// \file InitRevng.cpp
+
+//
+// This file is distributed under the MIT License. See LICENSE.md for details.
+//
+
+#include "llvm/CodeGen/CommandFlags.h"
+#include "llvm/InitializePasses.h"
+#include "llvm/PassRegistry.h"
+#include "llvm/Support/TargetSelect.h"
+
+#include "revng/Support/InitRevng.h"
+
+static std::optional<llvm::codegen::RegisterCodeGenFlags> CodegenFlags;
+
+void revng::InitRevng::initializeLLVMLibraries() {
+  CodegenFlags.emplace();
+  llvm::InitializeNativeTarget();
+  llvm::InitializeNativeTargetAsmPrinter();
+  llvm::InitializeNativeTargetAsmParser();
+
+  auto &Registry = *llvm::PassRegistry::getPassRegistry();
+  llvm::initializeCore(Registry);
+  llvm::initializeTransformUtils(Registry);
+  llvm::initializeScalarOpts(Registry);
+  llvm::initializeVectorization(Registry);
+  llvm::initializeInstCombine(Registry);
+  llvm::initializeIPO(Registry);
+  llvm::initializeAnalysis(Registry);
+  llvm::initializeCodeGen(Registry);
+  llvm::initializeGlobalISel(Registry);
+  llvm::initializeTarget(Registry);
+}

lib/Yield/Assembly/DisassemblyHelper.cpp

@@ -108,15 +108,15 @@ static void analyzeBasicBlocks(yield::Function &Function,
   }
 }
 
-yield::Function DH::disassemble(const model::Function &Function,
-                                const efa::ControlFlowGraph &Metadata,
-                                const RawBinaryView &BinaryView,
-                                const model::Binary &Binary,
-                                const model::AssemblyNameBuilder &NameBuilder) {
+void DH::disassemble(const model::Function &Function,
+                     const efa::ControlFlowGraph &Metadata,
+                     const RawBinaryView &BinaryView,
+                     const model::Binary &Binary,
+                     const model::AssemblyNameBuilder &NameBuilder,
+                     yield::Function &ResultFunction) {
   revng_log(Log, "Disassembling function at " << Function.Entry().toString());
   LoggerIndent Indent(Log);
 
-  yield::Function ResultFunction;
   ResultFunction.Entry() = Function.Entry();
   for (auto BasicBlockInserter = ResultFunction.Blocks().batch_insert();
        const efa::BasicBlock &BasicBlock : Metadata.Blocks()) {
@@ -208,8 +208,6 @@ yield::Function DH::disassemble(const model::Function &Function,
   }
 
   ResultFunction.verify(true);
-
-  return ResultFunction;
 }
 
 LLVMDisassemblerInterface &

lib/Yield/HexDump.cpp

@@ -22,6 +22,7 @@
 #include "revng/Pipes/Ranks.h"
 #include "revng/Support/IRHelpers.h"
 #include "revng/Support/MetaAddress/IntervalContainers.h"
+#include "revng/Yield/HexDump.h"
 
 using namespace llvm;
 
@@ -33,21 +34,15 @@ static FormattedNumber formatNumber(uint64_t Number, unsigned Width = 8) {
   return FormattedNumber(Number, 0, Width, true, false, false);
 };
 
-static void outputHexDump(const TupleTree<model::Binary> &Binary,
-                          const pipeline::LLVMContainer &ModuleContainer,
-                          const CFGMap &CFGMap,
-                          const BinaryFileContainer &SourceBinary,
-                          StringRef OutputPath) {
-  auto BufferOrError = MemoryBuffer::getFileOrSTDIN(*SourceBinary.path());
-  auto Buffer = cantFail(errorOrToExpected(std::move(BufferOrError)));
-  RawBinaryView BinaryView(*Binary.get(), Buffer->getBuffer());
+using CFG = efa::ControlFlowGraph;
+using CFGGetter = std::function<const CFG &(const MetaAddress &)>;
 
-  std::error_code ErrorCode;
-  raw_fd_ostream Output(OutputPath, ErrorCode, sys::fs::CD_CreateAlways);
-
-  revng_assert(not ErrorCode, "Could not open file!");
-
-  ControlFlowGraphCache ControlFlowGraphCache(CFGMap);
+static void outputHexDump(const model::Binary &Binary,
+                          llvm::ArrayRef<const llvm::Function *> Functions,
+                          CFGGetter CFGGetter,
+                          llvm::StringRef BinaryBuffer,
+                          llvm::raw_ostream &Output) {
+  RawBinaryView BinaryView(Binary, BinaryBuffer);
 
   using boost::icl::discrete_interval;
   using boost::icl::inplace_plus;
@@ -71,10 +66,9 @@ static void outputHexDump(const TupleTree<model::Binary> &Binary,
     return Tag;
   };
 
-  for (const Function &F :
-       FunctionTags::Isolated.functions(&ModuleContainer.getModule())) {
-    const efa::ControlFlowGraph &Metadata = ControlFlowGraphCache
-                                              .getControlFlowGraph(&F);
+  for (const Function *F : Functions) {
+    MetaAddress Address = getMetaAddressOfIsolatedFunction(*F);
+    const efa::ControlFlowGraph &Metadata = CFGGetter(Address);
     MetaAddress EntryAddress = Metadata.Entry();
 
     for (const Instruction &I : llvm::instructions(F)) {
@@ -281,12 +275,31 @@ public:
     if (not CFGList.contains(kinds::CFG.allTargets(EC.getContext())))
       return;
 
+    const model::Binary &Binary = *getModelFromContext(EC);
+
+    std::vector<const llvm::Function *> Functions;
+    for (const llvm::Function &F :
+         FunctionTags::Isolated.functions(&ModuleContainer.getModule())) {
+      Functions.push_back(&F);
+    }
+
+    ControlFlowGraphCache CFGCache(CFGMap);
+    auto CFGGetter =
+      [&CFGCache](const MetaAddress &Address) -> const efa::ControlFlowGraph & {
+      return CFGCache.getControlFlowGraph(Address);
+    };
+
+    auto Buffer = revng::cantFail(MemoryBuffer::getFile(*SourceBinary.path()));
+
+    std::error_code ErrorCode;
+    raw_fd_ostream OutputOS(Output.getOrCreatePath(),
+                            ErrorCode,
+                            sys::fs::CD_CreateAlways);
+
+    revng_assert(not ErrorCode, "Could not open file!");
+
     // Proceed with emission
-    outputHexDump(getModelFromContext(EC),
-                  ModuleContainer,
-                  CFGMap,
-                  SourceBinary,
-                  Output.getOrCreatePath());
+    outputHexDump(Binary, Functions, CFGGetter, Buffer->getBuffer(), OutputOS);
 
     EC.commitUniqueTarget(Output);
   }
@@ -295,3 +308,42 @@ public:
 } // namespace revng::pipes
   //
 static pipeline::RegisterPipe<revng::pipes::HexDumpPipe> X;
+
+namespace revng::pypeline::piperuns {
+
+void HexDump::run(const class Model &Model,
+                  llvm::StringRef Config,
+                  llvm::StringRef DynamicConfig,
+                  const BinariesContainer &BinaryContainer,
+                  const LLVMFunctionContainer &ModuleContainer,
+                  const CFGMap &CFG,
+                  HexDumpContainer &Output) {
+
+  const model::Binary &Binary = *Model.get().get();
+
+  auto Buffer = BinaryContainer.getFile(0);
+  auto OutputOS = Output.getOStream(ObjectID{});
+
+  std::vector<const llvm::Function *> Functions;
+  for (const model::Function &Function : Binary.Functions()) {
+    const llvm::Module &Module = ModuleContainer
+                                   .getModule(ObjectID(Function.Entry()));
+    for (const llvm::Function &LLVMFunction :
+         FunctionTags::Isolated.functions(&Module)) {
+      Functions.push_back(&LLVMFunction);
+    }
+  }
+
+  auto CFGGetter =
+    [&CFG](const MetaAddress &Address) -> const efa::ControlFlowGraph & {
+    return *CFG.getElement(ObjectID(Address));
+  };
+
+  ::revng::pipes::outputHexDump(Binary,
+                                Functions,
+                                CFGGetter,
+                                { Buffer.data(), Buffer.size() },
+                                *OutputOS);
+}
+
+} // namespace revng::pypeline::piperuns

lib/Yield/Pipes/AssemblyPipes.cpp

@@ -62,6 +62,45 @@ void ProcessAssembly::run(pipeline::ExecutionContext &Context,
   }
 }
 
+} // namespace revng::pipes
+
+namespace revng::pypeline::piperuns {
+
+ProcessAssembly::ProcessAssembly(const class Model &Model,
+                                 llvm::StringRef Config,
+                                 llvm::StringRef DynamicConfig,
+                                 const BinariesContainer &BinariesContainer,
+                                 const CFGMap &CFG,
+                                 AssemblyInternalContainer &Output) :
+  Binary(*Model.get().get()), CFG(CFG), Output(Output), NameBuilder(Binary) {
+  Helper = std::make_unique<DissassemblyHelper>();
+
+  auto BinaryBuffer = BinariesContainer.getFile(0);
+  BinaryView = std::make_unique<RawBinaryView>(Binary,
+                                               llvm::StringRef{
+                                                 BinaryBuffer.data(),
+                                                 BinaryBuffer.size() });
+};
+
+ProcessAssembly::~ProcessAssembly() = default;
+
+void ProcessAssembly::runOnFunction(const model::Function &TheFunction) {
+  ObjectID Object(TheFunction.Entry());
+  const auto &Metadata = CFG.getElement(Object);
+
+  TupleTree<yield::Function> &OutputFunction = Output.getElement(Object);
+  Helper->disassemble(TheFunction,
+                      *Metadata,
+                      *BinaryView,
+                      Binary,
+                      NameBuilder,
+                      *OutputFunction);
+}
+
+} // namespace revng::pypeline::piperuns
+
+namespace revng::pipes {
+
 void YieldAssembly::run(pipeline::ExecutionContext &Context,
                         const FunctionAssemblyStringMap &Input,
                         FunctionAssemblyPTMLStringMap &Output) {
@@ -108,3 +147,34 @@ static RegisterDefaultConstructibleContainer<FunctionAssemblyPTMLStringMap> X2;
 
 static pipeline::RegisterPipe<revng::pipes::ProcessAssembly> ProcessPipe;
 static pipeline::RegisterPipe<revng::pipes::YieldAssembly> YieldPipe;
+
+namespace revng::pypeline::piperuns {
+
+void YieldAssembly::runOnFunction(const model::Function &TheFunction) {
+  MetaAddress Address = TheFunction.Entry();
+  ObjectID Object(Address);
+  const TupleTree<yield::Function> &Function = Input.getElement(Object);
+
+  revng_assert(Function.verify());
+  revng_assert(Function->verify());
+  revng_assert(Function->Entry() == Address);
+
+  const model::Architecture::Values A = Model.Architecture();
+  auto CommentIndicator = model::Architecture::getAssemblyCommentIndicator(A);
+
+  const model::Configuration &Configuration = Model.Configuration();
+  uint64_t LineWidth = Configuration.CommentLineWidth();
+
+  std::string R = ptml::functionComment(B,
+                                        TheFunction,
+                                        Model,
+                                        CommentIndicator,
+                                        0,
+                                        LineWidth,
+                                        NameBuilder);
+  R += yield::ptml::functionAssembly(B, *Function, Model);
+  R = B.getTag(ptml::tags::Div, std::move(R)).toString();
+  *Output.getOStream(Object) << R;
+}
+
+} // namespace revng::pypeline::piperuns

python/revng/internal/cli/revng2.py

@@ -12,8 +12,8 @@ import os
 from pathlib import Path
 
 import click
-from xdg import xdg_cache_home
 
+from revng.internal.support import cache_directory
 from revng.pypeline.cli.project import project
 from revng.pypeline.cli.utils import PypeGroup
 from revng.pypeline.main import pype, run
@@ -53,10 +53,7 @@ def patch_pype():
                 "PIPELINE", Path(__file__).parent.parent / "pipeline.yml"
             )
         elif param.name == "cache_dir":
-            param.default = os.environ.get(
-                "CACHE_DIR",
-                xdg_cache_home() / "revng",
-            )
+            param.default = cache_directory()
 
 
 def main():

python/revng/internal/pipebox.py

@@ -2,6 +2,7 @@
 # This file is distributed under the MIT License. See LICENSE.md for details.
 #
 
+import signal
 import tarfile
 from io import BytesIO
 
@@ -18,6 +19,24 @@ from revng.support import get_root
 _module, _handles = import_pipebox([get_root() / "lib/librevngPipebox.so"])
 
 
+def initialize(argv: list[str] = []):
+    _module.initialize(
+        {
+            signal.SIGINT,
+            signal.SIGTERM,
+            signal.SIGHUP,
+            signal.SIGQUIT,
+        },
+        {
+            signal.SIGCHLD,
+            signal.SIGPIPE,
+            signal.SIGUSR1,
+            signal.SIGUSR2,
+        },
+        argv,
+    )
+
+
 class ImportFiles(Pipe):
     @classmethod
     def signature(cls) -> tuple[TaskArgument, ...]:
@@ -71,4 +90,7 @@ class ImportFiles(Pipe):
 
         containers[0].deserialize({root_object: buffer.getvalue()})
 
-        return [[(root_object, f"/Binaries/{i}/Hash") for i in indexes]]
+        dependencies = [(root_object, f"/Binaries/{i}/Hash") for i in indexes]
+        dependencies.append((root_object, "/Binaries"))
+
+        return [dependencies]

python/revng/internal/pipeline.yml

@@ -2,4 +2,202 @@
 # This file is distributed under the MIT License. See LICENSE.md for details.
 #
 
-# Placeholder for the revng pipeline
+containers:
+  - name: binaries-container
+    type: BinariesContainer
+  - name: llvm-root
+    type: LLVMRootContainer
+  - name: model-header
+    type: CBytesContainer
+  - name: model-types
+    type: PTMLCTypeContainer
+  - name: cfg-map
+    type: CFGMap
+  - name: llvm-functions
+    type: LLVMFunctionContainer
+  - name: hexdump
+    type: HexDumpContainer
+  - name: assembly-internal
+    type: AssemblyInternalContainer
+  - name: assembly
+    type: AssemblyContainer
+  - name: object-file
+    type: ObjectFileContainer
+  - name: translated
+    type: TranslatedContainer
+  - name: llvm-root-with-functions
+    type: LLVMRootContainer
+
+branches:
+  files-imported:
+    tasks:
+      - pipe: ImportFiles
+        arguments: [binaries-container]
+
+  lift:
+    from: files-imported
+    tasks:
+      - pipe: Lift
+        arguments: [binaries-container, llvm-root]
+      - pipe: PureLLVMPassesRootPipe
+        arguments: [llvm-root]
+        configuration:
+          Passes:
+            - globaldce
+      - savepoint: lifted
+        containers: [llvm-root]
+        artifacts:
+          - name: lift
+            container: llvm-root
+
+  collect-cfg:
+    from: lift
+    tasks:
+      - pipe: CollectCFG
+        arguments: [llvm-root, cfg-map]
+      - savepoint: cfg-computed
+        containers: [cfg-map]
+
+  isolate:
+    from: collect-cfg
+    tasks:
+      - pipe: Isolate
+        arguments: [cfg-map, llvm-root, llvm-functions]
+      - pipe: AttachDebugInfo
+        arguments: [cfg-map, llvm-functions]
+      - savepoint: isolate
+        containers: [llvm-functions]
+        artifacts:
+          - name: isolate
+            container: llvm-functions
+
+  enforce-abi:
+    from: isolate
+    tasks:
+      - pipe: EnforceABI
+        arguments: [cfg-map, llvm-functions]
+      - pipe: PureLLVMPassesPipe
+        arguments: [llvm-functions]
+        configuration:
+          Passes:
+            - strip-debug-info-from-helpers
+      # Note: we're running promote-csvs twice: it is important to run
+      #       it before inline-helpers so that mem2reg can constant
+      #       propagate helper arguments that are constant. This enables
+      #       us to inline less code, in particular for helpers such as
+      #       `cc_compute_c` and `cc_compute_all`.
+      - pipe: PromoteCSVs
+        arguments: [llvm-functions]
+      - pipe: PureLLVMPassesPipe
+        arguments: [llvm-functions]
+        configuration:
+          Passes:
+            - mem2reg
+            - inline-helpers
+      - pipe: AttachDebugInfo
+        arguments: [cfg-map, llvm-functions]
+      - pipe: PromoteCSVs
+        arguments: [llvm-functions]
+      - pipe: PureLLVMPassesPipe
+        arguments: [llvm-functions]
+        configuration:
+          Passes:
+            - remove-exceptional-functions
+            - strip-dead-debug-info
+      - savepoint: enforce-abi
+        containers: [llvm-functions]
+        artifacts:
+          - name: enforce-abi
+            container: llvm-functions
+
+  emit-model-header:
+    tasks:
+      - pipe: ModelToHeader
+        arguments: [model-header]
+      - savepoint: model-header
+        containers: [model-header]
+        artifacts:
+          - name: emit-model-header
+            container: model-header
+
+  emit-type-definition:
+    tasks:
+      - pipe: GenerateModelTypeDefinition
+        arguments: [model-types]
+      - savepoint: model-types
+        containers: [model-types]
+        artifacts:
+          - name: emit-type-definitions
+            container: model-types
+
+  hexdump:
+    from: isolate
+    tasks:
+      - pipe: HexDump
+        arguments:
+          - binaries-container
+          - llvm-functions
+          - cfg-map
+          - hexdump
+      - savepoint: hexdump
+        containers: [hexdump]
+        artifacts:
+          - name: hexdump
+            container: hexdump
+
+  process-assembly:
+    from: collect-cfg
+    tasks:
+      - pipe: ProcessAssembly
+        arguments: [binaries-container, cfg-map, assembly-internal]
+
+  disassemble:
+    from: process-assembly
+    tasks:
+      - pipe: YieldAssembly
+        arguments: [assembly-internal, assembly]
+      - savepoint: disassemble
+        containers: [assembly]
+        artifacts:
+          - name: disassemble
+            container: assembly
+
+  recompile:
+    from: lift
+    tasks:
+      - pipe: LinkSupport
+        arguments: [llvm-root]
+      - pipe: PureLLVMPassesRootPipe
+        arguments: [llvm-root]
+        configuration:
+          Passes: [drop-opaque-return-address]
+      - pipe: CompileRootModule
+        arguments: [llvm-root, object-file]
+      - pipe: LinkForTranslation
+        arguments: [binaries-container, object-file, translated]
+      - savepoint: recompile
+        containers: [translated]
+        artifacts:
+          - name: recompile
+            container: translated
+
+  recompile-isolated:
+    from: isolate
+    tasks:
+      - pipe: InvokeIsolatedFunctions
+        arguments: [llvm-root, llvm-functions, llvm-root-with-functions]
+      - pipe: LinkSupport
+        arguments: [llvm-root-with-functions]
+      - pipe: PureLLVMPassesRootPipe
+        arguments: [llvm-root-with-functions]
+        configuration:
+          Passes: [drop-opaque-return-address]
+      - pipe: CompileRootModule
+        arguments: [llvm-root-with-functions, object-file]
+      - pipe: LinkForTranslation
+        arguments: [binaries-container, object-file, translated]
+      - savepoint: recompile-isolated
+        containers: [translated]
+        artifacts:
+          - name: recompile-isolated
+            container: translated

python/revng/pypeline/cli/pipeline/run_pipe.py

@@ -156,10 +156,7 @@ def build_pipe_command(
         pypeline_logger.debug_log(f"and kwargs: {kwargs}")
 
         # Create the pipe
-        pipe = pipe_type(
-            name=pipe_name,
-            static_configuration=static_configuration,
-        )
+        pipe = pipe_type(static_configuration)
         # Load the model
         with open(model, "rb") as model_file:
             loaded_model = model_type.deserialize(model_file.read())

python/revng/pypeline/cli/project/artifact.py

@@ -84,6 +84,7 @@ class ArtifactGroup(PypeGroup):
             "objects",
             type=str,
             default=None,
+            required=False,
         )(run_artifact_command)
 
         return run_artifact_command
@@ -167,7 +168,7 @@ def build_artifact_command(
         ),
     )
     @click.pass_context
-    def run_analysis_command(
+    def run_artifact_command(
         ctx: click.Context,
         configuration: str,
         project_id: str,
@@ -197,7 +198,7 @@ def build_artifact_command(
             )
         )
 
-    return run_analysis_command
+    return run_artifact_command
 
 
 @click.group(

python/revng/pypeline/container.py

@@ -41,7 +41,7 @@ Configuration = Annotated[
 ]
 
 
-@dataclass(slots=True, frozen=True)
+@dataclass(slots=True, frozen=True, order=True)
 class ContainerDeclaration:
     """
     A ContainerDeclaration represents a container when describing a pipeline.

python/revng/pypeline/graph.py

@@ -13,7 +13,8 @@ def escape(string: str) -> str:
 @dataclass(slots=True)
 class Node:
     label: str
-    completed: bool = False
+    color: str = ""
+    bgcolor: str = ""
     entries: List[str] = field(default_factory=list)
 
     def name(self) -> str:
@@ -26,23 +27,25 @@ class Node:
         result += """    <table border="0" cellborder="1" cellspacing="0">\n"""
 
         cell_properties = ""
-        if self.completed:
-            cell_properties = ' bgcolor="lightgreen"'
+        if self.bgcolor:
+            cell_properties = f' bgcolor="{self.bgcolor}"'
 
         result += f"""      <tr><td{cell_properties}><b>{escape(self.label)}</b></td></tr>\n"""
 
         for index, entry in enumerate(self.entries):
             result += f"""      <tr><td port="entry_{index}">{escape(entry)}</td></tr>\n"""
         result += "    </table>\n"
-        result += "  >];\n"
-        result += "\n"
+        result += "  >"
+        if self.color:
+            result += f', color="{self.color}"'
+        result += "];\n\n"
         return result
 
     def __hash__(self) -> int:
         return hash(self.name())
 
 
-@dataclass(slots=True)
+@dataclass(frozen=True, slots=True)
 class Edge:
     source: Node
     destination: Node
@@ -50,22 +53,37 @@ class Edge:
     destination_port: int = 0
     head_label: str = ""
     tail_label: str = ""
+    style: str = ""
+    color: str = ""
 
     def to_graphviz(self) -> str:
         result = ""
         result += "  "
-        result += f"{escape(self.source.name())}:entry_{self.source_port}"
+        if self.source_port >= 0:
+            result += f"{escape(self.source.name())}:entry_{self.source_port}"
+        else:
+            result += f"{escape(self.source.name())}"
+
         result += " -> "
-        result += f"{escape(self.destination.name())}:entry_{self.destination_port}"
-        if self.head_label or self.tail_label:
-            result += " ["
-            if self.head_label:
-                result += f'headlabel="{escape(self.head_label)}"'
-            if self.tail_label:
-                if self.head_label:
-                    result += ","
-                result += f'taillabel="{escape(self.tail_label)}"'
-            result += "]"
+
+        if self.destination_port >= 0:
+            result += f"{escape(self.destination.name())}:entry_{self.destination_port}"
+        else:
+            result += f"{escape(self.destination.name())}"
+
+        attributes = []
+        if self.head_label:
+            attributes.append(f'headlabel="{escape(self.head_label)}"')
+        if self.tail_label:
+            attributes.append(f'taillabel="{escape(self.tail_label)}"')
+        if self.style:
+            attributes.append(f'style="{escape(self.style)}"')
+        if self.color:
+            attributes.append(f'color="{escape(self.color)}"')
+
+        if len(attributes) > 0:
+            result += f" [{','.join(attributes)}]"
+
         result += ";\n"
         return result
 

python/revng/pypeline/object.py

@@ -10,7 +10,7 @@ from enum import Enum
 
 # Builtin since python 3.9
 from graphlib import TopologicalSorter
-from typing import Iterator, Optional, Sequence, Set, cast
+from typing import Any, Callable, Iterator, Optional, Sequence, Set, cast
 
 from .graph import Graph
 from .utils.cabc import ABC, abstractmethod
@@ -303,6 +303,7 @@ class ObjectSet(MutableSet[ObjectID]):
         return result
 
     def __post_init__(self):
+        assert isinstance(self.kind, Kind)
         for obj in self.objects:
             assert isinstance(obj, ObjectID), f"Expected ObjectID, got {obj}"
             assert obj.kind() == self.kind
@@ -325,6 +326,35 @@ class ObjectSet(MutableSet[ObjectID]):
             return False
         return self.objects == other.objects
 
+    @classmethod
+    def _from_iterable(cls, it):
+        # This functions is called by various collections.abc.Set functions to
+        # compose the container for merge operations (e.g. `&`). These are
+        # overridden below. Throw an error in the offchance that we forgot to
+        # implement one of them.
+        raise NotImplementedError()
+
+    def _merge(
+        self,
+        other: Any,
+        merge_fun: Callable[[set[ObjectID], set[ObjectID]], set[ObjectID]],
+    ) -> ObjectSet:
+        assert isinstance(other, ObjectSet)
+        assert other.kind == self.kind
+        return ObjectSet(self.kind, merge_fun(self.objects, other.objects))
+
+    def __and__(self, other: Any) -> ObjectSet:
+        return self._merge(other, lambda x, y: x & y)
+
+    def __or__(self, other: Any) -> ObjectSet:
+        return self._merge(other, lambda x, y: x | y)
+
+    def __sub__(self, other: Any) -> ObjectSet:
+        return self._merge(other, lambda x, y: x - y)
+
+    def __xor__(self, other: Any) -> ObjectSet:
+        return self._merge(other, lambda x, y: x ^ y)
+
     def __repr__(self) -> str:
         return f"ObjectSet(kind={self.kind.serialize()}, objects={self.objects})"
 

python/revng/pypeline/pipeline.py

@@ -161,21 +161,19 @@ class Pipeline(Generic[C]):
         self, start: Optional[PipelineNode] = None, forward: bool = True, stable: bool = False
     ) -> Generator[PipelineNode, None, None]:
         """BFS walk of pipeline nodes"""
+        assert int(not forward) + int(stable) < 2, "forward=False,stable=True is unsupported"
+
         to_visit: List[PipelineNode] = [start or self.root]
         visited: Set[PipelineNode] = set()
 
         if forward:
 
             def successors(node):
-                if not stable:
-                    return node.successors
-                else:
-                    return node.sorted_successors()
+                return node.sorted_successors() if stable else node.successors
 
         else:
 
             def successors(node):
-                assert not stable
                 return node.predecessors
 
         while len(to_visit) > 0:
@@ -186,40 +184,77 @@ class Pipeline(Generic[C]):
                 if child_node not in visited:
                     to_visit.append(child_node)
 
-    def graph(self) -> Graph:
+    def graph(self, container_edges=False) -> Graph:
         """A graph for debugging purposes."""
 
         graph = Graph()
 
         nodes_map: Dict[PipelineNode, Graph.Node] = {}
 
-        def get_node(node: PipelineNode) -> Graph.Node:
-            if node not in nodes_map:
-                new_node = Graph.Node(node.task.name)
-                for argument in node.arguments:
-                    new_node.entries.append(argument.name)
-
-                nodes_map[node] = new_node
-                graph.nodes.add(new_node)
-
-            return nodes_map[node]
+        access_to_str = {
+            TaskArgumentAccess.READ: "R",
+            TaskArgumentAccess.WRITE: "W",
+            TaskArgumentAccess.READ_WRITE: "RW",
+        }
 
         for node in self.walk_pipeline():
-            graph_node = get_node(node)
-            node_inputs: List[ContainerDeclaration] = list(node.arguments)
+            if isinstance(node.task, Pipe):
+                new_node = Graph.Node(node.task.name)
+                for argument in node.arguments_with_access:
+                    new_node.entries.append(f"{argument.name} [{access_to_str[argument.access]}]")
+            else:
+                new_node = Graph.Node(node.task.name, color="#2A52BE", bgcolor="#6C83BE")
+                for argument in node.arguments_with_access:
+                    new_node.entries.append(argument.name)
+
+            nodes_map[node] = new_node
+            graph.nodes.add(new_node)
 
             for predecessor in node.predecessors:
-                for source_index, argument in enumerate(predecessor.task.arguments):
-                    if argument.access == TaskArgumentAccess.READ or argument not in node_inputs:
+                graph.edges.add(
+                    Graph.Edge(
+                        nodes_map[predecessor],
+                        new_node,
+                        source_port=-1,
+                        destination_port=-1,
+                        style="bold",
+                    )
+                )
+
+            if not container_edges:
+                continue
+
+            node_inputs: list[ContainerDeclaration] = list(node.arguments)
+            taken_inputs: set[int] = set()
+
+            for parent_node in self.walk_pipeline(node, forward=False):
+                if parent_node is node:
+                    continue
+
+                for source_index, parent_argument in enumerate(parent_node.arguments_with_access):
+                    parent_container_decl = parent_argument.to_container_decl()
+                    if (
+                        parent_argument.access == TaskArgumentAccess.READ
+                        or parent_container_decl not in node_inputs
+                    ):
+                        continue
+
+                    destination_index = node_inputs.index(parent_container_decl)
+                    if destination_index in taken_inputs:
+                        continue
+
+                    taken_inputs.add(destination_index)
+                    if parent_node in node.predecessors:
                         continue
 
-                    destination_index = node_inputs.index(argument)
                     graph.edges.add(
                         Graph.Edge(
-                            get_node(predecessor),
-                            graph_node,
+                            nodes_map[parent_node],
+                            new_node,
                             source_port=source_index,
                             destination_port=destination_index,
+                            style="dashed",
+                            color="#FD6D53",
                         )
                     )
 
@@ -266,7 +301,7 @@ class Pipeline(Generic[C]):
             if not node.predecessors:
                 assert node_ingoing_requests.empty(), (
                     f"Node {node} has no predecessors, but it still has "
-                    f"requests: {node_ingoing_requests}"
+                    f"requests: {node_ingoing_requests.minimize()}"
                 )
                 break
 
@@ -281,7 +316,59 @@ class Pipeline(Generic[C]):
             node = predecessor
             node_outgoing_requests = node_ingoing_requests
 
-        return Schedule(self.declarations, tasks[target_node], pipeline_configuration)
+        # Here the task list has been decided, these are now iterated to apply
+        # the following optimizations:
+        # * Reduce the set of container declarations to those that are actually
+        #   necessary to run the schedule
+        # * Compact incoming and outgoing of each task to their reduced version
+        # * Skip tasks where nothing in outgoing is actually written by the task
+        #
+        # These could be derived by using a dataflow-based approach to the
+        # pipeline, where each pipe binds its inputs to the predecessor's pipe
+        # output, but in the general case (with RW pipes) this turns into
+        # having a mini-programming language and applying SSA analysis and the
+        # like to deduplicate the containers (that, in the general case, need
+        # to be duplicated at each node of the dataflow).
+        scheduled_task: ScheduledTask | None = tasks[target_node]
+        parent_scheduled_task: ScheduledTask | None = None
+        used_declatations: set[str] = set()
+
+        while scheduled_task is not None:
+            # Assume that the schedule is a straight line, so a task has at
+            # most one dependency
+            assert len(scheduled_task.depends_on) in (0, 1)
+
+            # Reduce incoming and outgoing
+            scheduled_task.incoming = scheduled_task.incoming.minimize()
+            scheduled_task.outgoing = scheduled_task.outgoing.minimize()
+
+            # Figure out if a task will actually produce outputs
+            written_out = Requests()
+            for argument in scheduled_task.node.arguments_with_access:
+                if argument.access != TaskArgumentAccess.READ:
+                    container_decl = argument.to_container_decl()
+                    if container_decl in scheduled_task.outgoing:
+                        written_out[container_decl] = scheduled_task.outgoing[container_decl]
+
+            if written_out.empty() and parent_scheduled_task is not None:
+                # If here the task does not actually need to produce anything
+                # and can be skipped, by "glueing" its dependencies onto the parent
+                assert [scheduled_task] == parent_scheduled_task.depends_on
+                parent_scheduled_task.depends_on = scheduled_task.depends_on
+            else:
+                used_declatations.update(x.name for x in scheduled_task.node.arguments)
+                parent_scheduled_task = scheduled_task
+
+            if len(scheduled_task.depends_on) == 1:
+                scheduled_task = scheduled_task.depends_on[0]
+            else:
+                scheduled_task = None
+
+        return Schedule(
+            {v for v in self.declarations if v.name in used_declatations},
+            tasks[target_node],
+            pipeline_configuration,
+        )
 
     def get_artifact(
         self,
@@ -350,10 +437,7 @@ class Pipeline(Generic[C]):
         analysis_info.analysis.run(
             model=new_model,
             containers=[all_containers[decl] for decl in analysis_info.bindings],
-            incoming=[
-                requests.get(decl, ObjectSet(decl.container_type.kind, set()))
-                for decl in analysis_info.bindings
-            ],
+            incoming=[requests.get(decl) for decl in analysis_info.bindings],
             configuration=analysis_configuration,
         )
         invalidated = storage_provider.invalidate(ReadOnlyModel(new_model).diff(model))

python/revng/pypeline/pipeline_node.py

@@ -8,14 +8,16 @@ from hashlib import sha256
 from typing import TYPE_CHECKING, Annotated, List, Mapping, Optional, Sequence, Set, Union
 from typing import overload
 
-from .container import Configuration, ConfigurationId, ContainerDeclaration, ContainerSet
+from .container import Configuration, ConfigurationId, Container, ContainerDeclaration
+from .container import ContainerSet
 from .model import ReadOnlyModel
 from .object import ObjectSet
+from .storage.file_provider import FileProvider
 from .storage.storage_provider import SavePointsRange, StorageProvider, StorageProviderFileProvider
 from .task.pipe import Pipe
 from .task.requests import Requests
 from .task.savepoint import SavePoint
-from .task.task import ObjectDependencies, TaskArgumentAccess
+from .task.task import ObjectDependencies, PipeObjectDependencies, TaskArgument, TaskArgumentAccess
 
 if TYPE_CHECKING:
     from _typeshed import SupportsRichComparison
@@ -107,23 +109,27 @@ class PipelineNode:
             assert len(self.bindings) == len(task.arguments)
 
     @property
-    def arguments(self) -> list[ContainerDeclaration]:
+    def arguments_with_access(self) -> list[TaskArgument]:
         if isinstance(self.task, SavePoint):
             # SavePoints do not have bindings, so we return the task arguments directly
-            return [
-                ContainerDeclaration(
-                    name=x.name,
-                    container_type=x.container_type,
-                )
-                for x in self.task.arguments
-            ]
+            return self.task.arguments
         elif isinstance(self.task, Pipe):
             # For Pipes, we return the bindings, which are the pipeline declarations
             # that map to the task arguments
-            return self.bindings
+            result = []
+            for index, argument in enumerate(self.bindings):
+                pipe_arguments = self.task.arguments[index]
+                result.append(
+                    TaskArgument(argument.name, argument.container_type, pipe_arguments.access)
+                )
+            return result
         else:
             raise TypeError(f"Unsupported task type: {type(self.task)}")
 
+    @property
+    def arguments(self) -> list[ContainerDeclaration]:
+        return [x.to_container_decl() for x in self.arguments_with_access]
+
     def add_successor(self, node: PipelineNode) -> PipelineNode:
         node.predecessors.append(self)
         self.successors.append(node)
@@ -161,10 +167,7 @@ class PipelineNode:
             )
         elif isinstance(self.task, Pipe):
             # Use the bindings to map the requests to the pipe arguments
-            pipe_requests: list[ObjectSet] = [
-                requests.get(decl, ObjectSet(decl.container_type.kind, set()))
-                for decl in self.bindings
-            ]
+            pipe_requests: list[ObjectSet] = [requests.get(decl) for decl in self.bindings]
             # Ask the pipe for its prerequisites
             outgoing = self.task.prerequisites_for(
                 model=model,
@@ -213,14 +216,8 @@ class PipelineNode:
             return None
         elif isinstance(self.task, Pipe):
             pipe_containers = [containers[decl] for decl in self.bindings]
-            pipe_incoming = [
-                incoming.get(decl, ObjectSet(decl.container_type.kind, set()))
-                for decl in self.bindings
-            ]
-            pipe_outgoing = [
-                outgoing.get(decl, ObjectSet(decl.container_type.kind, set()))
-                for decl in self.bindings
-            ]
+            pipe_incoming = [incoming.get(decl) for decl in self.bindings]
+            pipe_outgoing = [outgoing.get(decl) for decl in self.bindings]
             deps = self.task.run(
                 file_provider=StorageProviderFileProvider(storage_provider),
                 model=model,
@@ -244,4 +241,32 @@ class PipelineNode:
             raise TypeError(f"Unsupported task type: {type(self.task)}")
 
     def __repr__(self):
-        return self.task.__repr__()
+        return f"<PipelineNode: {self.task!r}>"
+
+
+class DummyPipelineNode(PipelineNode):
+    """Dummy pipeline node, to be used in cases where multiple PipelineNode
+    branches need to be merged into one"""
+
+    class DummyPipe(Pipe):
+        def __init__(self, name: str):
+            self.name = name
+            self.static_configuration = ""
+
+        @classmethod
+        def signature(cls) -> tuple[TaskArgument, ...]:
+            return ()
+
+        def run(
+            self,
+            file_provider: FileProvider,
+            model: ReadOnlyModel,
+            containers: list[Container],
+            incoming: list[ObjectSet],
+            outgoing: list[ObjectSet],
+            configuration: Configuration,
+        ) -> PipeObjectDependencies:
+            return []
+
+    def __init__(self, name: str):
+        super().__init__(self.__class__.DummyPipe(name), [])

python/revng/pypeline/pipeline_parser.py

@@ -5,6 +5,7 @@
 from __future__ import annotations
 
 from dataclasses import dataclass, field
+from graphlib import TopologicalSorter
 from pathlib import Path
 from typing import Any, Optional
 
@@ -14,7 +15,7 @@ import yaml
 from .analysis import Analysis, AnalysisBinding
 from .container import Container, ContainerDeclaration
 from .pipeline import Artifact, Pipeline
-from .pipeline_node import PipelineNode
+from .pipeline_node import DummyPipelineNode, PipelineNode
 from .task.pipe import Pipe
 from .task.savepoint import SavePoint
 from .utils.registry import get_registry
@@ -55,14 +56,16 @@ def parse_pipe(
             )
         bindings.append(container_decls[arg])
 
-    name = task.get("name")
-    configuration = task.get("configuration", "")
+    configuration = task.get("configuration")
+    if configuration is None:
+        configuration_string = ""
+    elif isinstance(configuration, str):
+        configuration_string = configuration
+    else:
+        configuration_string = yaml.safe_dump(configuration)
 
     return PipelineNode(
-        task=pipe_type(
-            name=name,
-            static_configuration=configuration,
-        ),
+        task=pipe_type(configuration_string),
         bindings=bindings,
     )
 
@@ -206,7 +209,7 @@ def parse_branch(
     artifacts: set[Artifact],
     analyses: set[AnalysisBinding],
     container_decls: dict[str, ContainerDeclaration],
-) -> PipelineNode:
+) -> tuple[PipelineNode, PipelineNode]:
     """
     Parse the branch and return the last node in the branch.
     """
@@ -219,6 +222,7 @@ def parse_branch(
 
     # Parse the nodes
     tasks = node.content.get("tasks", [])
+    head = None
     for task in tasks:
         if "pipe" not in task and "savepoint" not in task:
             raise ValueError("Task must have either a pipe or a savepoint")
@@ -233,10 +237,13 @@ def parse_branch(
             container_decls=container_decls,
             parent=parent,
         )
+        if head is None:
+            head = res
         parent = res
 
     assert parent is not None, "A branch cannot be empty and not have a parent"
-    return parent
+    assert head is not None, "A branch needs to have at least one task"
+    return (head, parent)
 
 
 def parse_branches(
@@ -244,7 +251,7 @@ def parse_branches(
     artifacts: set[Artifact],
     analyses: set[AnalysisBinding],
     container_decls: dict[str, ContainerDeclaration],
-) -> PipelineNode:
+) -> list[PipelineNode]:
     """
     Parse the branches from the JSON value.
     The JSON value should contain a dictionary of branches with their names and tasks.
@@ -252,6 +259,7 @@ def parse_branches(
 
     # Find the root branch
     graph: dict[str, Node] = {}
+    roots: set[str] = set()
     for name, branch in branches.items():
         node = Node(content=branch)
         graph[name] = node
@@ -259,44 +267,31 @@ def parse_branches(
             node.is_root = False
             from_node = graph[branch["from"]]
             from_node.successors.add(name)
+        else:
+            roots.add(name)
 
-    # Check that there is only one root branch
-    roots = sorted(name for name, node in graph.items() if node.is_root)
-    if len(roots) != 1:
-        raise ValueError(f"There should be exactly one root branch but found {roots}")
-    root = roots[0]
+    sorter: TopologicalSorter[str] = TopologicalSorter()
+    for name, node in graph.items():
+        sorter.add(name)
+        for successor in node.successors:
+            sorter.add(successor, name)
 
     # Parse the branches in the correct order (DFS)
-    result: dict[str, PipelineNode] = {}
-    queue: list[str] = [root]
-    root_node: PipelineNode | None = None
-    while queue:
-        name = queue.pop(0)
-        node = graph[name]
-        # Parse the branch
-        result[name] = parse_branch(
-            node=node,
-            graph=result,
+    node_tips: dict[str, PipelineNode] = {}
+    result: list[PipelineNode] = []
+    for name in sorter.static_order():
+        first_node, last_node = parse_branch(
+            node=graph[name],
+            graph=node_tips,
             container_decls=container_decls,
             artifacts=artifacts,
             analyses=analyses,
         )
-        if root_node is None:
-            root_node = result[name]
-            # Result contains the END of each branch, so we need to find the root node
-            # by following the predecessors
-            while root_node.predecessors:
-                root_node = root_node.predecessors[0]
-        # Enqueue the successors
-        for successor in node.successors:
-            assert successor not in result, (
-                f"The pipeline has to be a tree, not a DAG. Branch {successor} "
-                "is defined multiple times"
-            )
-            queue.append(successor)
-    # Return the root node
-    assert root_node is not None, "The root node should be defined"
-    return root_node
+        node_tips[name] = last_node
+        if name in roots:
+            result.append(first_node)
+
+    return result
 
 
 def parse_container_decls(
@@ -352,12 +347,20 @@ def load_pipeline(values: Any) -> Pipeline:
     artifacts: set[Artifact] = set()
     analyses: set[AnalysisBinding] = set()
 
-    root: PipelineNode = parse_branches(
+    roots: list[PipelineNode] = parse_branches(
         branches=values["branches"],
         container_decls=container_decls,
         artifacts=artifacts,
         analyses=analyses,
     )
+
+    if len(roots) == 1:
+        root = roots[0]
+    else:
+        root = DummyPipelineNode("root")
+        for node in roots:
+            root.add_successor(node)
+
     return Pipeline(
         declarations=set(container_decls.values()),
         root=root,

python/revng/pypeline/schedule/schedule.py

@@ -59,7 +59,8 @@ class Schedule:
         def get_node(node: ScheduledTask) -> Graph.Node:
             if node not in nodes_map:
                 new_node = Graph.Node(node.node.task.name)
-                new_node.completed = node.completed
+                if node.completed:
+                    new_node.bgcolor = "lightgreen"
                 for argument in node.node.arguments:
                     new_node.entries.append(argument.name)
 
@@ -104,6 +105,10 @@ class Schedule:
         model: ReadOnlyModel,
         storage_provider: StorageProvider,
     ) -> ContainerSet:
+        for task in self.tasks:
+            if isinstance(task.node.task, Pipe):
+                task.node.task.check_precondition(model)
+
         # Produce a set of working containers
         working_containers: ContainerSet = {
             declaration: declaration.instance() for declaration in self.declarations
@@ -181,8 +186,8 @@ class Schedule:
 
                 args = []
                 for declaration in task.node.bindings:
-                    incoming = [x.serialize() for x in task.incoming.get(declaration, [])]
-                    outgoing = [x.serialize() for x in task.outgoing.get(declaration, [])]
+                    incoming = [x.serialize() for x in task.incoming.get(declaration)]
+                    outgoing = [x.serialize() for x in task.outgoing.get(declaration)]
                     args.append(
                         {"name": declaration.name, "incoming": incoming, "outgoing": outgoing}
                     )
@@ -203,8 +208,8 @@ class Schedule:
 
                 sp_containers = []
                 for declaration in self.declarations:
-                    incoming = [x.serialize() for x in task.incoming.get(declaration, [])]
-                    outgoing = [x.serialize() for x in task.outgoing.get(declaration, [])]
+                    incoming = [x.serialize() for x in task.incoming.get(declaration)]
+                    outgoing = [x.serialize() for x in task.outgoing.get(declaration)]
                     if len(incoming) == 0 and len(outgoing) == 0:
                         continue
 

python/revng/pypeline/schedule/scheduled_task.py

@@ -51,7 +51,7 @@ class ScheduledTask:
     to check that the task can run.
     """
 
-    depends_on: list[ScheduledTask] = field(default_factory=list)
+    depends_on: list[ScheduledTask] = field(default_factory=list, repr=False)
     """
     These are the scheduled tasks that this task depends on, after the scheduling
     phase this list will contain only 0 or 1 elements as it's a path, but

python/revng/pypeline/schema.yml

@@ -19,8 +19,6 @@ $defs:
           $ref: "#/$defs/container_decl"
       branches:
         type: object
-        required:
-          - root
         patternProperties:
           ^[a-zA-Z0-9_]+$:
             type: object
@@ -71,19 +69,16 @@ $defs:
       - pipe
       - arguments
     properties:
-      name:
-        type: string
-        description: The name of the pipe task.
-        example: my_pipe_task
       pipe:
         type: string
         description: The name of the pipe.
         example: my_pipe
       configuration:
-        type: string
-        description: The static configuration for the pipe, if any.
-        default: ""
-        example: intel_asm=1
+        type: [string, object]
+        description: |
+          The static configuration for the pipe, if any. Can be either a string
+          or an object. If an object the content will be re-serialized in YAML
+          and passed to the pipe constructor as a string.
       arguments:
         $ref: "#/$defs/args"
         description: The names of the containers to pass to the pipe.

python/revng/pypeline/task/pipe.py

@@ -38,12 +38,8 @@ class Pipe(ABC):
         argument should not be added.
         """
 
-    def __init__(
-        self,
-        static_configuration: str = "",
-        name: str | None = None,
-    ):
-        self.name: str = name or self.__class__.__name__
+    def __init__(self, static_configuration: str = ""):
+        self.name = self.__class__.__name__
         self.static_configuration: str = static_configuration
 
     @property
@@ -87,15 +83,7 @@ class Pipe(ABC):
             if decl.access == TaskArgumentAccess.WRITE:
                 continue
 
-            for ridx, object_list in enumerate(requests):
-                # The requests must be for the writeable containers,
-                # otherwise it's not possible to satisfy them
-                if self.arguments[ridx].access == TaskArgumentAccess.READ:
-                    assert len(object_list) == 0, (
-                        f"Expected an empty request for {self.arguments[ridx].name}, "
-                        f"but got {object_list}."
-                    )
-
+            for object_list in requests:
                 result[idx].update(
                     model.move_to_kind(
                         object_list,
@@ -105,6 +93,14 @@ class Pipe(ABC):
 
         return result
 
+    def check_precondition(self, model: ReadOnlyModel):
+        """
+        Checks that the pipe can be run successfully with the provided model.
+        Subclasses can optionally override this method if they wish to perform
+        checks before the `run` method. An exception should be thrown if some
+        property of the model would not allow running the pipe correctly.
+        """
+
     @abstractmethod
     def run(
         self,

python/revng/pypeline/task/requests.py

@@ -4,11 +4,14 @@
 
 from __future__ import annotations
 
-from typing import Dict, MutableMapping, Optional, Set
+from typing import Dict, MutableMapping, Optional, Set, TypeVar, cast, overload
 
 from revng.pypeline.container import ContainerDeclaration, ContainerSet
 from revng.pypeline.object import ObjectSet
 
+_DUMMY = object()
+T = TypeVar("T")
+
 
 class Requests(MutableMapping[ContainerDeclaration, ObjectSet]):
     """
@@ -76,6 +79,20 @@ class Requests(MutableMapping[ContainerDeclaration, ObjectSet]):
     def __getitem__(self, key: ContainerDeclaration) -> ObjectSet:
         return self.requests[key]
 
+    @overload
+    def get(self, key: ContainerDeclaration, /) -> ObjectSet: ...
+
+    @overload
+    def get(self, key: ContainerDeclaration, default: T, /) -> ObjectSet | T: ...
+
+    def get(self, key: ContainerDeclaration, default: T | object = _DUMMY, /) -> ObjectSet | T:
+        if key in self.requests:
+            return self.requests[key]
+        elif default is _DUMMY:
+            return ObjectSet(key.container_type.kind)
+        else:
+            return cast(T, default)
+
     def __setitem__(self, key: ContainerDeclaration, value: ObjectSet) -> None:
         if not isinstance(value, ObjectSet):
             raise TypeError(f"Expected ObjectSet, got {type(value)}")
@@ -104,3 +121,7 @@ class Requests(MutableMapping[ContainerDeclaration, ObjectSet]):
 
     def __eq__(self, other) -> bool:
         return self.requests == other.requests
+
+    def minimize(self) -> Requests:
+        """Return the same request object, but with all the empty values removed"""
+        return Requests({k: v for k, v in self.requests.items() if len(v) > 0})

python/revng/pypeline/task/savepoint.py

@@ -84,31 +84,20 @@ class SavePoint:
             set(incoming.keys()) | set(outgoing.keys())
         ), "SavePoint containers must be a subset of incoming and outgoing requests."
 
-        # Cache the containers present for this configuration
         for decl in self.to_save:
-            if decl not in incoming:
-                continue
-            if len(incoming[decl]) == 0:
-                continue
-            storage_provider.put(
-                ContainerLocation(
-                    savepoint_id=savepoint_range.start,
-                    container_id=decl.name,
-                    configuration_id=configuration_id,
-                ),
-                containers[decl].serialize(incoming[decl]),
-            )
-
-        # Fill the containers from our cache
-        for decl in self.to_save:
-            if decl not in outgoing:
-                continue
-            # Empty requests do not need to be restored
-            if len(outgoing[decl]) == 0:
-                continue
             location = ContainerLocation(
                 savepoint_id=savepoint_range.start,
                 container_id=decl.name,
                 configuration_id=configuration_id,
             )
-            containers[decl].deserialize(storage_provider.get(location, outgoing[decl]))
+
+            container_incoming = incoming.get(decl)
+            if len(container_incoming) > 0:
+                storage_provider.put(location, containers[decl].serialize(container_incoming))
+
+            # Compute the actual set of objects to load, if an object has
+            # already been saved from incoming do not re-load it from storage
+            container_outgoing = outgoing.get(decl) - container_incoming
+
+            if len(container_outgoing) > 0:
+                containers[decl].deserialize(storage_provider.get(location, container_outgoing))

share/revng/test/configuration/revng/pypeline-comparison.yml

@@ -0,0 +1,61 @@
+#
+# This file is distributed under the MIT License. See LICENSE.md for details.
+#
+
+commands:
+  # Prepare a project directory to run the tests
+  - type: revng.test-pypeline-comparison-prepare
+    from:
+      - type: revng-qa.compiled
+        filter: example-executable-1 and with-debug-info
+    suffix: /
+    command: |-
+      "${SOURCES_ROOT}/share/revng/test/tests/pypeline-comparison/prepare.py" "$INPUT" "$OUTPUT";
+
+  # Run lift comparison
+  - type: revng.test-pypeline-comparison-lift
+    from:
+      - type: revng.test-pypeline-comparison-prepare
+    command: |-
+      cd "$INPUT";
+      "${SOURCES_ROOT}/share/revng/test/tests/pypeline-comparison/compare.sh" model.yml input lift;
+
+  # Run disassemble comparison
+  - type: revng.test-pypeline-comparison-disassemble
+    from:
+      - type: revng.test-pypeline-comparison-prepare
+    command: |-
+      cd "$INPUT";
+      "${SOURCES_ROOT}/share/revng/test/tests/pypeline-comparison/compare.sh" model.yml input disassemble;
+
+  # Run emit-model-header comparison
+  - type: revng.test-pypeline-comparison-emit-model-header
+    from:
+      - type: revng.test-pypeline-comparison-prepare
+    command: |-
+      cd "$INPUT";
+      "${SOURCES_ROOT}/share/revng/test/tests/pypeline-comparison/compare.sh" model.yml input emit-model-header;
+
+  # Run emit-type-definitions comparison
+  - type: revng.test-pypeline-comparison-emit-type-definitions
+    from:
+      - type: revng.test-pypeline-comparison-prepare
+    command: |-
+      cd "$INPUT";
+      "${SOURCES_ROOT}/share/revng/test/tests/pypeline-comparison/compare.sh" model.yml input emit-type-definitions;
+
+  # Run hexdump comparison
+  - type: revng.test-pypeline-comparison-hexdump
+    from:
+      - type: revng.test-pypeline-comparison-prepare
+    command: |-
+      cd "$INPUT";
+      "${SOURCES_ROOT}/share/revng/test/tests/pypeline-comparison/compare.sh" model.yml input hexdump;
+
+  # Run isolate comparison
+  - type: revng.test-pypeline-comparison-isolate
+    from:
+      - type: revng.test-pypeline-comparison-prepare
+    command: |-
+      cd "$INPUT";
+      "${SOURCES_ROOT}/share/revng/test/tests/pypeline-comparison/compare_isolate.sh" model.yml input;

share/revng/test/tests/abi/verify-abi-test-binaries.sh

@@ -18,8 +18,7 @@ test -n "$OUTPUT_DIRECTORY"
 
 SCRIPT_DIRECTORY="$( dirname -- "$( readlink -f -- "$0"; )"; )"
 
-orc shell \
-  llvm-dwarfdump \
+llvm-dwarfdump \
   "$BINARY" \
   > "$OUTPUT_DIRECTORY/dwarf.dump"
 

share/revng/test/tests/pypeline-comparison/compare.sh

@@ -0,0 +1,137 @@
+#!/usr/bin/env bash
+
+#
+# This file is distributed under the MIT License. See LICENSE.md for details.
+#
+
+set -euo pipefail
+
+WORKDIR=$(mktemp --tmpdir -d tmp.revng-pypeline-compare.XXXXXXXXXX)
+trap 'rm -r "$WORKDIR"' EXIT
+
+MODEL="$1"
+BINARY="$2"
+ARTIFACT="$3"
+
+OLD_PATH="$WORKDIR/old"
+NEW_PATH="$WORKDIR/new"
+mkdir "$WORKDIR/cache"
+export REVNG_CACHE_DIR="$WORKDIR/cache"
+
+
+function files_in_dir() {
+    find "$1" -type f -printf '%f\n'
+}
+
+# JQ appends a newline '\n' to the output, this command runs it and strips it
+function jq_exact() {
+    jq "$@" | head -c -1
+}
+
+# Comparison between multiple artifact objects (e.g. function, type-definition)
+function compare() {
+    local PREFIX="$1"
+    local OLD_EXTRACTED NEW_EXTRACTED OLD_FILES
+    OLD_EXTRACTED="$WORKDIR/old_extracted"
+    NEW_EXTRACTED="$WORKDIR/new_extracted"
+    mkdir "$OLD_EXTRACTED" "$NEW_EXTRACTED"
+
+    # Extract all the files from the tar (old format), rename them without all
+    # the extensions
+    tar -C"$OLD_EXTRACTED" -xf "$OLD_PATH"
+    readarray -t OLD_FILES < <(files_in_dir "$OLD_EXTRACTED")
+
+    local FILE
+    for FILE in "${OLD_FILES[@]}"; do
+        local KEY_EXTRACTED
+        KEY_EXTRACTED=$(cut -d. -f1 <<< "$FILE")
+        mv "$OLD_EXTRACTED/$FILE" "$OLD_EXTRACTED/$KEY_EXTRACTED"
+    done
+
+    # Extract all the files from the JSON (new format), strip away the prefix
+    # from each entry
+    local KEY
+    jq -r '. | keys | .[]' "$NEW_PATH" | \
+    while IFS= read -r KEY; do
+        local KEY_EXTRACTED="${KEY/#$PREFIX/}"
+        jq_exact -r ".[\"$KEY\"]" "$NEW_PATH" > "$NEW_EXTRACTED/$KEY_EXTRACTED"
+    done
+
+    if ! diff <(files_in_dir "$OLD_EXTRACTED" | sort) <(files_in_dir "$NEW_EXTRACTED" | sort); then
+        echo "File list mismatch!"
+        return 1
+    fi
+
+    # Compare the files for each element
+    local OK=0
+    while IFS= read -r KEY; do
+        local DIFF_OUTPUT RC=0
+        DIFF_OUTPUT=$(mktemp --tmpdir="$WORKDIR")
+
+        diff -u "$OLD_EXTRACTED/$KEY" "$NEW_EXTRACTED/$KEY" > "$DIFF_OUTPUT" || RC=$?
+        if [[ "$RC" -ne 0 ]]; then
+            OK=1
+            echo "Comparison for $ARTIFACT: $KEY failed"
+            cat "$DIFF_OUTPUT"
+        fi
+    done < <(files_in_dir "$OLD_EXTRACTED")
+
+    return "$OK"
+}
+
+
+# Comparison of the `lift` artifact, this is special because the two formats
+# are quite different and need to be converted to `.ll` before comparing
+function compare_lift() {
+    # Convert old
+    zstdcat "$OLD_PATH" | revng opt -S > "$WORKDIR/old.ll"
+    # Convert new
+    jq_exact -r '.["/binary"]' "$NEW_PATH" | base64 -d | revng opt -S > "$WORKDIR/new.ll"
+
+    local DIFF_OUTPUT RC=0
+    DIFF_OUTPUT=$(mktemp --tmpdir="$WORKDIR")
+    diff -u "$WORKDIR/old.ll" "$WORKDIR/new.ll" > "$DIFF_OUTPUT" || RC=$?
+    if [[ "$RC" -ne 0 ]]; then
+        echo "Comparison for $ARTIFACT failed!"
+        cat "$DIFF_OUTPUT"
+    fi
+
+    return "$RC"
+}
+
+# Comparison between single-valued artifacts
+function compare_binary() {
+    local DIFF_OUTPUT RC=0
+    DIFF_OUTPUT=$(mktemp --tmpdir="$WORKDIR")
+
+    diff -u "$OLD_PATH" <(jq_exact -r '.["/binary"]' "$NEW_PATH") > "$DIFF_OUTPUT" || RC=$?
+    if [[ "$RC" -ne 0 ]]; then
+        echo "Comparison for $ARTIFACT failed!"
+        cat "$DIFF_OUTPUT"
+    fi
+
+    return "$RC"
+}
+
+# Compute the artifacts from both the old and new pipeline
+revng artifact "$ARTIFACT" --model="$MODEL" "$BINARY" -o "$OLD_PATH"
+revng2 project artifact "$ARTIFACT" -o "$NEW_PATH" 2>/dev/null
+
+RC=0
+# NOTE: this could be derived from `pipeline-description.yml`, but since this
+#       is temporary they are hardcorded here.
+if [[ "$ARTIFACT" = "disassemble" ]]; then
+    compare "/function/" || RC=$?
+elif [[ "$ARTIFACT" = "emit-type-definitions" ]]; then
+    compare "/type-definition/" || RC=$?
+elif [[ "$ARTIFACT" = "lift" ]]; then
+    compare_lift || RC=$?
+else
+    compare_binary || RC=$?
+fi
+
+if [[ "$RC" -ne 0 ]]; then
+    echo "Comparison for $ARTIFACT failed!"
+fi
+
+exit "$RC"

share/revng/test/tests/pypeline-comparison/compare_isolate.sh

@@ -0,0 +1,54 @@
+#!/usr/bin/env bash
+
+#
+# This file is distributed under the MIT License. See LICENSE.md for details.
+#
+
+set -euo pipefail
+
+WORKDIR=$(mktemp --tmpdir -d tmp.revng-pypeline-compare.XXXXXXXXXX)
+trap 'rm -r "$WORKDIR"' EXIT
+
+MODEL="$1"
+BINARY="$2"
+
+mkdir "$WORKDIR/cache" "$WORKDIR/old" "$WORKDIR/new"
+export REVNG_CACHE_DIR="$WORKDIR/cache"
+
+# This function takes a bitcode file and normalizes it a bit, with the following:
+# * Run it through globaldce and convert it to textual IR
+# * All metadata IDs are blanked to '!0'
+# * Metadata declarations are dropped
+# * Empty lines and lines starting with comments are dropped
+function normalize() {
+    revng opt -globaldce -S | sed 's;![0-9]\+;!0;g' | grep -v -e '^!0 = ' -e '^\s*$' -e '^;'
+}
+
+OK=0
+while IFS= read -r FUNCTION; do
+    # When working with an LLVMContext, types are pooled into the context. This
+    # poses a problem when adding a type with the same name twice; to fix this
+    # the context automatically appends a `.[0-9]+` suffix to these to
+    # disambiguate. Since this test relies on the diff being identical, both
+    # old and new pipeline commands need to be run individually on each
+    # function, otherwise the types will have the numeric suffix and the
+    # comparison will fail.
+
+    revng artifact isolate --model="$MODEL" "$BINARY" "$FUNCTION" | \
+        zstdcat | normalize > "$WORKDIR/old/$FUNCTION.ll"
+
+    OBJECT_ID="/function/$FUNCTION"
+    revng2 project artifact isolate "$OBJECT_ID" 2>/dev/null | \
+        jq -r ".[\"/function/$FUNCTION\"]" | base64 -d | normalize > "$WORKDIR/new/$FUNCTION.ll"
+
+    DIFF_OUTPUT="$WORKDIR/diff_output_$FUNCTION"
+    RC=0
+    diff -u "$WORKDIR/old/$FUNCTION.ll" "$WORKDIR/new/$FUNCTION.ll" > "$DIFF_OUTPUT" || RC=$?
+    if [[ "$RC" -ne 0 ]]; then
+        echo "Comparison failed for $FUNCTION"
+        cat "$DIFF_OUTPUT"
+        OK=1
+    fi
+done < <(yq -r '.Functions[].Entry' "$MODEL")
+
+exit "$OK"

share/revng/test/tests/pypeline-comparison/prepare.py

@@ -0,0 +1,48 @@
+#!/usr/bin/env python3
+
+#
+# This file is distributed under the MIT License. See LICENSE.md for details.
+#
+
+import os
+import sys
+from hashlib import file_digest
+from pathlib import Path
+from shutil import copy
+from subprocess import run
+from tempfile import TemporaryFile
+
+import yaml
+
+
+# This script prepares the output directory for `revng project ...` commands to
+# be run. Eventually it will be replaced by `revng project init` once that's
+# working.
+def main():
+    input_binary = Path(sys.argv[1])
+    output_dir = Path(sys.argv[2])
+
+    with TemporaryFile("wb+") as temp_model:
+        # Generate a standard model file from the binary, via initial auto-analysis
+        run(["revng", "analyze", "revng-initial-auto-analysis", input_binary], stdout=temp_model)
+        temp_model.seek(0)
+        model = yaml.safe_load(temp_model)
+
+    with open(input_binary, "rb") as f:
+        hash_ = file_digest(f, "sha256").hexdigest()
+        size = f.seek(0, os.SEEK_END)
+
+    # Patch the `Binaries` entry with the input file
+    model["Binaries"] = [{"Index": 0, "Hash": hash_, "Size": size, "Name": "input"}]
+    # Add a reference to the newly-added binary to all the segments
+    for segment in model["Segments"]:
+        segment["Binary"] = "/Binaries/0"
+
+    with open(output_dir / "model.yml", "w") as model_out:
+        yaml.safe_dump(model, model_out)
+
+    copy(input_binary, output_dir / "input")
+
+
+if __name__ == "__main__":
+    main()

tests/pypeline/MockPipeboxImpl.h

@@ -16,6 +16,8 @@ private:
 public:
   static constexpr llvm::StringRef Name = "StringContainer";
   static constexpr Kind Kind = Kinds::Function;
+  static constexpr llvm::StringRef MimeType = "application/x-unknown";
+
   std::set<ObjectID> objects() const;
 
   void deserialize(const std::map<const ObjectID *, llvm::ArrayRef<char>> Data);
@@ -28,10 +30,12 @@ public:
 };
 
 class AppendFooPipe {
+private:
+  using Access = revng::pypeline::Access;
+
 public:
   static constexpr llvm::StringRef Name = "AppendFooPipe";
-  using ArgumentsDocumentation = TypeList<
-    revng::pypeline::PipeArgumentDocumentation<"Container", "">>;
+  using Arguments = TypeList<revng::pypeline::PipeArgument<"Container", "">>;
 
   const std::string StaticConfiguration;
 

tests/pypeline/pipebox.py

@@ -485,15 +485,8 @@ class GeneratorPipe(Pipe):
             ),
         )
 
-    def __init__(
-        self,
-        static_configuration: str = "",
-        name: str | None = None,
-    ):
-        super().__init__(
-            name=name,
-            static_configuration=static_configuration,
-        )
+    def __init__(self, static_configuration: str = ""):
+        super().__init__(static_configuration)
 
     def run(
         self,

tests/pypeline/python_cpp_test.py

@@ -30,7 +30,7 @@ def check_names(ext):
     """Check that _pipebox has all the classes we expect it to have"""
 
     # Names that we know are always present in `_pipebox`
-    known_names = ("Buffer",)
+    known_names = ("Buffer", "initialize")
 
     names = [
         x for x in dir(ext) if not ((x.startswith("__") and x.endswith("__")) or x in known_names)
@@ -40,6 +40,8 @@ def check_names(ext):
     for type_ in (Analysis, Container, Kind, Model, ObjectID, Pipe):
         registry = get_registry(type_)
         for key, value in registry.items():
+            if key == "DummyPipe":
+                continue
             assert issubclass(value, type_)
             assert key in names
             assert getattr(ext, key) is value
@@ -207,6 +209,7 @@ def check_simple_pipeline():
 
 def main():
     ext, _ = import_pipebox(sys.argv[1:])
+    ext.initialize(set(), set(), [])
     initialize_pypeline()
     check_names(ext)
     check_pipeline()