diff --git a/include/revng/ADT/SetOperations.h b/include/revng/ADT/SetOperations.h index fc8ee8c66..abf0227e7 100644 --- a/include/revng/ADT/SetOperations.h +++ b/include/revng/ADT/SetOperations.h @@ -4,6 +4,8 @@ // This file is distributed under the MIT License. See LICENSE.md for details. // +#include "llvm/ADT/SetOperations.h" + template bool intersects(const S1Ty &S1, const S2Ty &S2) { S1Ty Intersection = llvm::set_intersection(S1, S2); diff --git a/include/revng/EarlyFunctionAnalysis/AttachDebugInfo.h b/include/revng/EarlyFunctionAnalysis/AttachDebugInfo.h new file mode 100644 index 000000000..973592b6a --- /dev/null +++ b/include/revng/EarlyFunctionAnalysis/AttachDebugInfo.h @@ -0,0 +1,36 @@ +#pragma once + +// +// This file is distributed under the MIT License. See LICENSE.md for details. +// + +#include "revng/EarlyFunctionAnalysis/CollectCFG.h" +#include "revng/PipeboxCommon/LLVMContainer.h" + +namespace revng::pypeline::piperuns { + +class AttachDebugInfo { +private: + const model::Binary &Binary; + const CFGMap &CFG; + LLVMFunctionContainer &ModuleContainer; + +public: + static constexpr llvm::StringRef Name = "AttachDebugInfo"; + using Arguments = TypeList< + PipeRunArgument, + PipeRunArgument>; + + AttachDebugInfo(const class Model &Model, + llvm::StringRef Config, + llvm::StringRef DynamicConfig, + const CFGMap &CFG, + LLVMFunctionContainer &ModuleContainer) : + Binary(*Model.get().get()), CFG(CFG), ModuleContainer(ModuleContainer){}; + + void runOnFunction(const model::Function &TheFunction); +}; + +} // namespace revng::pypeline::piperuns diff --git a/include/revng/EarlyFunctionAnalysis/CollectCFG.h b/include/revng/EarlyFunctionAnalysis/CollectCFG.h new file mode 100644 index 000000000..1e880a4b9 --- /dev/null +++ b/include/revng/EarlyFunctionAnalysis/CollectCFG.h @@ -0,0 +1,69 @@ +#pragma once + +// +// This file is distributed under the MIT License. See LICENSE.md for details. +// + +#include "revng/EarlyFunctionAnalysis/CFGAnalyzer.h" +#include "revng/EarlyFunctionAnalysis/ControlFlowGraph.h" +#include "revng/Pipebox/TupleTreeContainer.h" +#include "revng/PipeboxCommon/Common.h" +#include "revng/PipeboxCommon/LLVMContainer.h" + +namespace revng::pypeline { + +using CFGMap = TupleTreeContainer; + +namespace piperuns { + +namespace detail { + +struct GCBIRun { + GCBIRun(GeneratedCodeBasicInfo &GCBI, llvm::Module &Module) { + GCBI.run(Module); + } +}; + +} // namespace detail + +class CollectCFG { +private: + const class Model &Model; + CFGMap &Output; + + GeneratedCodeBasicInfo GCBI; + detail::GCBIRun GCBIRun; + efa::FunctionSummaryOracle Oracle; + efa::CFGAnalyzer Analyzer; + +public: + static constexpr llvm::StringRef Name = "CollectCFG"; + using Arguments = TypeList, + PipeRunArgument>; + + CollectCFG(const class Model &Model, + llvm::StringRef Config, + llvm::StringRef DynamicConfig, + LLVMRootContainer &Input, + CFGMap &Output); + void runOnFunction(const model::Function &TheFunction); +}; + +} // namespace piperuns + +} // namespace revng::pypeline diff --git a/include/revng/FunctionIsolation/EnforceABI.h b/include/revng/FunctionIsolation/EnforceABI.h new file mode 100644 index 000000000..637e5a77b --- /dev/null +++ b/include/revng/FunctionIsolation/EnforceABI.h @@ -0,0 +1,41 @@ +#pragma once + +// +// This file is distributed under the MIT License. See LICENSE.md for details. +// + +#include "revng/BasicAnalyses/GeneratedCodeBasicInfo.h" +#include "revng/EarlyFunctionAnalysis/CollectCFG.h" +#include "revng/Model/NameBuilder.h" +#include "revng/PipeboxCommon/LLVMContainer.h" + +class EnforceABI; + +namespace revng::pypeline::piperuns { + +class EnforceABI { +private: + const model::Binary &Binary; + LLVMFunctionContainer &Output; + const CFGMap &CFG; + model::CNameBuilder NameBuilder; + +public: + static constexpr llvm::StringRef Name = "EnforceABI"; + using Arguments = TypeList< + PipeRunArgument, + PipeRunArgument>; + + EnforceABI(const class Model &Model, + llvm::StringRef Config, + llvm::StringRef DynamicConfig, + const CFGMap &CFG, + LLVMFunctionContainer &Output) : + Binary(*Model.get().get()), Output(Output), CFG(CFG), NameBuilder(Binary){}; + + void runOnFunction(const model::Function &TheFunction); +}; + +} // namespace revng::pypeline::piperuns diff --git a/include/revng/FunctionIsolation/InvokeIsolatedFunctions.h b/include/revng/FunctionIsolation/InvokeIsolatedFunctions.h new file mode 100644 index 000000000..492c143f5 --- /dev/null +++ b/include/revng/FunctionIsolation/InvokeIsolatedFunctions.h @@ -0,0 +1,30 @@ +#pragma once + +// +// This file is distributed under the MIT License. See LICENSE.md for details. +// + +#include "revng/PipeboxCommon/LLVMContainer.h" + +namespace revng::pypeline::piperuns { + +class InvokeIsolatedFunctions { +public: + static constexpr llvm::StringRef Name = "InvokeIsolatedFunctions"; + using Arguments = TypeList< + PipeArgument<"RootModule", "Root module containing the root function">, + PipeArgument<"FunctionModules", + "LLVM Modules containing isolated functions">, + PipeArgument<"Output", + "Output LLVM Module with root, functions and dispatcher", + Access::Write>>; + + static void run(const class Model &Model, + llvm::StringRef Config, + llvm::StringRef DynamicConfig, + const LLVMRootContainer &Root, + const LLVMFunctionContainer &Functions, + LLVMRootContainer &Output); +}; + +} // namespace revng::pypeline::piperuns diff --git a/include/revng/FunctionIsolation/IsolateFunctions.h b/include/revng/FunctionIsolation/IsolateFunctions.h index 2ad8b06c3..55999f304 100644 --- a/include/revng/FunctionIsolation/IsolateFunctions.h +++ b/include/revng/FunctionIsolation/IsolateFunctions.h @@ -9,6 +9,7 @@ #include "llvm/Pass.h" #include "revng/BasicAnalyses/GeneratedCodeBasicInfo.h" +#include "revng/EarlyFunctionAnalysis/CollectCFG.h" class IsolateFunctions : public llvm::ModulePass { public: @@ -21,3 +22,48 @@ public: void getAnalysisUsage(llvm::AnalysisUsage &AU) const override; }; + +class IsolateFunctionsImpl; + +namespace revng::pypeline::piperuns { + +class Isolate { +private: + LLVMRootContainer &Root; + LLVMFunctionContainer &Output; + GeneratedCodeBasicInfo GCBI; + // unique_ptr to the implementation. This is a temporary measure until the + // old pipeline is dropped and the body of the `Impl` class can be inlined in + // this one. + std::unique_ptr Impl; + std::vector> IsolatedFunctions; + +public: + static constexpr llvm::StringRef Name = "Isolate"; + using Arguments = TypeList< + PipeRunArgument, + PipeRunArgument, + PipeRunArgument>; + + Isolate(const class Model &Model, + llvm::StringRef Config, + llvm::StringRef DynamicConfig, + const CFGMap &CFG, + LLVMRootContainer &Root, + LLVMFunctionContainer &Output); + ~Isolate(); + + void runOnFunction(const model::Function &TheFunction); +}; + +} // namespace revng::pypeline::piperuns diff --git a/include/revng/FunctionIsolation/PromoteCSVs.h b/include/revng/FunctionIsolation/PromoteCSVs.h index 94e5c283b..786ff90de 100644 --- a/include/revng/FunctionIsolation/PromoteCSVs.h +++ b/include/revng/FunctionIsolation/PromoteCSVs.h @@ -4,6 +4,30 @@ // This file is distributed under the MIT License. See LICENSE.md for details. // -#include "llvm/Pass.h" - #include "revng/BasicAnalyses/GeneratedCodeBasicInfo.h" +#include "revng/PipeboxCommon/LLVMContainer.h" + +namespace revng::pypeline::piperuns { + +class PromoteCSVs { +private: + const model::Binary &Binary; + LLVMFunctionContainer &ModuleContainer; + +public: + static constexpr llvm::StringRef Name = "PromoteCSVs"; + using Arguments = TypeList>; + + PromoteCSVs(const class Model &Model, + llvm::StringRef Config, + llvm::StringRef DynamicConfig, + LLVMFunctionContainer &ModuleContainer) : + Binary(*Model.get().get()), ModuleContainer(ModuleContainer) {} + + void runOnFunction(const model::Function &TheFunction); +}; + +} // namespace revng::pypeline::piperuns diff --git a/include/revng/HeadersGeneration/ModelToHeaderPipe.h b/include/revng/HeadersGeneration/ModelToHeaderPipe.h new file mode 100644 index 000000000..1a0a381d4 --- /dev/null +++ b/include/revng/HeadersGeneration/ModelToHeaderPipe.h @@ -0,0 +1,34 @@ +#pragma once + +// +// This file is distributed under the MIT License. See LICENSE.md for details. +// + +#include "llvm/ADT/StringRef.h" + +#include "revng/ADT/TypeList.h" +#include "revng/PipeboxCommon/Common.h" +#include "revng/PipeboxCommon/Model.h" +#include "revng/PipeboxCommon/RawContainer.h" + +namespace revng::pypeline { + +using CBytesContainer = BytesContainer<"CBytesContainer", "text/x.c+ptml">; + +namespace piperuns { + +class ModelToHeader { +public: + static constexpr llvm::StringRef Name = "ModelToHeader"; + using Arguments = TypeList< + PipeArgument<"Buffer", "The output C header of the model", Access::Write>>; + + static void run(const Model &TheModel, + llvm::StringRef StaticConfig, + llvm::StringRef DynamicConfig, + CBytesContainer &Buffer); +}; + +} // namespace piperuns + +} // namespace revng::pypeline diff --git a/include/revng/HeadersGeneration/ModelTypeDefinitionPipe.h b/include/revng/HeadersGeneration/ModelTypeDefinitionPipe.h new file mode 100644 index 000000000..172c41061 --- /dev/null +++ b/include/revng/HeadersGeneration/ModelTypeDefinitionPipe.h @@ -0,0 +1,44 @@ +#pragma once + +// +// This file is distributed under the MIT License. See LICENSE.md for details. +// + +#include "revng/ADT/TypeList.h" +#include "revng/PipeboxCommon/Common.h" +#include "revng/PipeboxCommon/Model.h" +#include "revng/PipeboxCommon/RawContainer.h" + +namespace revng::pypeline { + +using PTMLCTypeContainer = TypeDefinitionToBytesContainer<"PTMLCTypeContainer", + "text/x.c+ptml">; + +namespace piperuns { + +class GenerateModelTypeDefinition { +private: + const Model &Model; + PTMLCTypeContainer &Output; + +public: + static constexpr llvm::StringRef Name = "GenerateModelTypeDefinition"; + using Arguments = TypeList>; + + GenerateModelTypeDefinition(const class Model &Model, + llvm::StringRef Config, + llvm::StringRef DynamicConfig, + PTMLCTypeContainer &Output) : + Model(Model), Output(Output) {} + + void runOnTypeDefinition(const UpcastablePointer + &TypeDefinition); +}; + +} // namespace piperuns + +} // namespace revng::pypeline diff --git a/include/revng/Lift/Lift.h b/include/revng/Lift/Lift.h index 7353b7993..4ed67168d 100644 --- a/include/revng/Lift/Lift.h +++ b/include/revng/Lift/Lift.h @@ -8,6 +8,9 @@ #include "llvm/Pass.h" #include "revng/Lift/LoadBinaryPass.h" +#include "revng/PipeboxCommon/BinariesContainer.h" +#include "revng/PipeboxCommon/LLVMContainer.h" +#include "revng/PipeboxCommon/Model.h" #include "revng/Pipeline/ExecutionContext.h" #include "revng/Pipes/FunctionPass.h" #include "revng/Support/Debug.h" @@ -172,3 +175,31 @@ public: bool runOnModule(llvm::Module &M) override; }; + +namespace detail { + +llvm::Error liftCheckPrecondition(const model::Binary &Model); + +} + +namespace revng::pypeline::piperuns { + +class Lift { +public: + static constexpr llvm::StringRef Name = "Lift"; + using Arguments = TypeList, + PipeArgument<"Output", + "LLVM Module containing the lifted " + "binaries", + Access::Write>>; + +public: + static llvm::Error checkPrecondition(const class Model &Model); + static void run(const class Model &Model, + llvm::StringRef Config, + llvm::StringRef DynamicConfig, + const BinariesContainer &Binary, + LLVMRootContainer &ModuleContainer); +}; + +} // namespace revng::pypeline::piperuns diff --git a/include/revng/Lift/LinkSupportPipe.h b/include/revng/Lift/LinkSupportPipe.h index e00ab8ce7..27d65f4fc 100644 --- a/include/revng/Lift/LinkSupportPipe.h +++ b/include/revng/Lift/LinkSupportPipe.h @@ -8,6 +8,9 @@ #include "llvm/ADT/ArrayRef.h" +#include "revng/PipeboxCommon/Common.h" +#include "revng/PipeboxCommon/LLVMContainer.h" +#include "revng/PipeboxCommon/Model.h" #include "revng/Pipeline/ContainerSet.h" #include "revng/Pipeline/Contract.h" #include "revng/Pipeline/LLVMContainer.h" @@ -32,3 +35,20 @@ public: }; } // namespace revng::pipes + +namespace revng::pypeline::piperuns { + +class LinkSupport { +public: + static constexpr llvm::StringRef Name = "LinkSupport"; + using Arguments = TypeList< + PipeArgument<"Module", "Module to link support into">>; + +public: + static void run(const class Model &Model, + llvm::StringRef Config, + llvm::StringRef DynamicConfig, + LLVMRootContainer &ModuleContainer); +}; + +} // namespace revng::pypeline::piperuns diff --git a/include/revng/Model/RawBinaryView.h b/include/revng/Model/RawBinaryView.h index a10b9ef0c..5f8a8751f 100644 --- a/include/revng/Model/RawBinaryView.h +++ b/include/revng/Model/RawBinaryView.h @@ -74,6 +74,10 @@ public: unsigned LoadSize, bool IsLittleEndian) const; + // Factored-out checkPrecondition, should be added to each pipe that uses + // this class. Eventually this will be moved into Binary::verify. + static llvm::Error checkPrecondition(const model::Binary &Binary); + private: std::pair findOffsetInSegment(MetaAddress Address, uint64_t Size) const; diff --git a/include/revng/Pipebox/LLVMPipe.h b/include/revng/Pipebox/LLVMPipe.h new file mode 100644 index 000000000..aa0eeffba --- /dev/null +++ b/include/revng/Pipebox/LLVMPipe.h @@ -0,0 +1,105 @@ +#pragma once + +// +// This file is distributed under the MIT License. See LICENSE.md for details. +// + +#include "llvm/IR/LegacyPassManager.h" +#include "llvm/PassInfo.h" +#include "llvm/Support/Progress.h" + +#include "revng/PipeboxCommon/Common.h" +#include "revng/PipeboxCommon/LLVMContainer.h" +#include "revng/PipeboxCommon/Model.h" + +namespace revng::pypeline::pipes { + +namespace detail { + +/// Base class for both PureLLVMPassesRootPipe and PureLLVMPassesPipe, which +/// factors out some common logic. Both pipes run pure LLVM passes and do not +/// read the model, hence the fact that they are plain pipes and not piperuns. +class PureLLVMPassesPipeBase { +public: + struct Configuration { + std::vector Passes; + static Configuration parse(llvm::StringRef Input); + }; + +private: + std::vector PassInfos; + +protected: + std::string TaskName; + +public: + const std::string StaticConfiguration; + PureLLVMPassesPipeBase(llvm::StringRef StaticConfiguration); + +protected: + llvm::legacy::PassManager makePassManager() { + llvm::legacy::PassManager Manager; + for (const llvm::PassInfo *PassInfo : PassInfos) + Manager.add(PassInfo->createPass()); + return Manager; + } +}; + +} // namespace detail + +class PureLLVMPassesRootPipe : public detail::PureLLVMPassesPipeBase { +public: + static constexpr llvm::StringRef Name = "PureLLVMPassesRootPipe"; + using Arguments = TypeList< + PipeArgument<"Module", "LLVM Module to apply the LLVM passes to">>; + +public: + revng::pypeline::ObjectDependencies + run(const Model &TheModel, + const revng::pypeline::Request &Incoming, + const revng::pypeline::Request &Outgoing, + llvm::StringRef Configuration, + LLVMRootContainer &Container) { + if (Outgoing[0].size() == 0) + return {}; + + revng_assert(Outgoing[0].size() == 1); + revng_assert(Outgoing[0][0]->kind() == Kinds::Binary); + + llvm::Task T(1, TaskName); + T.advance("run on LLVMRootContainer", true); + llvm::legacy::PassManager Manager = makePassManager(); + Manager.run(Container.getModule()); + + return {}; + } +}; + +class PureLLVMPassesPipe : public detail::PureLLVMPassesPipeBase { +public: + static constexpr llvm::StringRef Name = "PureLLVMPassesPipe"; + using Arguments = TypeList< + PipeArgument<"Module", "LLVM Modules to apply the LLVM passes to">>; + +public: + revng::pypeline::ObjectDependencies + run(const Model &TheModel, + const revng::pypeline::Request &Incoming, + const revng::pypeline::Request &Outgoing, + llvm::StringRef Configuration, + LLVMFunctionContainer &Container) { + + llvm::Task T(Outgoing[0].size(), TaskName); + llvm::legacy::PassManager Manager = makePassManager(); + for (const ObjectID *Object : Outgoing[0]) { + revng_assert(Object->kind() == Kinds::Function); + const MetaAddress &Key = std::get(Object->key()); + T.advance(Key.toString(), true); + Manager.run(Container.getModule(*Object)); + } + + return {}; + } +}; + +} // namespace revng::pypeline::pipes diff --git a/include/revng/Pipebox/TupleTreeContainer.h b/include/revng/Pipebox/TupleTreeContainer.h new file mode 100644 index 000000000..c80647cdc --- /dev/null +++ b/include/revng/Pipebox/TupleTreeContainer.h @@ -0,0 +1,56 @@ +#pragma once + +// +// This file is distributed under the MIT License. See LICENSE.md for details. +// + +#include "revng/PipeboxCommon/Common.h" +#include "revng/PipeboxCommon/ObjectID.h" + +namespace revng::pypeline { + +template +class TupleTreeContainer { +public: + static constexpr llvm::StringRef Name = TheName; + static constexpr Kind Kind = TheKind; + static constexpr llvm::StringRef MimeType = "text/x.yaml"; + +private: + std::map> Map; + +public: + std::set objects() const { + return std::views::keys(Map) | revng::to>(); + } + + void + deserialize(const std::map> Data) { + for (const auto &[Key, Value] : Data) { + revng_assert(Key->kind() == Kind); + llvm::StringRef String{ Value.data(), Value.size() }; + Map[*Key] = llvm::cantFail(TupleTree::fromString(String)); + } + } + + std::map + serialize(const std::vector Objects) const { + std::map Result; + for (const ObjectID *Object : Objects) { + llvm::raw_svector_ostream OS(Result[*Object].data()); + Map.at(*Object).serialize(OS); + } + return Result; + } + + bool verify() const { return true; } + +public: + bool contains(const ObjectID &Key) const { return Map.contains(Key); } + TupleTree &getElement(const ObjectID &Key) { return Map[Key]; } + const TupleTree &getElement(const ObjectID &Key) const { + return Map.at(Key); + } +}; + +} // namespace revng::pypeline diff --git a/include/revng/PipeboxCommon/BinariesContainer.h b/include/revng/PipeboxCommon/BinariesContainer.h index b552d735c..4bba4e7ba 100644 --- a/include/revng/PipeboxCommon/BinariesContainer.h +++ b/include/revng/PipeboxCommon/BinariesContainer.h @@ -24,6 +24,7 @@ class BinariesContainer { public: static constexpr llvm::StringRef Name = "BinariesContainer"; static constexpr Kind Kind = Kinds::Binary; + static constexpr llvm::StringRef MimeType = "application/x-tar"; private: struct File { @@ -85,6 +86,8 @@ public: bool verify() const { return true; } public: + size_t size() const { return Files.size(); } + llvm::ArrayRef getFile(size_t Index) const { revng_assert(Index < Files.size()); return Files[Index].Contents.data(); diff --git a/include/revng/PipeboxCommon/Common.h b/include/revng/PipeboxCommon/Common.h index cdbf33a52..ada223139 100644 --- a/include/revng/PipeboxCommon/Common.h +++ b/include/revng/PipeboxCommon/Common.h @@ -25,14 +25,42 @@ using ModelPath = std::string; using ObjectDependencies = std::vector< std::vector>>; -template -struct PipeArgumentDocumentation { - static constexpr llvm::StringRef Name = N; - static constexpr llvm::StringRef HelpText = HT; +/// Defines the access of a container when declaring a Pipe{,Run}Argument. In +/// PipeRuns there needs to be exactly one container with either Write or +/// ReadWrite as that will be the one where the model dependencies will be +/// tracked upon. +enum class Access { + /// The requested container will only be read (request objects as a + /// dependency). Note that specifying this allows the container to be used + /// without `const`, this is intended but should really be used as a + /// last-resort in situations where the container remains conceptually const + /// but cannot be for performance reasons. + Read, + /// The requested container will be written (don't request objects as a + /// dependency, the pipe will produce them). This is required when a pipe is + /// the first to write to a container. + Write, + /// The requested container will be overwritten in-place (e.g. LLVM Pipe). + /// This still requests objects as a dependency. + ReadWrite, + /// Automatically detect the access based on const-ness + /// * const -> Read + /// * non-const -> ReadWrite + Auto, }; -template -struct PipeArgument : public PipeArgumentDocumentation { +template +struct PipeArgument { + static constexpr llvm::StringRef Name = N; + static constexpr llvm::StringRef HelpText = HT; + static constexpr Access Access = A; +}; + +template +struct PipeRunArgument : public PipeArgument { using Type = T; }; diff --git a/include/revng/PipeboxCommon/Concepts.h b/include/revng/PipeboxCommon/Concepts.h index a2399dd56..11bf13824 100644 --- a/include/revng/PipeboxCommon/Concepts.h +++ b/include/revng/PipeboxCommon/Concepts.h @@ -28,6 +28,7 @@ concept IsContainer = requires(T &A, const T &AConst) { requires HasName; { T() } -> std::same_as; { T::Kind } -> std::same_as; + { T::MimeType } -> std::same_as; { AConst.objects() } -> std::same_as>; { AConst.verify() } -> std::same_as; { @@ -127,29 +128,29 @@ struct PipeRunTraits { }; template -concept IsPipeArgumentDocumentation = requires { +concept IsPipeArgument = requires { + { T::Access } -> std::same_as; { T::Name } -> std::same_as; { T::HelpText } -> std::same_as; }; template List> -inline constexpr bool checkDocumentation() { +inline constexpr bool checkArguments() { constexpr bool Size = std::tuple_size_v; return compile_time::repeatAnd([]() { - return IsPipeArgumentDocumentation>; + return IsPipeArgument>; }); } template -inline constexpr size_t - DocSize = std::tuple_size_v; +inline constexpr size_t ArgSize = std::tuple_size_v; } // namespace detail template -concept HasArgumentsDocumenation = requires { - requires StrictSpecializationOf; - requires detail::checkDocumentation(); +concept HasArguments = requires { + requires StrictSpecializationOf; + requires detail::checkArguments(); }; template @@ -160,6 +161,12 @@ concept IsPipe = requires(T &A, llvm::StringRef StaticConfig) { requires HasName; { T(StaticConfig) } -> std::same_as; { A.StaticConfiguration } -> std::same_as; - requires HasArgumentsDocumenation; - requires PipeRunTraits::ContainerCount == detail::DocSize; + requires HasArguments; + requires PipeRunTraits::ContainerCount == detail::ArgSize; +}; + +/// Optional method that a pipe can implement +template +concept HasCheckPrecondition = requires(const T &A, const Model &Model) { + { A.checkPrecondition(Model) } -> std::same_as; }; diff --git a/include/revng/PipeboxCommon/Helpers/PipeRunPipes/Base.h b/include/revng/PipeboxCommon/Helpers/PipeRunPipes/Base.h index c1c821047..3699f83fc 100644 --- a/include/revng/PipeboxCommon/Helpers/PipeRunPipes/Base.h +++ b/include/revng/PipeboxCommon/Helpers/PipeRunPipes/Base.h @@ -12,28 +12,52 @@ namespace detail { -template T> +template +constexpr bool isReadOnly(const revng::pypeline::Access &Access) { + using AccessEnum = revng::pypeline::Access; + return Access == AccessEnum::Read + or (Access == AccessEnum::Auto and std::is_const_v); +} + +template T, typename PipeRunT> constexpr size_t writableContainersCount() { size_t Result = 0; forEach([&Result]() { - if constexpr (not std::is_const_v) + using Argument = std::tuple_element_t; + if constexpr (not isReadOnly(Argument::Access)) Result += 1; }); return Result; } -template T> - requires(writableContainersCount() == 1) +template T, typename PipeRunT> + requires(writableContainersCount() == 1) constexpr size_t writableContainerIndex() { int Result = -1; forEach([&Result]() { - if constexpr (not std::is_const_v) { + using Argument = std::tuple_element_t; + if constexpr (not isReadOnly(Argument::Access)) { Result = I; } }); return Result; } +template +concept HasPipeRunCheckPrecondition = requires(const Model &Model) { + { T::checkPrecondition(Model) } -> std::same_as; +}; + +template +struct CheckPreconditionMixin {}; + +template +struct CheckPreconditionMixin { + llvm::Error checkPrecondition(const Model &Model) const { + return T::checkPrecondition(Model); + } +}; + } // namespace detail template @@ -43,7 +67,7 @@ concept SingleOutputPipeBaseCompatible = requires { }; template -class SingleOutputPipeBase { +class SingleOutputPipeBase : public detail::CheckPreconditionMixin { public: static constexpr llvm::StringRef Name = T::Name; using ContainerTypes = PipeRunContainerTypes; @@ -56,7 +80,7 @@ public: protected: static constexpr size_t ContainerCount = std::tuple_size_v; static constexpr size_t - OutputContainerIndex = detail::writableContainerIndex(); + OutputContainerIndex = detail::writableContainerIndex(); using OutputContainerType = std::tuple_element_t; }; diff --git a/include/revng/PipeboxCommon/Helpers/PipeRunPipes/FunctionPipe.h b/include/revng/PipeboxCommon/Helpers/PipeRunPipes/FunctionPipe.h index b0f28a5e5..44105d3d8 100644 --- a/include/revng/PipeboxCommon/Helpers/PipeRunPipes/FunctionPipe.h +++ b/include/revng/PipeboxCommon/Helpers/PipeRunPipes/FunctionPipe.h @@ -4,6 +4,8 @@ // This file is distributed under the MIT License. See LICENSE.md for details. // +#include "llvm/Support/Progress.h" + #include "revng/PipeboxCommon/Helpers/PipeRunPipes/Base.h" #include "revng/PipeboxCommon/Helpers/PipeRunPipes/Helpers.h" @@ -26,7 +28,7 @@ private: static_assert(Base::OutputContainerType::Kind == Kinds::Function); public: - using ArgumentsDocumentation = T::Arguments; + using Arguments = T::Arguments; public: template @@ -38,17 +40,26 @@ public: llvm::StringRef Configuration, Args &...Containers) { ObjectDependenciesHelper ODH(Model, Outgoing, this->ContainerCount); + + llvm::Task T1(3, "Running " + this->Name); + T1.advance("prologue", true); + T Instance(Model, this->StaticConfiguration, Configuration, Containers...); const model::Binary &Binary = *Model.get().get(); + auto &RequestedFunctions = Outgoing.at(this->OutputContainerIndex); - for (const ObjectID *Object : Outgoing.at(this->OutputContainerIndex)) { - auto Committer = ODH.getCommitterFor(*Object, this->OutputContainerIndex); - + T1.advance("run on functions", true); + llvm::Task T2(RequestedFunctions.size(), "Running pipe on functions"); + for (const ObjectID *Object : RequestedFunctions) { const MetaAddress &Entry = std::get(Object->key()); + T2.advance(Entry.toString(), true); + + auto Committer = ODH.getCommitterFor(*Object, this->OutputContainerIndex); const model::Function &Function = Binary.Functions().at(Entry); Instance.runOnFunction(Function); } + T1.advance("epilogue"); return ODH.takeDependencies(); } }; diff --git a/include/revng/PipeboxCommon/Helpers/PipeRunPipes/Helpers.h b/include/revng/PipeboxCommon/Helpers/PipeRunPipes/Helpers.h index a51563d45..449bad673 100644 --- a/include/revng/PipeboxCommon/Helpers/PipeRunPipes/Helpers.h +++ b/include/revng/PipeboxCommon/Helpers/PipeRunPipes/Helpers.h @@ -135,23 +135,23 @@ private: namespace detail { template -concept IsPipeArgument = requires { - requires IsPipeArgumentDocumentation; +concept IsPipeRunArgument = requires { + requires IsPipeArgument; requires IsContainerArgument; }; template List> -inline constexpr bool checkArguments() { +inline constexpr bool checkPipeRunArguments() { constexpr bool Size = std::tuple_size_v; return compile_time::repeatAnd([]() { - return IsPipeArgument>; + return IsPipeRunArgument>; }); } template -concept HasArguments = requires { +concept HasPipeRunArguments = requires { requires StrictSpecializationOf; - requires checkArguments(); + requires checkPipeRunArguments(); }; template @@ -164,13 +164,13 @@ template concept IsSingleObjectPipeRun = requires { requires HasName; requires detail::HasStaticRun; - requires not detail::HasArguments; + requires not detail::HasPipeRunArguments; }; template concept IsMultipleObjectsPipeRun = requires { requires HasName; - requires detail::HasArguments; + requires detail::HasPipeRunArguments; requires not detail::HasStaticRun; }; diff --git a/include/revng/PipeboxCommon/Helpers/PipeRunPipes/LLVMFunctionPassPipe.h b/include/revng/PipeboxCommon/Helpers/PipeRunPipes/LLVMFunctionPassPipe.h index 7569d9a74..254415178 100644 --- a/include/revng/PipeboxCommon/Helpers/PipeRunPipes/LLVMFunctionPassPipe.h +++ b/include/revng/PipeboxCommon/Helpers/PipeRunPipes/LLVMFunctionPassPipe.h @@ -6,6 +6,7 @@ #include "llvm/IR/LegacyPassManager.h" #include "llvm/Pass.h" +#include "llvm/Support/Progress.h" #include "revng/Model/FunctionTags.h" #include "revng/PipeboxCommon/Helpers/PipeRunPipes/Base.h" @@ -25,6 +26,13 @@ inline constexpr llvm::StringRef PassNamePrefix = "LLVMFunctionPassPipe " inline constexpr llvm::StringRef PassArgumentPrefix = "llvm-function-pass-pipe-" "for-"; +template TL> +constexpr bool checkAnalyses() { + return compile_time::repeatAnd>([]() { + return std::is_base_of_v>; + }); +} + } // namespace detail template @@ -32,6 +40,7 @@ concept IsLLVMFunctionPassPipe = requires(T &PipeRun, const model::Function &Function, llvm::Function &LLVMFunction) { requires IsMultipleObjectsPipeRun; + requires detail::checkAnalyses(); requires hasConstructor< T, // The Structure of the constructor is: @@ -42,19 +51,23 @@ concept IsLLVMFunctionPassPipe = requires(T &PipeRun, { PipeRun.runOnFunction(Function, LLVMFunction) } -> std::same_as; }; -template +template class LLVMFunctionPassPipe : public SingleOutputPipeBase { private: using Base = SingleOutputPipeBase; - using LLVMRootContainer = revng::pypeline::LLVMRootContainer; - static_assert(std::is_same_v); + static_assert(anyOf()); + static constexpr bool + SingleModule = not std::is_same_v; using ContainerTypesRef = detail::TupleWithRef; class Pass : public llvm::ModulePass { private: ObjectDependenciesHelper &ODH; const revng::pypeline::Request &Outgoing; + std::optional PipeRun; const Model &Model; llvm::StringRef StaticConfiguration; @@ -79,7 +92,17 @@ private: Model(Model), StaticConfiguration(StaticConfiguration), Configuration(Configuration), - Containers(Containers) {} + Containers(Containers) { + + compile_time::callWithIndexSequence< + Base::ContainerCount>([&]() { + PipeRun.emplace(*this, + Model, + StaticConfiguration, + Configuration, + std::get(Containers)...); + }); + } void getAnalysisUsage(llvm::AnalysisUsage &AU) const override { forEach([&AU]() { @@ -88,37 +111,64 @@ private: } bool runOnModule(llvm::Module &Module) override { - std::map AddressToFunction; - for (llvm::Function &Function : Module.functions()) { - if (not FunctionTags::Isolated.isTagOf(&Function)) - continue; + if constexpr (SingleModule) + return runOnSingleModule(Module); + else + return runOnFunctionModule(Module); + } - AddressToFunction.emplace(getMetaAddressOfIsolatedFunction(Function), - &Function); + bool runOnSingleModule(llvm::Module &Module) { + std::map AddressToFunction; + for (llvm::Function *Function : getFunctions(Module)) { + AddressToFunction.emplace(getMetaAddressOfIsolatedFunction(*Function), + Function); } - T Instance = compile_time::callWithIndexSequence< - Base::ContainerCount>([&]() { - return T{ *this, - Model, - this->StaticConfiguration, - Configuration, - std::get(Containers)... }; - }); - const model::Binary &Binary = *Model.get().get(); - for (const ObjectID *Object : Outgoing.at(Base::OutputContainerIndex)) { + auto &RequestedFunctions = Outgoing.at(Base::OutputContainerIndex); + llvm::Task T1(RequestedFunctions.size(), "Running pipe on functions"); + for (const ObjectID *Object : RequestedFunctions) { + const MetaAddress &Entry = std::get(Object->key()); + T1.advance(Entry.toString(), true); + auto Committer = ODH.getCommitterFor(*Object, Base::OutputContainerIndex); - - const MetaAddress &Entry = std::get(Object->key()); const model::Function &Function = Binary.Functions().at(Entry); llvm::Function *LLVMFunction = AddressToFunction.at(Function.Entry()); - Instance.runOnFunction(Function, *LLVMFunction); + PipeRun->runOnFunction(Function, *LLVMFunction); } return true; } + + bool runOnFunctionModule(llvm::Module &Module) { + std::set Functions = getFunctions(Module); + revng_assert(Functions.size() == 1); + llvm::Function &LLVMFunction = **Functions.begin(); + MetaAddress Address = getMetaAddressOfIsolatedFunction(LLVMFunction); + ObjectID Object(Address); + const model::Binary &Binary = *Model.get().get(); + + { + auto Committer = ODH.getCommitterFor(Object, + Base::OutputContainerIndex); + const model::Function &Function = Binary.Functions().at(Address); + PipeRun->runOnFunction(Function, LLVMFunction); + } + + return true; + } + + private: + static std::set getFunctions(llvm::Module &Module) { + std::set Result; + for (llvm::Function &Function : Module.functions()) { + if (FunctionTags::Isolated.isTagOf(&Function) + and not Function.isDeclaration()) + Result.insert(&Function); + } + return Result; + } }; private: @@ -131,7 +181,7 @@ private: true }; public: - using ArgumentsDocumentation = T::Arguments; + using Arguments = T::Arguments; public: template @@ -149,6 +199,8 @@ public: Manager.add(new A()); }); + llvm::Task T1(3, "Running " + this->Name); + T1.advance("prologue", true); ContainerTypesRef ContainersRef(Containers...); Manager.add(new Pass(ODH, Outgoing, @@ -156,10 +208,23 @@ public: this->StaticConfiguration, Configuration, ContainersRef)); - LLVMRootContainer - &ModuleContainer = std::get(ContainersRef); - Manager.run(ModuleContainer.getModule()); + auto &ModuleContainer = std::get(ContainersRef); + T1.advance("run on functions", true); + if constexpr (SingleModule) { + Manager.run(ModuleContainer.getModule()); + } else { + auto &RequestedFunctions = Outgoing[this->OutputContainerIndex]; + llvm::Task T2(RequestedFunctions.size(), "Running pipe on functions"); + for (const ObjectID *Object : RequestedFunctions) { + MetaAddress Address = std::get(Object->key()); + T2.advance(Address.toString(), true); + + Manager.run(ModuleContainer.getModule(Address)); + } + } + + T1.advance("epilogue", true); return ODH.takeDependencies(); } }; diff --git a/include/revng/PipeboxCommon/Helpers/PipeRunPipes/SingleOutputPipe.h b/include/revng/PipeboxCommon/Helpers/PipeRunPipes/SingleOutputPipe.h index fca333c3d..ee7d18cdf 100644 --- a/include/revng/PipeboxCommon/Helpers/PipeRunPipes/SingleOutputPipe.h +++ b/include/revng/PipeboxCommon/Helpers/PipeRunPipes/SingleOutputPipe.h @@ -4,6 +4,8 @@ // This file is distributed under the MIT License. See LICENSE.md for details. // +#include "llvm/Support/Progress.h" + #include "revng/PipeboxCommon/Concepts.h" #include "revng/PipeboxCommon/Helpers/PipeRunPipes/Base.h" #include "revng/PipeboxCommon/Helpers/PipeRunPipes/Helpers.h" @@ -12,7 +14,7 @@ template concept IsSingleOutputPipeRun = requires { requires IsSingleObjectPipeRun; - requires HasArgumentsDocumenation; + requires HasArguments; requires SpecializationOf, TypeList>; }; @@ -22,7 +24,7 @@ private: using Base = SingleOutputPipeBase; public: - using ArgumentsDocumentation = T::ArgumentsDocumentation; + using Arguments = T::Arguments; public: template @@ -33,9 +35,14 @@ public: const revng::pypeline::Request &Outgoing, llvm::StringRef Configuration, Args &...Containers) { - revng_assert(Outgoing.at(this->OutputContainerIndex).size() == 1); - ObjectDependenciesHelper ODH(Model, Outgoing, this->ContainerCount); + auto &RequestedOutputs = Outgoing.at(this->OutputContainerIndex); + if (RequestedOutputs.size() == 0) + return ODH.takeDependencies(); + + revng_assert(RequestedOutputs.size() == 1); + llvm::Task T1(1, "Running " + this->Name); + T1.advance("Running 'run'", true); T::run(Model, this->StaticConfiguration, Configuration, Containers...); ODH.commitUniqueTarget(this->OutputContainerIndex); return ODH.takeDependencies(); diff --git a/include/revng/PipeboxCommon/Helpers/PipeRunPipes/TypeDefinitionPipe.h b/include/revng/PipeboxCommon/Helpers/PipeRunPipes/TypeDefinitionPipe.h index 04ea76a03..d51fcb134 100644 --- a/include/revng/PipeboxCommon/Helpers/PipeRunPipes/TypeDefinitionPipe.h +++ b/include/revng/PipeboxCommon/Helpers/PipeRunPipes/TypeDefinitionPipe.h @@ -4,6 +4,8 @@ // This file is distributed under the MIT License. See LICENSE.md for details. // +#include "llvm/Support/Progress.h" + #include "revng/PipeboxCommon/Helpers/PipeRunPipes/Base.h" #include "revng/PipeboxCommon/Helpers/PipeRunPipes/Helpers.h" @@ -28,7 +30,7 @@ private: static_assert(Base::OutputContainerType::Kind == Kinds::TypeDefinition); public: - using ArgumentsDocumentation = T::Arguments; + using Arguments = T::Arguments; public: template @@ -40,18 +42,29 @@ public: llvm::StringRef Configuration, Args &...Containers) { ObjectDependenciesHelper ODH(Model, Outgoing, this->ContainerCount); + + llvm::Task T1(3, "Running " + this->Name); + T1.advance("prologue"); + T Instance(Model, this->StaticConfiguration, Configuration, Containers...); const model::Binary &Binary = *Model.get().get(); + auto &RequestedTypeDefinitions = Outgoing.at(this->OutputContainerIndex); - for (const ObjectID *Object : Outgoing.at(this->OutputContainerIndex)) { - auto Committer = ODH.getCommitterFor(*Object, this->OutputContainerIndex); - + T1.advance("run on type definitions", true); + llvm::Task T2(RequestedTypeDefinitions.size(), + "Running pipe on type definitions"); + for (const ObjectID *Object : RequestedTypeDefinitions) { using TDK = model::TypeDefinition::Key; const TDK &Entry = std::get(Object->key()); + T2.advance(toString(Entry), true); + + auto Committer = ODH.getCommitterFor(*Object, this->OutputContainerIndex); const UpcastablePointer &TypeDefinition = Binary.TypeDefinitions().at(Entry); Instance.runOnTypeDefinition(TypeDefinition); } + + T1.advance("epilogue"); return ODH.takeDependencies(); } }; diff --git a/include/revng/PipeboxCommon/Helpers/Python/Helpers.h b/include/revng/PipeboxCommon/Helpers/Python/Helpers.h index b0bc0e37d..7a9a8ad79 100644 --- a/include/revng/PipeboxCommon/Helpers/Python/Helpers.h +++ b/include/revng/PipeboxCommon/Helpers/Python/Helpers.h @@ -12,6 +12,7 @@ #include "revng/PipeboxCommon/Common.h" #include "revng/PipeboxCommon/Helpers/Helpers.h" +#include "revng/PipeboxCommon/Model.h" #include "revng/PipeboxCommon/ObjectID.h" namespace revng::pypeline::helpers { @@ -71,6 +72,22 @@ public: const Py_buffer *operator->() { return &Buffer; } }; +inline Model &convertReadOnlyModel(nanobind::object &TheModel) { + nanobind::object ReadOnlyModel = importObject("revng.pypeline.model." + "ReadOnlyModel"); + revng_assert(nanobind::isinstance(TheModel, ReadOnlyModel)); + nanobind::object ActualModel = nanobind::getattr(TheModel, "downcast")(); + return *nanobind::cast(ActualModel); +} + +template +inline nanobind::capsule makeCapsule(ArgsT &&...Args) { + return nanobind::capsule(new T(std::forward(Args)...), + [](void *Ptr) noexcept { + delete static_cast(Ptr); + }); +} + } // namespace python } // namespace revng::pypeline::helpers diff --git a/include/revng/PipeboxCommon/Helpers/Python/RunPipe.h b/include/revng/PipeboxCommon/Helpers/Python/RunPipe.h index cf0e4ada4..ed7959580 100644 --- a/include/revng/PipeboxCommon/Helpers/Python/RunPipe.h +++ b/include/revng/PipeboxCommon/Helpers/Python/RunPipe.h @@ -21,14 +21,9 @@ inline ObjectDependencies runPipe(T &Handle, Request Incoming, Request Outgoing, llvm::StringRef Configuration) { - nanobind::object ReadOnlyModel = importObject("revng.pypeline.model." - "ReadOnlyModel"); - revng_assert(nanobind::isinstance(TheModel, ReadOnlyModel)); - nanobind::object ActualModel = nanobind::getattr(TheModel, "downcast")(); - const Model *CppModel = nanobind::cast(ActualModel); - + const Model &CppModel = convertReadOnlyModel(TheModel); return revng::pypeline::helpers::runPipe(Handle, - *CppModel, + CppModel, Incoming, Outgoing, Configuration, diff --git a/include/revng/PipeboxCommon/Helpers/Python/SignatureHelper.h b/include/revng/PipeboxCommon/Helpers/Python/SignatureHelper.h index 003e881b9..d69229838 100644 --- a/include/revng/PipeboxCommon/Helpers/Python/SignatureHelper.h +++ b/include/revng/PipeboxCommon/Helpers/Python/SignatureHelper.h @@ -62,8 +62,7 @@ private: forEach([&Result, &TaskArgument, &TaskArgumentAccess]() { - using Argument = std::tuple_element_t; + using Argument = std::tuple_element_t; // Create a Kwargs dictionary, this will be passed to the constructor of // TaskArgument nanobind::dict Kwargs; @@ -75,11 +74,8 @@ private: // nanobind::type returns a reference, so we need to borrow it and // increase the reference count Kwargs["container_type"] = nanobind::borrow(nanobind::type()); - // If the argument is const then the access is READ, otherwise READ_WRITE - if constexpr (std::is_const_v) - Kwargs["access"] = nanobind::getattr(TaskArgumentAccess, "READ"); - else - Kwargs["access"] = nanobind::getattr(TaskArgumentAccess, "READ_WRITE"); + Kwargs["access"] = nanobind::getattr(TaskArgumentAccess, + getAccess(Argument::Access)); // kwargs_proxy is a special nanobind class that allows passing a // nanobind::dict as kwargs, this is equivalent to doing @@ -92,6 +88,26 @@ private: return nanobind::tuple(Result); } + /// Given a container type A and a pipe's Access, determine the correct member + /// of the `TaskArgumentAccess` python enum. + template + static const char *getAccess(const Access &Access) { + switch (Access) { + case Access::Read: + return "READ"; + case Access::Write: + return "WRITE"; + case Access::ReadWrite: + return "READ_WRITE"; + case Access::Auto: + // If the argument is const then the access is READ, otherwise READ_WRITE + if constexpr (std::is_const_v) + return "READ"; + else + return "READ_WRITE"; + } + } + /// Given an analysis type T, compute its signature static nanobind::object parseAnalysisSignature() { nanobind::list Result; diff --git a/include/revng/PipeboxCommon/Helpers/Registrars.h b/include/revng/PipeboxCommon/Helpers/Registrars.h index 4299d2707..dc7988ee8 100644 --- a/include/revng/PipeboxCommon/Helpers/Registrars.h +++ b/include/revng/PipeboxCommon/Helpers/Registrars.h @@ -24,6 +24,8 @@ #include "revng/PipeboxCommon/Helpers/Python/RunPipe.h" #include "revng/PipeboxCommon/Helpers/Python/SignatureHelper.h" +inline Logger PypelineRegisterLogger("pypeline-register"); + template struct RegisterAnalysis { RegisterAnalysis() { @@ -33,7 +35,8 @@ struct RegisterAnalysis { // Python python::Registry.registerModuleInitializer([](nanobind::module_ &M, python::BaseClasses &BC) { - nanobind::class_(M, T::Name.data(), BC.BaseAnalysis) + std::string Name = T::Name.str(); + nanobind::class_(M, Name.c_str(), BC.BaseAnalysis) .def_ro_static("name", &T::Name) .def(nanobind::init<>()) .def_static("signature", @@ -66,8 +69,10 @@ struct RegisterContainer { // Python python::Registry.registerModuleInitializer([](nanobind::module_ &M, python::BaseClasses &BC) { - nanobind::class_(M, T::Name.data(), BC.BaseContainer) + std::string Name = T::Name.str(); + nanobind::class_(M, Name.c_str(), BC.BaseContainer) .def_ro_static("kind", &T::Kind) + .def_static("mime_type", []() { return T::MimeType; }) .def(nanobind::init<>()) .def("objects", &python::ContainerIO::objects) .def("verify", &T::verify) @@ -84,17 +89,41 @@ struct RegisterContainer { } }; +namespace detail { + +template +void checkPipeArgumentAccess() { + if (not PypelineRegisterLogger.isEnabled()) + return; + + using CT = PipeRunTraits::ContainerTypes; + using AccessEnum = revng::pypeline::Access; + forEach([]() { + using Argument = std::tuple_element_t; + if (Argument::Access == AccessEnum::Read and not std::is_const_v) { + revng_log(PypelineRegisterLogger, + T::Name << " has the " << I + << "th argument with READ access but marked non-const"); + } + }); +} + +} // namespace detail + template struct RegisterPipe { RegisterPipe() { using namespace nanobind::literals; using namespace revng::pypeline::helpers; + detail::checkPipeArgumentAccess(); + // Python python::Registry.registerModuleInitializer([](nanobind::module_ &M, python::BaseClasses &BC) { - nanobind::class_(M, T::Name.data(), BC.BasePipe) - .def_ro_static("name", &T::Name) + std::string Name = T::Name.str(); + auto PipeClass = nanobind::class_(M, Name.c_str(), BC.BasePipe); + PipeClass.def_ro_static("name", &T::Name) .def_static("signature", &python::SignatureHelper::getSignature, nanobind::sig("def signature() -> " @@ -111,6 +140,14 @@ struct RegisterPipe { "incoming"_a, "outgoing"_a, "configuration"_a); + + if constexpr (HasCheckPrecondition) { + PipeClass + .def("check_precondition", [](T &Handle, nanobind::object TheModel) { + const Model &CppModel = python::convertReadOnlyModel(TheModel); + return Handle.checkPrecondition(CppModel); + }); + } }); // Native diff --git a/include/revng/PipeboxCommon/LLVMContainer.h b/include/revng/PipeboxCommon/LLVMContainer.h index e7467e06d..e337c64e1 100644 --- a/include/revng/PipeboxCommon/LLVMContainer.h +++ b/include/revng/PipeboxCommon/LLVMContainer.h @@ -8,12 +8,10 @@ #include "llvm/Bitcode/BitcodeWriter.h" #include "llvm/IR/LLVMContext.h" #include "llvm/IR/Module.h" -#include "llvm/IR/Verifier.h" #include "revng/PipeboxCommon/Common.h" #include "revng/PipeboxCommon/ObjectID.h" #include "revng/Support/IRHelpers.h" -#include "revng/Support/ZstdStream.h" namespace revng::pypeline { @@ -21,6 +19,7 @@ class LLVMRootContainer { public: static constexpr llvm::StringRef Name = "LLVMRootContainer"; static constexpr Kind Kind = Kinds::Binary; + static constexpr llvm::StringRef MimeType = "application/x.llvm.bc"; private: llvm::LLVMContext Context; @@ -45,14 +44,9 @@ public: return; revng_assert(Data.size() == 1); - for (const auto &[Key, Value] : Data) { - revng_assert(Key->kind() == Kind); - - llvm::SmallVector DecompressedData = zstdDecompress(Value); - llvm::MemoryBufferRef Ref{ - { DecompressedData.data(), DecompressedData.size() }, "input" - }; - + for (const auto &[Object, Buffer] : Data) { + revng_assert(Object->kind() == Kind); + llvm::MemoryBufferRef Ref{ { Buffer.data(), Buffer.size() }, "input" }; Module = llvm::cantFail(llvm::parseBitcodeFile(Ref, Context)); } } @@ -64,10 +58,8 @@ public: revng_assert(Objects.size() == 1 and Objects[0]->kind() == Kind); std::map Result; - llvm::raw_svector_ostream OS(Result[*Objects[0]].data()); - ZstdCompressedOstream CompressedOS(OS, 3); - llvm::WriteBitcodeToFile(*Module, CompressedOS); - CompressedOS.flush(); + writeBitcode(*Module, Result[*Objects[0]].data()); + return Result; } @@ -79,6 +71,73 @@ public: public: const llvm::Module &getModule() const { return *Module; } llvm::Module &getModule() { return *Module; } + + void assign(std::unique_ptr &&NewModule) { + if (&Context == &NewModule->getContext()) + Module = std::move(NewModule); + else + Module = cloneIntoContext(*NewModule, Context); + } +}; + +class LLVMFunctionContainer { +public: + static constexpr llvm::StringRef Name = "LLVMFunctionContainer"; + static constexpr Kind Kind = Kinds::Function; + static constexpr llvm::StringRef MimeType = "application/x.llvm.bc"; + +private: + llvm::LLVMContext Context; + std::map> Modules; + +public: + LLVMFunctionContainer() {} + +public: + std::set objects() const { + return std::views::keys(Modules) | revng::to>(); + } + + void + deserialize(const std::map> Data) { + for (auto const &[Object, Buffer] : Data) { + llvm::MemoryBufferRef BufferRef({ Buffer.data(), Buffer.size() }, + "newBuffer"); + Modules[*Object] = llvm::cantFail(llvm::parseBitcodeFile(BufferRef, + Context)); + } + } + + std::map + serialize(const std::vector Objects) const { + std::map Result; + for (const ObjectID *Object : Objects) + writeBitcode(*Modules.at(*Object), Result[*Object].data()); + return Result; + } + + bool verify() const { + for (auto const &[_, Module] : Modules) + revng::forceVerify(&*Module); + return true; + } + +public: + llvm::LLVMContext &getContext() { return Context; } + const llvm::LLVMContext &getContext() const { return Context; } + + const llvm::Module &getModule(const ObjectID &ID) const { + return *Modules.at(ID); + } + + llvm::Module &getModule(const ObjectID &ID) { return *Modules.at(ID); } + + void assign(const ObjectID &ID, std::unique_ptr &&NewModule) { + if (&Context == &NewModule->getContext()) + Modules[ID] = std::move(NewModule); + else + Modules[ID] = cloneIntoContext(*NewModule, Context); + } }; } // namespace revng::pypeline diff --git a/include/revng/PipeboxCommon/RawContainer.h b/include/revng/PipeboxCommon/RawContainer.h index 5a8472494..271aaef9f 100644 --- a/include/revng/PipeboxCommon/RawContainer.h +++ b/include/revng/PipeboxCommon/RawContainer.h @@ -11,11 +11,12 @@ namespace revng::pypeline { namespace detail { -template +template class RawContainer { public: static constexpr llvm::StringRef Name = TheName; static constexpr Kind Kind = TheKind; + static constexpr llvm::StringRef MimeType = TheMime; private: std::map Map; @@ -49,7 +50,7 @@ public: public: bool contains(const ObjectID &Key) const { return Map.contains(Key); } - std::unique_ptr getOStream(const ObjectID &Key) { + std::unique_ptr getOStream(const ObjectID &Key) { revng_assert(Key.kind() == Kind); revng_assert(not Map.contains(Key)); return std::make_unique(Map[Key].data()); @@ -63,19 +64,20 @@ public: } }; -template -using RC = RawContainer; +template +using RC = RawContainer; + +constexpr auto TD = Kinds::TypeDefinition; } // namespace detail -using BytesContainer = detail::RC; +template +using BytesContainer = detail::RC; -using FunctionToBytesContainer = detail::RC; +template +using FunctionToBytesContainer = detail::RC; -using TypeDefinitionToBytesContainer = detail::RC; +template +using TypeDefinitionToBytesContainer = detail::RC; } // namespace revng::pypeline diff --git a/include/revng/Pipes/FunctionPass.h b/include/revng/Pipes/FunctionPass.h index 79f9f4c89..1214a0e7e 100644 --- a/include/revng/Pipes/FunctionPass.h +++ b/include/revng/Pipes/FunctionPass.h @@ -29,6 +29,7 @@ protected: public: FunctionPassImpl(llvm::ModulePass &Pass) : Pass(&Pass) {} + FunctionPassImpl() : Pass(nullptr){}; virtual ~FunctionPassImpl() = default; @@ -43,6 +44,7 @@ public: public: template T &getAnalysis(ArgType &&...Arg) { + revng_check(Pass != nullptr); return Pass->getAnalysis(std::forward(Arg)...); } }; diff --git a/include/revng/Recompile/CompileModulePipe.h b/include/revng/Recompile/CompileModulePipe.h index 4e6254481..f4e764dd0 100644 --- a/include/revng/Recompile/CompileModulePipe.h +++ b/include/revng/Recompile/CompileModulePipe.h @@ -8,6 +8,9 @@ #include "llvm/ADT/ArrayRef.h" +#include "revng/PipeboxCommon/LLVMContainer.h" +#include "revng/PipeboxCommon/Model.h" +#include "revng/PipeboxCommon/RawContainer.h" #include "revng/Pipeline/ContainerSet.h" #include "revng/Pipeline/Context.h" #include "revng/Pipeline/Contract.h" @@ -52,3 +55,30 @@ public: }; } // namespace revng::pipes + +namespace revng::pypeline { + +using ObjectFileContainer = BytesContainer<"ObjectFileContainer", + "application/x-object">; + +namespace piperuns { + +class CompileRootModule { +public: + static constexpr llvm::StringRef Name = "CompileRootModule"; + using Arguments = TypeList< + PipeArgument<"Input", + "The LLVM module that will be compiled", + Access::Read>, + PipeArgument<"Output", "The compiled object file", Access::Write>>; + + static void run(const Model &TheModel, + llvm::StringRef StaticConfig, + llvm::StringRef DynamicConfig, + LLVMRootContainer &Input, + ObjectFileContainer &Output); +}; + +} // namespace piperuns + +} // namespace revng::pypeline diff --git a/include/revng/Recompile/LinkForTranslationPipe.h b/include/revng/Recompile/LinkForTranslationPipe.h index b8bc0bae6..2c28a6f64 100644 --- a/include/revng/Recompile/LinkForTranslationPipe.h +++ b/include/revng/Recompile/LinkForTranslationPipe.h @@ -8,6 +8,9 @@ #include "llvm/ADT/ArrayRef.h" +#include "revng/PipeboxCommon/BinariesContainer.h" +#include "revng/PipeboxCommon/Model.h" +#include "revng/PipeboxCommon/RawContainer.h" #include "revng/Pipeline/ContainerSet.h" #include "revng/Pipeline/Context.h" #include "revng/Pipeline/Contract.h" @@ -16,6 +19,7 @@ #include "revng/Pipeline/Target.h" #include "revng/Pipes/FileContainer.h" #include "revng/Pipes/Kinds.h" +#include "revng/Recompile/CompileModulePipe.h" namespace revng::pipes { @@ -44,3 +48,32 @@ public: }; } // namespace revng::pipes + +namespace revng::pypeline { + +using TranslatedContainer = BytesContainer<"TranslatedContainer", + "application/x-executable">; + +namespace piperuns { + +class LinkForTranslation { +public: + static constexpr llvm::StringRef Name = "LinkForTranslation"; + using Arguments = TypeList< + PipeArgument<"Binaries", "The input binaries">, + PipeArgument<"ObjectFile", "The complied object file">, + PipeArgument<"Output", "The output executable", Access::Write>>; + + static llvm::Error checkPrecondition(const class Model &Model); + + static void run(const Model &TheModel, + llvm::StringRef StaticConfig, + llvm::StringRef DynamicConfig, + const BinariesContainer &Binaries, + const ObjectFileContainer &ObjectFile, + TranslatedContainer &Output); +}; + +} // namespace piperuns + +} // namespace revng::pypeline diff --git a/include/revng/Support/Error.h b/include/revng/Support/Error.h index 5c4480958..49baad730 100644 --- a/include/revng/Support/Error.h +++ b/include/revng/Support/Error.h @@ -33,16 +33,17 @@ inline T cantFail(llvm::ErrorOr Obj) { template T> inline llvm::Error joinErrors(T &Container) { auto Size = std::ranges::distance(Container); - revng_check(Size > 0); + if (Size == 0) + return llvm::Error::success(); auto Iter = Container.begin(); llvm::Error Result{ std::move(*Iter) }; - if (Size == 1) { + if (Size == 1) return Result; - } - for (Iter++; Iter < Container.end(); Iter++) { + + for (Iter++; Iter < Container.end(); Iter++) Result = llvm::joinErrors(std::move(Result), std::move(*Iter)); - } + return Result; } diff --git a/include/revng/Support/IRHelpers.h b/include/revng/Support/IRHelpers.h index de09f5bd7..eb5d05dbf 100644 --- a/include/revng/Support/IRHelpers.h +++ b/include/revng/Support/IRHelpers.h @@ -1546,3 +1546,18 @@ inline void linkModules(std::unique_ptr &&Source, llvm::Module &Destination) { linkModules(std::move(Source), Destination, std::nullopt); } + +/// Clone the module, keeping only the functions specified in \p ToClone . +/// This requires the source module to be writable in order to guarantee that +/// the metadata attached to the cloned functions is also preserved. +std::unique_ptr +cloneFiltered(llvm::Module &Module, std::set &ToClone); + +/// Serialize a module to bitcode, this is preferable over +/// llvm::WriteBitcodeToFile as it does not create a redundant temporary buffer +void writeBitcode(const llvm::Module &Module, + llvm::SmallVectorImpl &Output); + +/// Copy a module to a new LLVMContext. +std::unique_ptr cloneIntoContext(const llvm::Module &Module, + llvm::LLVMContext &NewContext); diff --git a/include/revng/Support/InitRevng.h b/include/revng/Support/InitRevng.h index 37cdaaff2..56e1042ef 100644 --- a/include/revng/Support/InitRevng.h +++ b/include/revng/Support/InitRevng.h @@ -19,14 +19,24 @@ namespace revng { /// By default this performs the regular LLVM initialization steps. /// This is required in order to initialize the stack trace printers on signal. class InitRevng : public llvm::InitLLVM { +private: + static inline bool Initialized = false; + public: + InitRevng(int &Argc, auto **&Argv, const char *Overview) : + InitRevng(Argc, Argv, Overview, {}) {} + InitRevng(int &Argc, auto **&Argv, const char *Overview, llvm::ArrayRef CategoriesToHide) : InitLLVM(Argc, Argv, true) { + revng_assert(not Initialized); + Initialized = true; + OnQuit->install(); + initializeLLVMLibraries(); llvm::setBugReportMsg("PLEASE submit a bug report to " "https://github.com/revng/revng and include the " @@ -63,6 +73,9 @@ public: } ~InitRevng() { OnQuit->quit(); } + +private: + void initializeLLVMLibraries(); }; } // namespace revng diff --git a/include/revng/Yield/Assembly/DisassemblyHelper.h b/include/revng/Yield/Assembly/DisassemblyHelper.h index 4ad897a1a..f994e2481 100644 --- a/include/revng/Yield/Assembly/DisassemblyHelper.h +++ b/include/revng/Yield/Assembly/DisassemblyHelper.h @@ -33,7 +33,18 @@ public: const efa::ControlFlowGraph &Metadata, const RawBinaryView &BinaryView, const model::Binary &Binary, - const model::AssemblyNameBuilder &NameBuilder); + const model::AssemblyNameBuilder &NameBuilder) { + yield::Function Result; + disassemble(Function, Metadata, BinaryView, Binary, NameBuilder, Result); + return Result; + } + + void disassemble(const model::Function &Function, + const efa::ControlFlowGraph &Metadata, + const RawBinaryView &BinaryView, + const model::Binary &Binary, + const model::AssemblyNameBuilder &NameBuilder, + yield::Function &ResultFunction); private: LLVMDisassemblerInterface & diff --git a/include/revng/Yield/HexDump.h b/include/revng/Yield/HexDump.h new file mode 100644 index 000000000..749e4ef8e --- /dev/null +++ b/include/revng/Yield/HexDump.h @@ -0,0 +1,44 @@ +#pragma once + +// +// This file is distributed under the MIT License. See LICENSE.md for details. +// + +#include "revng/EarlyFunctionAnalysis/CollectCFG.h" +#include "revng/PipeboxCommon/BinariesContainer.h" +#include "revng/PipeboxCommon/LLVMContainer.h" +#include "revng/PipeboxCommon/RawContainer.h" + +namespace revng::pypeline { + +using HexDumpContainer = BytesContainer<"HexDumpContainer", + "text/x.hexdump+ptml">; + +namespace piperuns { + +class HexDump { +public: + static constexpr llvm::StringRef Name = "HexDump"; + using Arguments = TypeList< + PipeArgument<"Binary", "The binaries to create the hexdump out of">, + PipeArgument<"Module", "The LLVM Module(s) with lifted functions">, + PipeArgument<"CFG", "The per-function CFG data">, + PipeArgument<"Output", "The hexdump of the input binaries", Access::Write>>; + +public: + static llvm::Error checkPrecondition(const class Model &Model) { + return RawBinaryView::checkPrecondition(*Model.get().get()); + } + + static void run(const class Model &Model, + llvm::StringRef Config, + llvm::StringRef DynamicConfig, + const BinariesContainer &BinaryContainer, + const LLVMFunctionContainer &ModuleContainer, + const CFGMap &CFG, + HexDumpContainer &Output); +}; + +} // namespace piperuns + +} // namespace revng::pypeline diff --git a/include/revng/Yield/Pipes/ProcessAssembly.h b/include/revng/Yield/Pipes/ProcessAssembly.h index 0dc51aa06..a5eeb2679 100644 --- a/include/revng/Yield/Pipes/ProcessAssembly.h +++ b/include/revng/Yield/Pipes/ProcessAssembly.h @@ -11,12 +11,18 @@ #include "llvm/Support/raw_ostream.h" #include "revng/EarlyFunctionAnalysis/CFGStringMap.h" +#include "revng/EarlyFunctionAnalysis/CollectCFG.h" +#include "revng/Pipebox/TupleTreeContainer.h" +#include "revng/PipeboxCommon/BinariesContainer.h" #include "revng/Pipeline/Contract.h" #include "revng/Pipeline/Target.h" #include "revng/Pipes/FileContainer.h" #include "revng/Pipes/StringMap.h" +#include "revng/Yield/Function.h" #include "revng/Yield/Pipes/YieldControlFlow.h" +class DissassemblyHelper; + namespace revng::pipes { class ProcessAssembly { @@ -44,3 +50,51 @@ public: }; } // namespace revng::pipes + +namespace revng::pypeline { + +using AssemblyInternalContainer = TupleTreeContainer; + +namespace piperuns { + +class ProcessAssembly { +private: + const model::Binary &Binary; + const CFGMap &CFG; + AssemblyInternalContainer &Output; + + std::unique_ptr Helper; + std::unique_ptr BinaryView; + model::AssemblyNameBuilder NameBuilder; + +public: + static constexpr llvm::StringRef Name = "ProcessAssembly"; + using Arguments = TypeList< + PipeRunArgument, + PipeRunArgument, + PipeRunArgument>; + + ProcessAssembly(const class Model &Model, + llvm::StringRef Config, + llvm::StringRef DynamicConfig, + const BinariesContainer &BinariesContainer, + const CFGMap &CFG, + AssemblyInternalContainer &Output); + ~ProcessAssembly(); + + static llvm::Error checkPrecondition(const class Model &Model) { + return RawBinaryView::checkPrecondition(*Model.get().get()); + } + + void runOnFunction(const model::Function &TheFunction); +}; + +} // namespace piperuns + +} // namespace revng::pypeline diff --git a/include/revng/Yield/Pipes/YieldAssembly.h b/include/revng/Yield/Pipes/YieldAssembly.h index 314d65e67..c397f885e 100644 --- a/include/revng/Yield/Pipes/YieldAssembly.h +++ b/include/revng/Yield/Pipes/YieldAssembly.h @@ -10,8 +10,11 @@ #include "llvm/ADT/ArrayRef.h" #include "llvm/Support/raw_ostream.h" +#include "revng/PTML/Tag.h" +#include "revng/PipeboxCommon/RawContainer.h" #include "revng/Pipeline/Contract.h" #include "revng/Pipes/StringMap.h" +#include "revng/Yield/Pipes/ProcessAssembly.h" #include "revng/Yield/Pipes/YieldControlFlow.h" namespace revng::pipes { @@ -36,3 +39,46 @@ public: }; } // namespace revng::pipes + +namespace revng::pypeline { + +using AssemblyContainer = FunctionToBytesContainer<"AssemblyContainer", + "text/x.asm+ptml">; + +namespace piperuns { + +class YieldAssembly { +private: + const model::Binary &Model; + const AssemblyInternalContainer &Input; + AssemblyContainer &Output; + + model::CNameBuilder NameBuilder; + ptml::MarkupBuilder B; + +public: + static constexpr llvm::StringRef Name = "YieldAssembly"; + using Arguments = TypeList, + PipeRunArgument>; + + YieldAssembly(const class Model &Model, + llvm::StringRef Config, + llvm::StringRef DynamicConfig, + const AssemblyInternalContainer &Input, + AssemblyContainer &Output) : + Model(*Model.get().get()), + Input(Input), + Output(Output), + NameBuilder(*Model.get().get()) {} + + void runOnFunction(const model::Function &TheFunction); +}; + +} // namespace piperuns + +} // namespace revng::pypeline diff --git a/lib/EarlyFunctionAnalysis/AttachDebugInfo.cpp b/lib/EarlyFunctionAnalysis/AttachDebugInfo.cpp index bc3080788..efe78d6d6 100644 --- a/lib/EarlyFunctionAnalysis/AttachDebugInfo.cpp +++ b/lib/EarlyFunctionAnalysis/AttachDebugInfo.cpp @@ -30,6 +30,7 @@ #include "llvm/IR/Metadata.h" #include "revng/BasicAnalyses/GeneratedCodeBasicInfo.h" +#include "revng/EarlyFunctionAnalysis/AttachDebugInfo.h" #include "revng/EarlyFunctionAnalysis/ControlFlowGraphCache.h" #include "revng/Model/FunctionTags.h" #include "revng/Model/LoadModelPass.h" @@ -47,16 +48,43 @@ using namespace llvm; static Logger Log("attach-debug-info"); class AttachDebugInfo : public pipeline::FunctionPassImpl { +private: + using CFG = efa::ControlFlowGraph; + using CFGGetterType = std::function; + private: llvm::Module &M; DIBuilder DIB; DICompileUnit *CU = nullptr; + GeneratedCodeBasicInfo &GCBI; + CFGGetterType CFGGetter; + public: AttachDebugInfo(llvm::ModulePass &Pass, const model::Binary &Binary, llvm::Module &M) : - pipeline::FunctionPassImpl(Pass), M(M), DIB(M) {} + pipeline::FunctionPassImpl(Pass), + M(M), + DIB(M), + GCBI(getAnalysis().getGCBI()) { + ControlFlowGraphCache &Cache = getAnalysis() + .get(); + CFGGetter = + [&Cache](const MetaAddress &Address) -> const efa::ControlFlowGraph & { + return Cache.getControlFlowGraph(Address); + }; + } + + AttachDebugInfo(const model::Binary &Binary, + llvm::Module &M, + GeneratedCodeBasicInfo &GCBI, + CFGGetterType CFGGetter) : + pipeline::FunctionPassImpl(), + M(M), + DIB(M), + GCBI(GCBI), + CFGGetter(CFGGetter) {} static void getAnalysisUsage(llvm::AnalysisUsage &AU) { AU.setPreservesAll(); @@ -231,10 +259,6 @@ bool AttachDebugInfo::prologue() { bool AttachDebugInfo::runOnFunction(const model::Function &ModelFunction, llvm::Function &F) { - ControlFlowGraphCache *Cache = &getAnalysis() - .get(); - auto &GCBI = getAnalysis().getGCBI(); - // Skip functions with debug-info. if (F.getSubprogram()) return true; @@ -242,7 +266,7 @@ bool AttachDebugInfo::runOnFunction(const model::Function &ModelFunction, // Skip declarations revng_assert(not F.isDeclaration()); - auto FM = Cache->getControlFlowGraph(&F); + auto FM = CFGGetter(ModelFunction.Entry()); revng_log(Log, "Metadata for Function " << F.getName() << ":" << FM.Entry().toString()); @@ -321,3 +345,36 @@ struct AttachDebugInfoToABIEnforcedPipe { }; static pipeline::RegisterPipe Y2; + +namespace revng::pypeline::piperuns { + +void AttachDebugInfo::runOnFunction(const model::Function &TheFunction) { + const MetaAddress &Address = TheFunction.Entry(); + llvm::Module &Module = ModuleContainer.getModule(ObjectID(Address)); + + GeneratedCodeBasicInfo GCBI(Binary); + GCBI.run(Module); + + auto CFGGetter = + [*this](const MetaAddress &Address) -> const efa::ControlFlowGraph & { + return *CFG.getElement(ObjectID(Address)); + }; + + llvm::Function *LLVMFunction = nullptr; + for (llvm::Function &Function : Module) { + if (FunctionTags::Isolated.isTagOf(&Function) + and not Function.isDeclaration()) { + revng_assert(LLVMFunction == nullptr); + LLVMFunction = &Function; + } + } + revng_assert(LLVMFunction != nullptr); + + // TODO: inline the body of prologue and epilogue here + ::AttachDebugInfo Impl(Binary, Module, GCBI, CFGGetter); + Impl.prologue(); + Impl.runOnFunction(TheFunction, *LLVMFunction); + Impl.epilogue(); +} + +} // namespace revng::pypeline::piperuns diff --git a/lib/EarlyFunctionAnalysis/CollectCFG.cpp b/lib/EarlyFunctionAnalysis/CollectCFG.cpp index 6ae14c765..64b294544 100644 --- a/lib/EarlyFunctionAnalysis/CollectCFG.cpp +++ b/lib/EarlyFunctionAnalysis/CollectCFG.cpp @@ -7,6 +7,7 @@ #include "revng/BasicAnalyses/GeneratedCodeBasicInfo.h" #include "revng/EarlyFunctionAnalysis/CFGAnalyzer.h" #include "revng/EarlyFunctionAnalysis/CFGStringMap.h" +#include "revng/EarlyFunctionAnalysis/CollectCFG.h" #include "revng/EarlyFunctionAnalysis/ControlFlowGraph.h" #include "revng/EarlyFunctionAnalysis/FunctionSummaryOracle.h" #include "revng/Model/Binary.h" @@ -89,3 +90,50 @@ static pipeline::RegisterPipe X; } // namespace revng::pipes static pipeline::RegisterDefaultConstructibleContainer X2; + +namespace revng::pypeline::piperuns { + +using FSO = efa::FunctionSummaryOracle; + +CollectCFG::CollectCFG(const class Model &Model, + llvm::StringRef Config, + llvm::StringRef DynamicConfig, + LLVMRootContainer &Input, + CFGMap &Output) : + Model(Model), + Output(Output), + GCBI(*Model.get().get()), + GCBIRun(GCBI, Input.getModule()), + Oracle(FSO::importBasicPrototypeData(Input.getModule(), + GCBI, + *Model.get().get())), + Analyzer(Input.getModule(), GCBI, Model.get(), Oracle) { +} + +void CollectCFG::runOnFunction(const model::Function &Function) { + MetaAddress EntryAddress = Function.Entry(); + const model::Binary &Binary = *Model.get().get(); + + // Recover the control-flow graph of the function + TupleTree New; + New->Entry() = EntryAddress; + New->Blocks() = std::move(Analyzer.analyze(EntryAddress).CFG); + + if (DebugNames) { + auto Function = Binary.Functions().at(EntryAddress); + New->Name() = Function.Name(); + } + + if (New->Blocks().size() > 0) + revng_assert(New->Blocks().contains(BasicBlockID(New->Entry()))); + + // Run final steps on the CFG + New->simplify(Binary); + + if (New->Blocks().size() > 0) + revng_assert(New->Blocks().contains(BasicBlockID(New->Entry()))); + + Output.getElement(ObjectID(EntryAddress)) = std::move(New); +} + +} // namespace revng::pypeline::piperuns diff --git a/lib/FunctionIsolation/EnforceABI.cpp b/lib/FunctionIsolation/EnforceABI.cpp index c4ca72d05..e63714148 100644 --- a/lib/FunctionIsolation/EnforceABI.cpp +++ b/lib/FunctionIsolation/EnforceABI.cpp @@ -26,6 +26,7 @@ #include "revng/BasicAnalyses/GeneratedCodeBasicInfo.h" #include "revng/EarlyFunctionAnalysis/CallEdge.h" #include "revng/EarlyFunctionAnalysis/ControlFlowGraphCache.h" +#include "revng/FunctionIsolation/EnforceABI.h" #include "revng/FunctionIsolation/StructInitializers.h" #include "revng/Model/FunctionTags.h" #include "revng/Model/IRHelpers.h" @@ -48,6 +49,8 @@ using namespace llvm; class EnforceABI final : public pipeline::FunctionPassImpl { private: using UsedRegisters = abi::FunctionType::UsedRegisters; + using CFG = efa::ControlFlowGraph; + using CFGGetterType = std::function; private: const model::Binary &Binary; @@ -56,8 +59,7 @@ private: std::vector OldFunctions; Function *FunctionDispatcher = nullptr; StructInitializers Initializers; - ControlFlowGraphCache &Cache; - pipeline::LoadExecutionContextPass &LECP; + CFGGetterType CFGGetter; GeneratedCodeBasicInfo &GCBI; public: @@ -68,9 +70,25 @@ public: Binary(Binary), M(M), Initializers(&M), - Cache(getAnalysis().get()), - LECP(getAnalysis()), - GCBI(getAnalysis().getGCBI()) {} + GCBI(getAnalysis().getGCBI()) { + ControlFlowGraphCache &Cache = getAnalysis() + .get(); + CFGGetter = + [&Cache](const MetaAddress &Address) -> const efa::ControlFlowGraph & { + return Cache.getControlFlowGraph(Address); + }; + } + + EnforceABI(const model::Binary &Binary, + llvm::Module &M, + GeneratedCodeBasicInfo &GCBI, + CFGGetterType CFGGetter) : + pipeline::FunctionPassImpl(), + Binary(Binary), + M(M), + Initializers(&M), + CFGGetter(CFGGetter), + GCBI(GCBI) {} ~EnforceABI() final = default; @@ -336,7 +354,8 @@ void EnforceABI::handleRegularFunctionCall(const MetaAddress &CallerAddress, // Identify the corresponding call site in the model Function *CallerFunction = Call->getParent()->getParent(); - const efa::ControlFlowGraph &FM = Cache.getControlFlowGraph(CallerFunction); + const efa::ControlFlowGraph + &FM = CFGGetter(getMetaAddressOfIsolatedFunction(*CallerFunction)); const efa::BasicBlock *CallerBlock = FM.findBlock(GCBI, Call->getParent()); revng_assert(CallerBlock != nullptr); @@ -539,3 +558,26 @@ struct EnforceABIPipe { }; static pipeline::RegisterPipe Y; + +namespace revng::pypeline::piperuns { + +void EnforceABI::runOnFunction(const model::Function &TheFunction) { + llvm::Module &Module = Output.getModule(ObjectID(TheFunction.Entry())); + llvm::Function *LLVMFunction = Module.getFunction(NameBuilder + .llvmName(TheFunction)); + + GeneratedCodeBasicInfo GCBI(Binary); + GCBI.run(Module); + + auto CFGGetter = + [this](const MetaAddress &Address) -> const efa::ControlFlowGraph & { + return *CFG.getElement(ObjectID(Address)); + }; + + ::EnforceABI Impl(Binary, Module, GCBI, CFGGetter); + Impl.prologue(); + Impl.runOnFunction(TheFunction, *LLVMFunction); + Impl.epilogue(); +} + +} // namespace revng::pypeline::piperuns diff --git a/lib/FunctionIsolation/InvokeIsolatedFunctions.cpp b/lib/FunctionIsolation/InvokeIsolatedFunctions.cpp index 9dd91fdc9..22330d1c5 100644 --- a/lib/FunctionIsolation/InvokeIsolatedFunctions.cpp +++ b/lib/FunctionIsolation/InvokeIsolatedFunctions.cpp @@ -14,6 +14,7 @@ #include "revng/ABI/FunctionType/Layout.h" #include "revng/BasicAnalyses/GeneratedCodeBasicInfo.h" #include "revng/EarlyFunctionAnalysis/ControlFlowGraphCache.h" +#include "revng/FunctionIsolation/InvokeIsolatedFunctions.h" #include "revng/Model/IRHelpers.h" #include "revng/Model/NameBuilder.h" #include "revng/Pipeline/AllRegistries.h" @@ -28,15 +29,14 @@ using namespace llvm; class InvokeIsolatedFunctionsImpl { private: - using FunctionInfo = tuple; + using FunctionInfo = tuple; using FunctionMap = std::map; private: const model::Binary &Binary; Function &RootFunction; Module &RootModule; + const Module *FunctionModule; LLVMContext &Context; GeneratedCodeBasicInfo GCBI; FunctionMap Map; @@ -44,10 +44,11 @@ private: public: InvokeIsolatedFunctionsImpl(const model::Binary &Binary, llvm::Module &RootModule, - const llvm::Module &FunctionModule) : + const llvm::Module *FunctionModule) : Binary(Binary), RootFunction(*RootModule.getFunction("root")), RootModule(RootModule), + FunctionModule(FunctionModule), Context(RootModule.getContext()), GCBI(Binary) { @@ -55,7 +56,7 @@ public: model::CNameBuilder NameBuilder = Binary; for (const model::Function &Function : Binary.Functions()) { - auto *F = FunctionModule.getFunction(NameBuilder.llvmName(Function)); + auto *F = FunctionModule->getFunction(NameBuilder.llvmName(Function)); revng_assert(F != nullptr); Map[Function.key()] = { &Function, nullptr, F }; } @@ -172,11 +173,15 @@ public: } } - llvm::Function * + llvm::Function *NewDeclaration = nullptr; + if (&RootModule != FunctionModule) { NewDeclaration = llvm::Function::Create(F->getFunctionType(), llvm::Function::ExternalLinkage, F->getName(), RootModule); + } else { + NewDeclaration = F; + } // Emit the invoke instruction, propagating debug info auto *NewInvoke = Builder.CreateInvoke(NewDeclaration, @@ -194,6 +199,56 @@ public: } }; +static void populateFunctionDispatcher(const model::Binary &Binary, + llvm::Module &Module) { + GeneratedCodeBasicInfo GCBI(Binary); + GCBI.run(Module); + + llvm::LLVMContext &Context = Module.getContext(); + llvm::Function *FunctionDispatcher = getIRHelper("function_dispatcher", + Module); + BasicBlock *Dispatcher = BasicBlock::Create(Context, + "function_dispatcher", + FunctionDispatcher, + nullptr); + + BasicBlock *Unexpected = BasicBlock::Create(Context, + "unexpectedpc", + FunctionDispatcher, + nullptr); + revng::NonDebugInfoCheckingIRBuilder UnreachableBuilder(Unexpected); + UnreachableBuilder.CreateUnreachable(); + setBlockType(Unexpected->getTerminator(), BlockType::UnexpectedPCBlock); + + // TODO: the checks should be enabled conditionally based on the user. + revng::NonDebugInfoCheckingIRBuilder Builder(Context); + + // Create all the entries of the dispatcher + ProgramCounterHandler::DispatcherTargets Targets; + for (llvm::Function &F : Module.functions()) { + if (not FunctionTags::Isolated.isTagOf(&F)) + continue; + + MetaAddress Address = getMetaAddressOfIsolatedFunction(F); + BasicBlock *Trampoline = BasicBlock::Create(Context, + F.getName() + "_trampoline", + FunctionDispatcher, + nullptr); + Targets.emplace_back(Address, Trampoline); + + Builder.SetInsertPoint(Trampoline); + Builder.CreateCall(&F); + Builder.CreateRetVoid(); + } + + // Create switch + Builder.SetInsertPoint(Dispatcher); + GCBI.programCounterHandler()->buildDispatcher(Targets, + Builder, + Unexpected, + {}); +} + struct InvokeIsolatedPipe { static constexpr auto Name = "invoke-isolated-functions"; @@ -220,9 +275,11 @@ public: // Clone the container OutputRootContainer.cloneFrom(InputRootContainer); + populateFunctionDispatcher(*revng::getModelFromContext(EC), + FunctionContainer.getModule()); InvokeIsolatedFunctionsImpl Impl(*revng::getModelFromContext(EC), OutputRootContainer.getModule(), - FunctionContainer.getModule()); + &FunctionContainer.getModule()); Impl.run(); const llvm::Module &FunctionModule = FunctionContainer.getModule(); @@ -235,3 +292,35 @@ public: }; static pipeline::RegisterPipe Y; + +namespace revng::pypeline::piperuns { + +void InvokeIsolatedFunctions::run(const class Model &Model, + llvm::StringRef Config, + llvm::StringRef DynamicConfig, + const LLVMRootContainer &Root, + const LLVMFunctionContainer &Functions, + LLVMRootContainer &Output) { + const model::Binary &Binary = *Model.get().get(); + + // Clone the container + llvm::LLVMContext &OutputContext = Output.getModule().getContext(); + std::unique_ptr + OutputModule = cloneIntoContext(Root.getModule(), OutputContext); + + // Merge all the functions into a single Module + llvm::Linker Linker(*OutputModule); + for (auto &Object : Functions.objects()) { + const llvm::Module &Module = Functions.getModule(Object); + Linker.linkInModule(cloneIntoContext(Module, OutputContext), + llvm::Linker::OverrideFromSrc); + } + + populateFunctionDispatcher(Binary, *OutputModule); + InvokeIsolatedFunctionsImpl Impl(Binary, *OutputModule, OutputModule.get()); + Impl.run(); + + Output.assign(std::move(OutputModule)); +} + +} // namespace revng::pypeline::piperuns diff --git a/lib/FunctionIsolation/IsolateFunctions.cpp b/lib/FunctionIsolation/IsolateFunctions.cpp index db5cb25ed..4cf8e7a45 100644 --- a/lib/FunctionIsolation/IsolateFunctions.cpp +++ b/lib/FunctionIsolation/IsolateFunctions.cpp @@ -194,6 +194,8 @@ public: class IsolateFunctionsImpl { private: using SuccessorsContainer = std::map; + using CFG = efa::ControlFlowGraph; + using CFGGetterType = std::function; private: Function *RootFunction = nullptr; @@ -206,24 +208,21 @@ private: std::map IsolatedFunctionsMap; std::map DynamicFunctionsMap; - ControlFlowGraphCache *Cache = nullptr; + CFGGetterType CFGGetter; FunctionType *IsolatedFunctionType = nullptr; - pipeline::LoadExecutionContextPass &LECP; public: IsolateFunctionsImpl(Function *RootFunction, GeneratedCodeBasicInfo &GCBI, const model::Binary &Binary, - ControlFlowGraphCache &Cache, - pipeline::LoadExecutionContextPass &LECP) : + CFGGetterType CFGGetter) : RootFunction(RootFunction), TheModule(RootFunction->getParent()), Context(TheModule->getContext()), GCBI(GCBI), Binary(Binary), NameBuilder(Binary), - Cache(&Cache), - LECP(LECP) { + CFGGetter(CFGGetter) { IsolatedFunctionType = createFunctionType(Context); } @@ -250,7 +249,9 @@ public: auto &gcbi() const { return GCBI; } public: - void run(); + void prologue(); + llvm::Function *runOnFunction(const MetaAddress &Address); + void epilogue(); void emitAbort(revng::IRBuilder &Builder, const Twine &Reason, @@ -280,54 +281,11 @@ public: } private: - /// Populate the function_dispatcher, needed to handle the indirect calls - void populateFunctionDispatcher(); - void handleUnexpectedPCCloned(efa::OutlinedFunction &Outlined); void handleAnyPCJumps(efa::OutlinedFunction &Outlined, const efa::ControlFlowGraph &FM); }; -void IFI::populateFunctionDispatcher() { - - BasicBlock *Dispatcher = BasicBlock::Create(Context, - "function_dispatcher", - FunctionDispatcher, - nullptr); - - BasicBlock *Unexpected = BasicBlock::Create(Context, - "unexpectedpc", - FunctionDispatcher, - nullptr); - const DebugLoc &Dbg = GCBI.unexpectedPC()->getTerminator()->getDebugLoc(); - emitUnreachable(Unexpected, "An unexpected function has been called", Dbg); - setBlockType(Unexpected->getTerminator(), BlockType::UnexpectedPCBlock); - - // TODO: the checks should be enabled conditionally based on the user. - revng::NonDebugInfoCheckingIRBuilder Builder(Context); - - // Create all the entries of the dispatcher - ProgramCounterHandler::DispatcherTargets Targets; - for (auto &[Address, F] : IsolatedFunctionsMap) { - BasicBlock *Trampoline = BasicBlock::Create(Context, - F->getName() + "_trampoline", - FunctionDispatcher, - nullptr); - Targets.emplace_back(Address, Trampoline); - - Builder.SetInsertPoint(Trampoline); - Builder.CreateCall(F); - Builder.CreateRetVoid(); - } - - // Create switch - Builder.SetInsertPoint(Dispatcher); - GCBI.programCounterHandler()->buildDispatcher(Targets, - Builder, - Unexpected, - {}); -} - template static bool allOrNone(const T &Range, const F &Predicate, bool Default = false) { @@ -533,9 +491,7 @@ public: } }; -void IsolateFunctionsImpl::run() { - Task T(6, "IsolateFunctions"); - +void IsolateFunctionsImpl::prologue() { auto SimpleFunctionType = createFunctionType(Context); FunctionDispatcher = createIRHelper("function_dispatcher", *TheModule, @@ -548,7 +504,6 @@ void IsolateFunctionsImpl::run() { // // TODO: we can (and should) push processing of dynamic functions into the // loop emitting individual local functions, and make it lazy - T.advance("Create dynamic functions", true); Task DynamicFunctionsTask(Binary.ImportedDynamicFunctions().size(), "Dynamic functions creation"); for (const model::DynamicFunction &Function : @@ -585,66 +540,46 @@ void IsolateFunctionsImpl::run() { DynamicFunctionsMap[Name] = NewFunction; } +} - using namespace efa; - using llvm::BasicBlock; +llvm::Function *IsolateFunctionsImpl::runOnFunction(const MetaAddress &Entry) { - T.advance("Create dynamic functions", true); + revng_assert(Entry.isValid()); + const efa::ControlFlowGraph &FM = CFGGetter(Entry); - // Obtain the set of requested targets - pipeline::ExecutionContext &Context = *LECP.get(); - const pipeline::TargetsList &RequestedTargets = LECP.getRequestedTargets(); + // Get or create the llvm::Function + Function *F = getLocalFunction(Entry); - Task IsolateTask(RequestedTargets.size(), "Isolating functions"); - for (const pipeline::Target &Target : RequestedTargets) { - IsolateTask.advance(Target.toString(), true); - Context.getContext().pushReadFields(); + // Decorate the function as appropriate + F->addFnAttr(Attribute::NullPointerIsValid); + F->addFnAttr(Attribute::NoMerge); + IsolatedFunctionsMap[Entry] = F; + revng_assert(F != nullptr); - auto Entry = MetaAddress::fromString(Target.getPathComponents()[0]); - revng_assert(Entry.isValid()); - const efa::ControlFlowGraph &FM = Cache->getControlFlowGraph(Entry); + // Outline the function (later on we'll steal its body and move it into F) + CallIsolatedFunction CallHandler(*this, FM); + FunctionOutliner Outliner(*TheModule, Binary, GCBI); + efa::OutlinedFunction Outlined = Outliner.outline(Entry, &CallHandler); - // Get or create the llvm::Function - Function *F = getLocalFunction(Entry); + handleUnexpectedPCCloned(Outlined); - // Decorate the function as appropriate - F->addFnAttr(Attribute::NullPointerIsValid); - F->addFnAttr(Attribute::NoMerge); - IsolatedFunctionsMap[Entry] = F; - revng_assert(F != nullptr); + handleAnyPCJumps(Outlined, FM); - // Outline the function (later on we'll steal its body and move it into F) - CallIsolatedFunction CallHandler(*this, FM); - FunctionOutliner Outliner(*TheModule, Binary, GCBI); - OutlinedFunction Outlined = Outliner.outline(Entry, &CallHandler); + if (Outlined.Function) + for (BasicBlock &BB : *Outlined.Function) + revng_assert(BB.getTerminator() != nullptr); - handleUnexpectedPCCloned(Outlined); + // Steal the function body and let the outlined function be destroyed + moveBlocksInto(*Outlined.Function, *F); - handleAnyPCJumps(Outlined, FM); - - if (Outlined.Function) - for (BasicBlock &BB : *Outlined.Function) - revng_assert(BB.getTerminator() != nullptr); - - // Steal the function body and let the outlined function be destroyed - moveBlocksInto(*Outlined.Function, *F); - - // Commit the produced target - Context.commit(Target, LECP.getContainerName()); - - Context.getContext().popReadFields(); - } + return F; +} +void IsolateFunctionsImpl::epilogue() { + llvm::Task T(3, "Isolate: epilogue"); T.advance("Verify module", true); revng::verify(TheModule); - // Create the functions and basic blocks needed for the correct execution of - // the exception handling mechanism - - // Populate the function_dispatcher - T.advance("Populate function_dispatcher", true); - populateFunctionDispatcher(); - // Cleanup root T.advance("Cleanup", true); EliminateUnreachableBlocks(*RootFunction, nullptr, false); @@ -753,17 +688,100 @@ bool IF::runOnModule(Module &TheModule) { const auto &ModelWrapper = getAnalysis().get(); const model::Binary &Binary = *ModelWrapper.getReadOnlyModel(); + auto &LECP = getAnalysis(); + pipeline::ExecutionContext &Context = *LECP.get(); + const pipeline::TargetsList &RequestedTargets = LECP.getRequestedTargets(); + + auto &CFGC = getAnalysis().get(); + auto CFGGetter = + [&CFGC](const MetaAddress &Address) -> const efa::ControlFlowGraph & { + return CFGC.getControlFlowGraph(Address); + }; + + llvm::Task MainTask(3, "Isolate functions"); + MainTask.advance("Isolate: prologue"); + // Create an object of type IsolateFunctionsImpl and run the pass - IFI Impl(TheModule.getFunction("root"), - GCBI, - Binary, - getAnalysis().get(), - getAnalysis()); - Impl.run(); + IFI Impl(TheModule.getFunction("root"), GCBI, Binary, CFGGetter); + + Impl.prologue(); + + MainTask.advance("Isolate: run on functions"); + Task IsolateTask(RequestedTargets.size(), "Isolating functions"); + for (const pipeline::Target &Target : RequestedTargets) { + Context.getContext().pushReadFields(); + + auto Entry = MetaAddress::fromString(Target.getPathComponents()[0]); + IsolateTask.advance(Entry.toString(), true); + Impl.runOnFunction(Entry); + + // Commit the produced target + Context.commit(Target, LECP.getContainerName()); + Context.getContext().popReadFields(); + } + + MainTask.advance("Isolate: epilogue"); + Impl.epilogue(); return false; } +namespace revng::pypeline::piperuns { + +Isolate::Isolate(const class Model &Model, + llvm::StringRef Config, + llvm::StringRef DynamicConfig, + const CFGMap &CFG, + LLVMRootContainer &Root, + LLVMFunctionContainer &Output) : + Root(Root), Output(Output), GCBI(*Model.get().get()) { + llvm::Module &Module = Root.getModule(); + GCBI.run(Module); + + auto CFGGetter = + [&CFG](const MetaAddress &Address) -> const efa::ControlFlowGraph & { + return *CFG.getElement(ObjectID(Address)); + }; + + // TODO: inline Impl + Impl = std::make_unique(Module.getFunction("root"), + GCBI, + *Model.get().get(), + CFGGetter); + Impl->prologue(); +} + +void Isolate::runOnFunction(const model::Function &TheFunction) { + llvm::Function *Function = Impl->runOnFunction(TheFunction.Entry()); + IsolatedFunctions.push_back({ TheFunction.Entry(), Function }); +} + +Isolate::~Isolate() { + Impl->epilogue(); + + auto ClonedModule = cloneIntoContext(Root.getModule(), Output.getContext()); + std::set ExternalFunctions; + for (llvm::Function &ModuleFunction : ClonedModule->functions()) { + if (not FunctionTags::Root.isTagOf(&ModuleFunction) + and not FunctionTags::Isolated.isTagOf(&ModuleFunction)) { + ExternalFunctions.insert(&ModuleFunction); + } + } + + llvm::Task T(IsolatedFunctions.size(), + "Splitting functions into individual modules"); + for (auto &[Address, Function] : IsolatedFunctions) { + T.advance(Address.toString(), true); + std::set ToClone; + ToClone.insert(ClonedModule->getFunction(Function->getName())); + ToClone.insert(ExternalFunctions.begin(), ExternalFunctions.end()); + + Output.assign(ObjectID(Address), ::cloneFiltered(*ClonedModule, ToClone)); + } +} + +} // namespace revng::pypeline::piperuns + void IsolateFunctions::getAnalysisUsage(llvm::AnalysisUsage &AU) const { AU.setPreservesAll(); AU.addRequired(); diff --git a/lib/FunctionIsolation/PromoteCSVs.cpp b/lib/FunctionIsolation/PromoteCSVs.cpp index 121358173..6c55eb939 100644 --- a/lib/FunctionIsolation/PromoteCSVs.cpp +++ b/lib/FunctionIsolation/PromoteCSVs.cpp @@ -66,12 +66,17 @@ private: SetVector CSVs; model::Architecture::Values Architecture; const model::NamingConfiguration &Configuration; + GeneratedCodeBasicInfo &GCBI; + llvm::Module &Module; public: - PromoteCSVs(ModulePass &Pass, const model::Binary &Binary, Module &M); + PromoteCSVs(ModulePass &Pass, const model::Binary &Binary, llvm::Module &M); + PromoteCSVs(const model::Binary &Binary, + llvm::Module &M, + GeneratedCodeBasicInfo &GCBI); public: - bool prologue() final { return false; } + bool prologue(); bool runOnFunction(const model::Function &ModelFunction, llvm::Function &Function) final; @@ -98,20 +103,35 @@ private: PromoteCSVs::PromoteCSVs(ModulePass &Pass, const model::Binary &Binary, - Module &M) : + llvm::Module &M) : pipeline::FunctionPassImpl(Pass), Initializers(&M), CSVInitializers(&M, false), Architecture(Binary.Architecture()), - Configuration(Binary.Configuration().Naming()) { + Configuration(Binary.Configuration().Naming()), + GCBI(getAnalysis().getGCBI()), + Module(M) { +} +PromoteCSVs::PromoteCSVs(const model::Binary &Binary, + llvm::Module &M, + GeneratedCodeBasicInfo &GCBI) : + pipeline::FunctionPassImpl(), + Initializers(&M), + CSVInitializers(&M, false), + Architecture(Binary.Architecture()), + Configuration(Binary.Configuration().Naming()), + GCBI(GCBI), + Module(M) { +} + +bool PromoteCSVs::prologue() { CSVInitializers.setMemoryEffects(MemoryEffects::readOnly()); CSVInitializers.addFnAttribute(Attribute::NoUnwind); CSVInitializers.addFnAttribute(Attribute::WillReturn); CSVInitializers.setTags({ &FunctionTags::OpaqueCSVValue }); // Record existing initializers - auto &GCBI = getAnalysis().getGCBI(); const auto &PCCSVs = GCBI.programCounterHandler()->pcCSVs(); const auto &R = llvm::concat(GCBI.csvs(), PCCSVs); SmallVector CSVsToSort{ R.begin(), R.end() }; @@ -121,11 +141,13 @@ PromoteCSVs::PromoteCSVs(ModulePass &Pass, continue; CSVs.insert(CSV); - if (auto *F = M.getFunction(Configuration.OpaqueCSVValuePrefix() - + CSV->getName().str())) + if (auto *F = Module.getFunction(Configuration.OpaqueCSVValuePrefix() + + CSV->getName().str())) if (FunctionTags::OpaqueCSVValue.isTagOf(F)) CSVInitializers.record(CSV->getName(), F); } + + return false; } // TODO: assign alias information @@ -610,3 +632,29 @@ struct PromoteCSVsPipe { }; static pipeline::RegisterLLVMPass Y; + +namespace revng::pypeline::piperuns { + +void PromoteCSVs::runOnFunction(const model::Function &TheFunction) { + llvm::Module &Module = ModuleContainer + .getModule(ObjectID(TheFunction.Entry())); + GeneratedCodeBasicInfo GCBI(Binary); + GCBI.run(Module); + + ::PromoteCSVs Impl(Binary, Module, GCBI); + Impl.prologue(); + + llvm::Function *LLVMFunction = nullptr; + for (llvm::Function &Function : Module.functions()) { + if (FunctionTags::Isolated.isTagOf(&Function) + and not Function.isDeclaration()) { + revng_assert(LLVMFunction == nullptr); + LLVMFunction = &Function; + } + } + + Impl.runOnFunction(TheFunction, *LLVMFunction); + Impl.epilogue(); +} + +} // namespace revng::pypeline::piperuns diff --git a/lib/HeadersGeneration/ModelToHeaderPipe.cpp b/lib/HeadersGeneration/ModelToHeaderPipe.cpp index d6f2b848a..f9a679cfd 100644 --- a/lib/HeadersGeneration/ModelToHeaderPipe.cpp +++ b/lib/HeadersGeneration/ModelToHeaderPipe.cpp @@ -3,6 +3,7 @@ // #include "revng/HeadersGeneration/ConfigurationHelpers.h" +#include "revng/HeadersGeneration/ModelToHeaderPipe.h" #include "revng/HeadersGeneration/Options.h" #include "revng/HeadersGeneration/PTMLHeaderBuilder.h" #include "revng/Pipeline/AllRegistries.h" @@ -76,3 +77,24 @@ static pipeline::RegisterDefaultConstructibleContainer } // namespace revng::pipes static pipeline::RegisterPipe Y; + +namespace revng::pypeline::piperuns { + +void ModelToHeader::run(const Model &TheModel, + llvm::StringRef StaticConfig, + llvm::StringRef DynamicConfig, + CBytesContainer &Buffer) { + const model::Binary &Binary = *TheModel.get().get(); + std::unique_ptr Out = Buffer.getOStream(ObjectID()); + ptml::ModelCBuilder + B(*Out, + Binary, + /* EnableTaglessMode = */ false, + { .EnableStackFrameInlining = revng::options::EnableStackFrameInlining, + .EnablePrintingOfTheMaximumEnumValue = true, + .ExplicitTargetPointerSize = getExplicitPointerSize(Binary) }); + ptml::HeaderBuilder(B).printModelHeader(); + Out->flush(); +} + +} // namespace revng::pypeline::piperuns diff --git a/lib/HeadersGeneration/ModelTypeDefinitionPipe.cpp b/lib/HeadersGeneration/ModelTypeDefinitionPipe.cpp index 449a68495..dfabbba7d 100644 --- a/lib/HeadersGeneration/ModelTypeDefinitionPipe.cpp +++ b/lib/HeadersGeneration/ModelTypeDefinitionPipe.cpp @@ -3,6 +3,7 @@ // #include "revng/Backend/DecompileFunction.h" +#include "revng/HeadersGeneration/ModelTypeDefinitionPipe.h" #include "revng/HeadersGeneration/PTMLHeaderBuilder.h" #include "revng/Model/Binary.h" #include "revng/Pipeline/AllRegistries.h" @@ -60,4 +61,22 @@ static RegisterDefaultConstructibleContainer F2; } // end namespace revng::pipes +namespace revng::pypeline::piperuns { + +using TD = UpcastablePointer; +void GenerateModelTypeDefinition::runOnTypeDefinition(const TD + &TypeDefinition) { + auto OS = Output.getOStream(ObjectID(TypeDefinition->key())); + ptml::ModelCBuilder B(*OS, + *Model.get().get(), + true, + { .EnablePrintingOfTheMaximumEnumValue = true, + .EnableExplicitPadding = false }); + + upcast(TypeDefinition, + [&B](const T &Upcasted) { B.printDefinition(Upcasted); }); +} + +} // namespace revng::pypeline::piperuns + static pipeline::RegisterPipe X2; diff --git a/lib/Lift/Lift.cpp b/lib/Lift/Lift.cpp index 851777288..791f45c27 100644 --- a/lib/Lift/Lift.cpp +++ b/lib/Lift/Lift.cpp @@ -11,6 +11,7 @@ #include "revng/Support/ResourceFinder.h" #include "CodeGenerator.h" +#include "PostLiftVerifyPass.h" using namespace llvm::cl; @@ -100,3 +101,61 @@ bool LiftPass::runOnModule(llvm::Module &M) { return false; } + +namespace revng::pypeline::piperuns { + +void Lift::run(const class Model &TheModel, + llvm::StringRef Config, + llvm::StringRef DynamicConfig, + const BinariesContainer &Binary, + LLVMRootContainer &ModuleContainer) { + llvm::Task T(6, "Lift"); + const TupleTree &Model = TheModel.get(); + + T.advance("findFiles", false); + const auto Paths = findExternalFilePaths(Model->Architecture()); + + // Look for the library in the system's paths + T.advance("Load libtcg", false); + auto TheLibTcg = LibTcg::get(Model->Architecture()); + + // Get access to raw binary data + revng_assert(Binary.size() == 1); + llvm::ArrayRef File = Binary.getFile(0); + RawBinaryView RawBinary(*Model, { File.data(), File.size() }); + llvm::Module &Module = ModuleContainer.getModule(); + + T.advance("Construct CodeGenerator", false); + CodeGenerator Generator(RawBinary, + &Module, + Model, + Paths.LibHelpers, + Paths.EarlyLinked, + model::Architecture::x86_64); + + std::optional EntryPointAddressOptional; + if (EntryPointAddress.getNumOccurrences() != 0) + EntryPointAddressOptional = EntryPointAddress; + T.advance("Translate", true); + + Generator.translate(TheLibTcg, EntryPointAddressOptional); + + T.advance("Sort Module", true); + sortModule(Module); + + T.advance("Verify Module", true); + // TODO: convert this from a pass to a free-standing function + PostLiftVerifyPass{}.runOnModule(Module); + + // TODO: substitute with strip-dead-debug-info once the old pipeline + // is dropped + pruneDICompileUnits(Module); +} + +llvm::Error Lift::checkPrecondition(const class Model &Model) { + const model::Binary &Binary = *Model.get().get(); + return revng::joinErrors(::detail::liftCheckPrecondition(Binary), + RawBinaryView::checkPrecondition(Binary)); +} + +} // namespace revng::pypeline::piperuns diff --git a/lib/Lift/LiftPipe.cpp b/lib/Lift/LiftPipe.cpp index 477ef2a3e..77c85e34e 100644 --- a/lib/Lift/LiftPipe.cpp +++ b/lib/Lift/LiftPipe.cpp @@ -132,9 +132,9 @@ Lift::invalidate(const BinaryFileContainer &SourceBinary, return {}; } -llvm::Error Lift::checkPrecondition(const pipeline::Context &Context) const { - const auto &Model = *getModelFromContext(Context); +namespace detail { +llvm::Error liftCheckPrecondition(const model::Binary &Model) { if (Model.Architecture() == model::Architecture::Invalid) { return revng::createError("Cannot lift binary with architecture invalid."); } @@ -148,6 +148,13 @@ llvm::Error Lift::checkPrecondition(const pipeline::Context &Context) const { return llvm::Error::success(); } +} // namespace detail + +llvm::Error Lift::checkPrecondition(const pipeline::Context &Context) const { + const auto &Model = *getModelFromContext(Context); + return ::detail::liftCheckPrecondition(Model); +} + static_assert(pipeline::HasInvalidate); static RegisterPipe E1; diff --git a/lib/Lift/LinkSupportPipe.cpp b/lib/Lift/LinkSupportPipe.cpp index 8db571208..742048703 100644 --- a/lib/Lift/LinkSupportPipe.cpp +++ b/lib/Lift/LinkSupportPipe.cpp @@ -59,11 +59,10 @@ static llvm::StringRef getSupportName(model::Architecture::Values V) { return "Invalid"; } -static std::string getSupportPath(const Context &Context) { - const auto &Model = getModelFromContext(Context); +static std::string getSupportPath(const model::Binary &Model) { const char *SupportConfig = Tracing ? "trace" : "normal"; - auto ArchName = getSupportName(Model->Architecture()).str(); + auto ArchName = getSupportName(Model.Architecture()).str(); std::string SupportSearchPath = ("/share/revng/support-" + ArchName + "-" + SupportConfig + ".ll"); @@ -80,7 +79,7 @@ void revng::pipes::LinkSupport::run(ExecutionContext &EC, if (ModuleContainer.enumerate().empty()) return; - std::string SupportPath = getSupportPath(EC.getContext()); + std::string SupportPath = getSupportPath(*getModelFromContext(EC)); llvm::SMDiagnostic Err; auto Module = llvm::parseIRFile(SupportPath, @@ -97,3 +96,23 @@ void revng::pipes::LinkSupport::run(ExecutionContext &EC, } static pipeline::RegisterPipe E4; + +namespace revng::pypeline::piperuns { + +void LinkSupport::run(const class Model &Model, + llvm::StringRef Config, + llvm::StringRef DynamicConfig, + LLVMRootContainer &ModuleContainer) { + std::string SupportPath = getSupportPath(*Model.get().get()); + + llvm::SMDiagnostic Err; + llvm::LLVMContext &Context = ModuleContainer.getModule().getContext(); + auto Module = llvm::parseIRFile(SupportPath, Err, Context); + revng_assert(Module != nullptr); + + bool Failed = llvm::Linker::linkModules(ModuleContainer.getModule(), + std::move(Module)); + revng_assert(not Failed); +} + +} // namespace revng::pypeline::piperuns diff --git a/lib/Model/RawBinaryView.cpp b/lib/Model/RawBinaryView.cpp index 435c8dad1..4586bb14b 100644 --- a/lib/Model/RawBinaryView.cpp +++ b/lib/Model/RawBinaryView.cpp @@ -190,3 +190,19 @@ RawBinaryView::findOffsetInSegment(MetaAddress Address, uint64_t Size) const { return { nullptr, 0 }; } + +llvm::Error RawBinaryView::checkPrecondition(const model::Binary &Binary) { + llvm::SmallVector Errors; + if (Binary.Binaries().size() != 1) + Errors.push_back(revng::createError("Binaries should contain exactly 1 " + "entry")); + + for (const model::Segment &Segment : Binary.Segments()) { + if (not Segment.Binary().isValid()) + Errors.push_back(revng::createError("Segment " + toString(Segment.key()) + + " does not reference a valid " + "entry in Binaries")); + } + + return revng::joinErrors(Errors); +} diff --git a/lib/Pipebox/CMakeLists.txt b/lib/Pipebox/CMakeLists.txt index 61ff1037c..d82adedb3 100644 --- a/lib/Pipebox/CMakeLists.txt +++ b/lib/Pipebox/CMakeLists.txt @@ -2,10 +2,20 @@ # This file is distributed under the MIT License. See LICENSE.md for details. # -revng_add_library_internal(revngPipebox SHARED Pipebox.cpp) +revng_add_library_internal(revngPipebox SHARED Pipebox.cpp LLVMPipe.cpp) llvm_map_components_to_libnames(LLVM_LIBRARIES Support Core) -target_link_libraries(revngPipebox ${Python_LIBRARIES} ${LLVM_LIBRARIES} - revngModelToHeader nanobind revngSupport) +target_link_libraries( + revngPipebox + ${Python_LIBRARIES} + ${LLVM_LIBRARIES} + revngModelToHeader + revngLift + revngFunctionIsolation + revngRecompile + revngYield + revngYieldPipes + nanobind + revngSupport) target_include_directories(revngPipebox PUBLIC ${Python_INCLUDE_DIRS}) diff --git a/lib/Pipebox/LLVMPipe.cpp b/lib/Pipebox/LLVMPipe.cpp new file mode 100644 index 000000000..7e61a94ba --- /dev/null +++ b/lib/Pipebox/LLVMPipe.cpp @@ -0,0 +1,43 @@ +// +// This file is distributed under the MIT License. See LICENSE.md for details. +// + +#include "llvm/PassRegistry.h" + +#include "revng/Pipebox/LLVMPipe.h" + +namespace rpp = revng::pypeline::pipes; +using Configuration = rpp::detail::PureLLVMPassesPipeBase::Configuration; + +template<> +struct llvm::yaml::MappingTraits { + static void mapping(IO &IO, Configuration &Fields) { + IO.mapRequired("Passes", Fields.Passes); + } +}; + +Configuration Configuration::parse(llvm::StringRef Input) { + return llvm::cantFail(fromString(Input)); +} + +namespace revng::pypeline::pipes::detail { + +PureLLVMPassesPipeBase::PureLLVMPassesPipeBase(llvm::StringRef + StaticConfiguration) : + StaticConfiguration(StaticConfiguration) { + + Configuration Configuration = Configuration::parse(StaticConfiguration); + llvm::PassRegistry &Registry = *llvm::PassRegistry::getPassRegistry(); + for (llvm::StringRef PassName : Configuration.Passes) { + const llvm::PassInfo *PassInfo = Registry.getPassInfo(PassName); + revng_assert(PassInfo != nullptr, + ("llvm-pipe: Requested pass " + PassName + + " was not found in the PassRegistry") + .str() + .c_str()); + PassInfos.push_back(PassInfo); + } + TaskName = "PureLLVMPasses(" + llvm::join(Configuration.Passes, ",") + ")"; +} + +} // namespace revng::pypeline::pipes::detail diff --git a/lib/Pipebox/Pipebox.cpp b/lib/Pipebox/Pipebox.cpp index f5567782f..b4612f9ca 100644 --- a/lib/Pipebox/Pipebox.cpp +++ b/lib/Pipebox/Pipebox.cpp @@ -2,9 +2,25 @@ // This file is distributed under the MIT License. See LICENSE.md for details. // +#include "revng/EarlyFunctionAnalysis/AttachDebugInfo.h" +#include "revng/EarlyFunctionAnalysis/CollectCFG.h" +#include "revng/FunctionIsolation/EnforceABI.h" +#include "revng/FunctionIsolation/InvokeIsolatedFunctions.h" +#include "revng/FunctionIsolation/IsolateFunctions.h" +#include "revng/FunctionIsolation/PromoteCSVs.h" +#include "revng/HeadersGeneration/ModelToHeaderPipe.h" +#include "revng/HeadersGeneration/ModelTypeDefinitionPipe.h" +#include "revng/Lift/Lift.h" +#include "revng/Lift/LinkSupportPipe.h" +#include "revng/Pipebox/LLVMPipe.h" #include "revng/PipeboxCommon/BinariesContainer.h" #include "revng/PipeboxCommon/Helpers/Registrars.h" #include "revng/PipeboxCommon/RawContainer.h" +#include "revng/Recompile/CompileModulePipe.h" +#include "revng/Recompile/LinkForTranslationPipe.h" +#include "revng/Yield/HexDump.h" +#include "revng/Yield/Pipes/ProcessAssembly.h" +#include "revng/Yield/Pipes/YieldAssembly.h" using namespace revng::pypeline; @@ -12,8 +28,40 @@ using namespace revng::pypeline; // Containers // -static RegisterContainer C1; -static RegisterContainer C2; -static RegisterContainer C3; -static RegisterContainer C4; +static RegisterContainer C1; +static RegisterContainer C3; +static RegisterContainer C4; static RegisterContainer C5; +static RegisterContainer C6; +static RegisterContainer C7; +static RegisterContainer C8; +static RegisterContainer C9; +static RegisterContainer C10; +static RegisterContainer C11; +static RegisterContainer C12; + +// +// Pipes +// + +using namespace revng::pypeline::pipes; +using namespace revng::pypeline::piperuns; +namespace piperuns = revng::pypeline::piperuns; + +static RegisterSingleOutputPipe P1; +static RegisterPipe P2; +static RegisterPipe P3; +static RegisterSingleOutputPipe P4; +static RegisterTypeDefinitionPipe P5; +static RegisterFunctionPipe P6; +static RegisterFunctionPipe P7; +static RegisterFunctionPipe P8; +static RegisterFunctionPipe P9; +static RegisterFunctionPipe P10; +static RegisterSingleOutputPipe P11; +static RegisterFunctionPipe P12; +static RegisterFunctionPipe P13; +static RegisterSingleOutputPipe P14; +static RegisterSingleOutputPipe P15; +static RegisterSingleOutputPipe P16; +static RegisterSingleOutputPipe P17; diff --git a/lib/PipeboxCommon/Python/_pipebox.cpp b/lib/PipeboxCommon/Python/_pipebox.cpp index f41d4e1f8..eb454cd20 100644 --- a/lib/PipeboxCommon/Python/_pipebox.cpp +++ b/lib/PipeboxCommon/Python/_pipebox.cpp @@ -2,34 +2,71 @@ // This file is distributed under the MIT License. See LICENSE.md for details. // +#include + #include "nanobind/nanobind.h" #include "nanobind/stl/optional.h" #include "nanobind/stl/set.h" #include "nanobind/stl/string.h" #include "nanobind/stl/vector.h" +#include "llvm/Support/Signals.h" + +#include "revng/ADT/SetOperations.h" #include "revng/PipeboxCommon/Helpers/Python/Casters.h" #include "revng/PipeboxCommon/Helpers/Python/Helpers.h" #include "revng/PipeboxCommon/Helpers/Python/Registry.h" #include "revng/PipeboxCommon/Model.h" #include "revng/Support/InitRevng.h" -NB_MODULE(_pipebox, m) { - { - // Do InitRevng, use a capsule to call the destructor when the Python module - // is unloaded - int Argc = 1; - const char *Argv[] = { "" }; - const char **ArgvPtr = Argv; - nanobind::capsule Capsule(new revng::InitRevng(Argc, ArgvPtr, "", {}), - [](void *Ptr) noexcept { - delete static_cast(Ptr); - }); - nanobind::setattr(m, "__init_revng__", Capsule); - } +struct SignalHandler { + bool IsTerminating; + sighandler_t Handler; +}; +static std::map SavedSignals; + +static void handleSignal(int SigNo) { + const SignalHandler &Handler = SavedSignals[SigNo]; + if (Handler.IsTerminating) + llvm::sys::RunInterruptHandlers(); + Handler.Handler(SigNo); +} + +NB_MODULE(_pipebox, m) { using namespace revng::pypeline::helpers::python; + auto Initialize = [m](std::set TerminatingSignals, + std::set NonterminatingSignals, + std::vector ArgVector) { + revng_assert(not nanobind::hasattr(m, "__init_revng__")); + revng_assert(not intersects(TerminatingSignals, NonterminatingSignals)); + + // Save the signal pointers for later + for (int SigNumber : + llvm::concat(TerminatingSignals, NonterminatingSignals)) { + sighandler_t Handler = signal(SigNumber, SIG_DFL); + if (Handler != SIG_ERR && Handler != NULL) { + bool IsTerminating = TerminatingSignals.contains(SigNumber); + SavedSignals[SigNumber] = { IsTerminating, Handler }; + } + } + + int Argc = ArgVector.size() + 1; + const char *Argv[Argc]; + Argv[0] = ""; + for (size_t I = 0; I < ArgVector.size(); I++) + Argv[I + 1] = ArgVector[I].c_str(); + + const char **ArgvPtr = Argv; + // use a capsule to call the destructor when the Python module is unloaded + m.attr("__init_revng__") = makeCapsule(Argc, ArgvPtr, ""); + + for (int SigNumber : std::ranges::views::keys(SavedSignals)) + signal(SigNumber, &handleSignal); + }; + m.def("initialize", Initialize); + // Register the Buffer class nanobind::class_(m, "Buffer") .def(nanobind::init<>()) @@ -94,6 +131,9 @@ NB_MODULE(_pipebox, m) { nanobind::object ModelBaseClass = importObject("revng.pypeline.model.Model"); nanobind::class_(m, "Model", ModelBaseClass) .def(nanobind::init<>()) + .def_static("model_name", []() { return nanobind::str("model.yml"); }) + .def_static("mime_type", + []() { return nanobind::str("application/x-yaml"); }) .def("diff", [](Model &Handle, nanobind::handle_t Other) { return Handle.diff(*nanobind::cast(Other)); diff --git a/lib/Pipeline/LLVMContainer.cpp b/lib/Pipeline/LLVMContainer.cpp index ed9ce3200..019c67baa 100644 --- a/lib/Pipeline/LLVMContainer.cpp +++ b/lib/Pipeline/LLVMContainer.cpp @@ -61,116 +61,17 @@ void pipeline::makeGlobalObjectsArray(llvm::Module &Module, GlobalArrayName); } -struct FunctionsMetadata { - static inline const char *MetadataName = "revng.functions-metatadata-backup"; - - static void dropBackup(llvm::Module &M) { - M.eraseNamedMetadata(M.getNamedMetadata(MetadataName)); - } - - static void backup(llvm::Module &M) { - using namespace llvm; - - LLVMContext &Context = M.getContext(); - - // Create the named metadata - NamedMDNode *BackupNMD = M.getOrInsertNamedMetadata(MetadataName); - BackupNMD->clearOperands(); - - // Backup saving names, the metadata kind IDs might change - SmallVector MDKindNames; - M.getMDKindNames(MDKindNames); - - for (Function &F : M) { - // Ignore unnamed functions - if (F.getName().empty()) - continue; - - // Collect metadata for the function - SmallVector, 8> MDs; - F.getAllMetadata(MDs); - - // Skip if no metadata - if (MDs.empty()) - continue; - - // Create an MDNode for the function's metadata - // Format: [FunctionName, MDKind1, MDNode1, MDKind2, MDNode2, ...] - SmallVector BackupEntry; - BackupEntry.push_back(MDString::get(Context, F.getName())); - - for (const auto &MD : MDs) { - BackupEntry.push_back(MDString::get(Context, MDKindNames[MD.first])); - BackupEntry.push_back(MD.second); - } - - // Append entry - BackupNMD->addOperand(MDNode::get(Context, BackupEntry)); - } - } - - static void restore(llvm::Module &M) { - using namespace llvm; - - NamedMDNode *BackupNMD = M.getNamedMetadata(MetadataName); - if (!BackupNMD) - return; - - // dumpModule(&M, "/tmp/asd3.ll"); - - // Iterate over all operands in the backup NamedMDNode - for (MDNode *N : BackupNMD->operands()) { - if (N->getNumOperands() == 0) - continue; - - // Get the function name - StringRef FuncName = cast(N->getOperand(0))->getString(); - - // Find the function by name - Function *F = M.getFunction(FuncName); - if (F == nullptr) - continue; - - // Clear existing metadata - F->clearMetadata(); - - auto OperandCount = N->getNumOperands(); - revng_assert(OperandCount >= 3 and OperandCount % 2 == 1); - - // Restore metadata (operands come in pairs: kind name, MDNode) - for (unsigned I = 1; I < OperandCount; I += 2) { - auto *KindMD = cast(N->getOperand(I).get()); - MDNode *MD = cast(N->getOperand(I + 1)); - F->setMetadata(KindMD->getString(), MD); - } - } - } -}; - std::unique_ptr LLVMContainer::cloneFiltered(const TargetsList &Targets) const { using InspectorT = LLVMKind; auto ToClone = InspectorT::functions(Targets, *this->self()); auto ToCloneUntracked = InspectorT::untrackedFunctions(*this->self()); - const auto Filter = [&ToClone, &ToCloneUntracked](const auto &GlobalSym) { - if (not llvm::isa(GlobalSym)) - return true; + std::set ToCloneAll; + ToCloneAll.insert(ToClone.begin(), ToClone.end()); + ToCloneAll.insert(ToCloneUntracked.begin(), ToCloneUntracked.end()); - const auto &F = llvm::cast(GlobalSym); - return ToClone.contains(F) or ToCloneUntracked.contains(F); - }; - - llvm::ValueToValueMapTy Map; - - FunctionsMetadata::backup(*Module.get()); - - revng::verify(Module.get()); - auto Cloned = llvm::CloneModule(*Module, Map, Filter); - - FunctionsMetadata::restore(*Cloned.get()); - FunctionsMetadata::dropBackup(*Module.get()); - FunctionsMetadata::dropBackup(*Cloned.get()); + auto Cloned = ::cloneFiltered(*this->Module, ToCloneAll); return std::make_unique(this->name(), this->TheContext, diff --git a/lib/Pipes/CMakeLists.txt b/lib/Pipes/CMakeLists.txt index a5b3426ff..368ac0d40 100644 --- a/lib/Pipes/CMakeLists.txt +++ b/lib/Pipes/CMakeLists.txt @@ -14,32 +14,6 @@ revng_add_library_internal( GlobalsAnalyses.cpp DebugInfoHelpers.cpp) -llvm_map_components_to_libnames( - LLVM_LIBRARIES - AggressiveInstCombine - Analysis - AsmParser - BitWriter - CodeGen - Core - Coroutines - Extensions - GlobalISel - IPO - InstCombine - Instrumentation - MC - Passes - Remarks - ScalarOpts - Support - Target - TransformUtils - Vectorize - X86AsmParser - X86CodeGen - X86Desc - X86Info) - +llvm_map_components_to_libnames(LLVM_LIBRARIES Core MC Support) target_link_libraries(revngPipes revngPipeline revngModel revngSupport revngStorage ${LLVM_LIBRARIES}) diff --git a/lib/Pipes/Pipes.cpp b/lib/Pipes/Pipes.cpp index a13dcb8d5..0cfb902fa 100644 --- a/lib/Pipes/Pipes.cpp +++ b/lib/Pipes/Pipes.cpp @@ -74,25 +74,7 @@ public: void registerKinds(KindsRegistry &KindDictionary) override {} - void libraryInitialization() override { - - llvm::codegen::RegisterCodeGenFlags Reg; - llvm::InitializeNativeTarget(); - llvm::InitializeNativeTargetAsmPrinter(); - llvm::InitializeNativeTargetAsmParser(); - - auto &Registry = *llvm::PassRegistry::getPassRegistry(); - llvm::initializeCore(Registry); - llvm::initializeTransformUtils(Registry); - llvm::initializeScalarOpts(Registry); - llvm::initializeVectorization(Registry); - llvm::initializeInstCombine(Registry); - llvm::initializeIPO(Registry); - llvm::initializeAnalysis(Registry); - llvm::initializeCodeGen(Registry); - llvm::initializeGlobalISel(Registry); - llvm::initializeTarget(Registry); - } + void libraryInitialization() override {} ~LLVMPipelineRegistry() override = default; }; diff --git a/lib/Recompile/CompileModulePipe.cpp b/lib/Recompile/CompileModulePipe.cpp index dfa113f72..d5c6087b4 100644 --- a/lib/Recompile/CompileModulePipe.cpp +++ b/lib/Recompile/CompileModulePipe.cpp @@ -82,30 +82,19 @@ public: } }; -static void compileModuleRunImpl(const Context &Context, - LLVMContainer &Module, - ObjectFileContainer &TargetBinary) { +// This function is used by both the old and the new pipeline +static void compileModuleRunImpl(const model::Binary &Binary, + llvm::Module *M, + llvm::raw_pwrite_stream &Output) { using namespace revng; - - auto Enumeration = Module.enumerate(); - if (not Enumeration.contains(pipeline::Target(kinds::Root)) - and not Enumeration.contains(pipeline::Target(kinds::IsolatedRoot))) - return; - - if (Enumeration.contains(pipeline::Target(kinds::IsolatedRoot)) - and not Enumeration.contains(kinds::Isolated.allTargets(Context))) - return; - StringMap &RegOptions(getRegisteredOptions()); getOption(RegOptions, "disable-machine-licm")->setInitialValue(true); - llvm::Module *M = &Module.getModule(); - M->getContext() .setDiagnosticHandler(std::make_unique()); { - auto Architecture = getModelFromContext(Context)->Architecture(); + auto Architecture = Binary.Architecture(); auto ArchName = model::Architecture::getQEMUName(Architecture).str(); // Note: here we use the full version of the helpers, i.e., where we all the @@ -127,7 +116,7 @@ static void compileModuleRunImpl(const Context &Context, F.setSection(""); OriginalAssemblyAnnotationWriter OAAW(M->getContext()); - createSelfReferencingDebugInfo(M, Module.name(), &OAAW); + createSelfReferencingDebugInfo(M, "llvm-container", &OAAW); // Get the target specific parser. std::string Error; @@ -186,12 +175,8 @@ static void compileModuleRunImpl(const Context &Context, TargetLibraryInfoImpl TLII(Triple(M->getTargetTriple())); PM.add(new TargetLibraryInfoWrapperPass(TLII)); - std::error_code EC; - raw_fd_ostream OutputStream(TargetBinary.getOrCreatePath(), EC); - revng_assert(!EC); - bool Err = Target->addPassesToEmitFile(PM, - OutputStream, + Output, nullptr, CGFT_ObjectFile, true); @@ -200,6 +185,32 @@ static void compileModuleRunImpl(const Context &Context, revng::verify(M); PM.run(*M); revng::verify(M); +} + +// This function is a wrapper of the previous `compileModuleRunImpl` and it's +// used only by the old pipeline in both `CompileModule::run` and +// `CompileIsolatedModule::run` +static void compileModuleRunImpl(const Context &Context, + LLVMContainer &Module, + ObjectFileContainer &TargetBinary) { + using namespace revng; + + auto Enumeration = Module.enumerate(); + if (not Enumeration.contains(pipeline::Target(kinds::Root)) + and not Enumeration.contains(pipeline::Target(kinds::IsolatedRoot))) + return; + + if (Enumeration.contains(pipeline::Target(kinds::IsolatedRoot)) + and not Enumeration.contains(kinds::Isolated.allTargets(Context))) + return; + + std::error_code EC; + raw_fd_ostream OutputStream(TargetBinary.getOrCreatePath(), EC); + revng_assert(!EC); + + compileModuleRunImpl(*getModelFromContext(Context), + &Module.getModule(), + OutputStream); auto Path = TargetBinary.path(); @@ -224,3 +235,17 @@ void CompileIsolatedModule::run(ExecutionContext &EC, static RegisterPipe E2; static RegisterPipe E3; + +namespace revng::pypeline::piperuns { + +void CompileRootModule::run(const Model &TheModel, + llvm::StringRef StaticConfig, + llvm::StringRef DynamicConfig, + LLVMRootContainer &Input, + ObjectFileContainer &Output) { + compileModuleRunImpl(*TheModel.get().get(), + &Input.getModule(), + *Output.getOStream(ObjectID{})); +} + +} // namespace revng::pypeline::piperuns diff --git a/lib/Recompile/LinkForTranslationPipe.cpp b/lib/Recompile/LinkForTranslationPipe.cpp index a7a39f779..3f30537f2 100644 --- a/lib/Recompile/LinkForTranslationPipe.cpp +++ b/lib/Recompile/LinkForTranslationPipe.cpp @@ -34,3 +34,50 @@ void LinkForTranslation::run(ExecutionContext &EC, } static RegisterPipe E5; + +namespace revng::pypeline::piperuns { + +llvm::Error LinkForTranslation::checkPrecondition(const class Model &Model) { + if (Model.get().get()->Binaries().size() != 1) + return revng::createError("Binaries must have exactly one element"); + return llvm::Error::success(); +} + +static void writeToFile(llvm::StringRef Path, llvm::StringRef Buffer) { + std::error_code EC; + llvm::raw_fd_ostream OS(Path, EC); + revng_assert(not EC); + OS << Buffer; +} + +void LinkForTranslation::run(const Model &TheModel, + llvm::StringRef StaticConfig, + llvm::StringRef DynamicConfig, + const BinariesContainer &Binaries, + const ObjectFileContainer &ObjectFile, + TranslatedContainer &Output) { + // TODO: some of the operations in linkForTranslation should be converted to + // in-memory counterparts to avoid serializing everything. + TemporaryFile Binary("revng-lft-binary"); + auto BinaryArrayRef = Binaries.getFile(0); + writeToFile(Binary.path(), { BinaryArrayRef.begin(), BinaryArrayRef.size() }); + + TemporaryFile Object("revng-lft-object", "o"); + writeToFile(Object.path(), + ObjectFile.getMemoryBuffer(ObjectID{})->getBuffer()); + + TemporaryFile TempOutput("revng-lft-output"); + + linkForTranslation(*TheModel.get().get(), + Binary.path(), + Object.path(), + TempOutput.path()); + + auto Buffer = revng::cantFail(llvm::MemoryBuffer::getFile(TempOutput.path())); + { + auto OutputOS = Output.getOStream(ObjectID{}); + *OutputOS << Buffer->getBuffer(); + } +} + +} // namespace revng::pypeline::piperuns diff --git a/lib/Support/CMakeLists.txt b/lib/Support/CMakeLists.txt index 3349060dc..cc11cf764 100644 --- a/lib/Support/CMakeLists.txt +++ b/lib/Support/CMakeLists.txt @@ -14,6 +14,7 @@ revng_add_library_internal( CustomizedLLVMPasses.cpp Debug.cpp ExplicitSpecializations.cpp + InitRevng.cpp IRHelpers.cpp LDDTree.cpp MetaAddress.cpp @@ -35,6 +36,29 @@ if(NOT LibArchive_FOUND) message(FATAL_ERROR "libarchive not found") endif() +llvm_map_components_to_libnames( + LLVM_LIBRARIES + Analysis + BitReader + BitWriter + CodeGen + Core + GlobalISel + InstCombine + IPO + IRReader + Linker + Object + Passes + ScalarOpts + Support + Target + TransformUtils + Vectorize + X86AsmParser + X86CodeGen + X86Desc + X86Info) target_link_libraries(revngSupport z zstd ${LibArchive_LIBRARIES} ${LLVM_LIBRARIES}) diff --git a/lib/Support/IRHelpers.cpp b/lib/Support/IRHelpers.cpp index ddb3bbe14..d4eddfc32 100644 --- a/lib/Support/IRHelpers.cpp +++ b/lib/Support/IRHelpers.cpp @@ -11,6 +11,8 @@ #include "llvm/ADT/PostOrderIterator.h" #include "llvm/ADT/SetVector.h" #include "llvm/ADT/SmallSet.h" +#include "llvm/Bitcode/BitcodeReader.h" +#include "llvm/Bitcode/BitcodeWriter.h" #include "llvm/IR/DebugInfo.h" #include "llvm/IR/DebugInfoMetadata.h" #include "llvm/IR/Dominators.h" @@ -20,6 +22,8 @@ #include "llvm/Linker/Linker.h" #include "llvm/Support/FileSystem.h" #include "llvm/Support/raw_os_ostream.h" +#include "llvm/Transforms/Utils/Cloning.h" +#include "llvm/Transforms/Utils/ValueMapper.h" #include "revng/ADT/Queue.h" #include "revng/ADT/RecursiveCoroutine.h" @@ -691,3 +695,130 @@ void linkModules(std::unique_ptr &&Source, F->setLinkage(FinalLinkage.value_or(Linkage)); } } + +struct FunctionsMetadata { + static inline const char *MetadataName = "revng.functions-metatadata-backup"; + + static void dropBackup(llvm::Module &M) { + M.eraseNamedMetadata(M.getNamedMetadata(MetadataName)); + } + + static void backup(llvm::Module &M) { + using namespace llvm; + + LLVMContext &Context = M.getContext(); + + // Create the named metadata + NamedMDNode *BackupNMD = M.getOrInsertNamedMetadata(MetadataName); + BackupNMD->clearOperands(); + + // Backup saving names, the metadata kind IDs might change + SmallVector MDKindNames; + M.getMDKindNames(MDKindNames); + + for (Function &F : M) { + // Ignore unnamed functions + if (F.getName().empty()) + continue; + + // Collect metadata for the function + SmallVector, 8> MDs; + F.getAllMetadata(MDs); + + // Skip if no metadata + if (MDs.empty()) + continue; + + // Create an MDNode for the function's metadata + // Format: [FunctionName, MDKind1, MDNode1, MDKind2, MDNode2, ...] + SmallVector BackupEntry; + BackupEntry.push_back(MDString::get(Context, F.getName())); + + for (const auto &MD : MDs) { + BackupEntry.push_back(MDString::get(Context, MDKindNames[MD.first])); + BackupEntry.push_back(MD.second); + } + + // Append entry + BackupNMD->addOperand(MDNode::get(Context, BackupEntry)); + } + } + + static void restore(llvm::Module &M) { + using namespace llvm; + + NamedMDNode *BackupNMD = M.getNamedMetadata(MetadataName); + if (!BackupNMD) + return; + + // Iterate over all operands in the backup NamedMDNode + for (MDNode *N : BackupNMD->operands()) { + if (N->getNumOperands() == 0) + continue; + + // Get the function name + StringRef FuncName = cast(N->getOperand(0))->getString(); + + // Find the function by name + Function *F = M.getFunction(FuncName); + if (F == nullptr) + continue; + + // Clear existing metadata + F->clearMetadata(); + + auto OperandCount = N->getNumOperands(); + revng_assert(OperandCount >= 3 and OperandCount % 2 == 1); + + // Restore metadata (operands come in pairs: kind name, MDNode) + for (unsigned I = 1; I < OperandCount; I += 2) { + auto *KindMD = cast(N->getOperand(I).get()); + MDNode *MD = cast(N->getOperand(I + 1)); + F->setMetadata(KindMD->getString(), MD); + } + } + } +}; + +std::unique_ptr +cloneFiltered(llvm::Module &Module, std::set &ToClone) { + const auto Filter = [&ToClone](const auto &GlobalSym) { + if (not llvm::isa(GlobalSym)) + return true; + + const auto &F = llvm::cast(GlobalSym); + return ToClone.contains(F); + }; + + llvm::ValueToValueMapTy Map; + + FunctionsMetadata::backup(Module); + + revng::verify(&Module); + auto Cloned = llvm::CloneModule(Module, Map, Filter); + + FunctionsMetadata::restore(*Cloned.get()); + FunctionsMetadata::dropBackup(Module); + FunctionsMetadata::dropBackup(*Cloned.get()); + + return Cloned; +} + +void writeBitcode(const llvm::Module &Module, + llvm::SmallVectorImpl &Output) { + llvm::BitcodeWriter Writer(Output); + Writer.writeModule(Module); + Writer.writeSymtab(); + Writer.writeStrtab(); +} + +std::unique_ptr cloneIntoContext(const llvm::Module &Module, + llvm::LLVMContext &NewContext) { + revng_assert(&Module.getContext() != &NewContext); + + llvm::SmallVector Buffer; + writeBitcode(Module, Buffer); + + llvm::MemoryBufferRef BufferRef{ { Buffer.data(), Buffer.size() }, "input" }; + return llvm::cantFail(llvm::parseBitcodeFile(BufferRef, NewContext)); +} diff --git a/lib/Support/InitRevng.cpp b/lib/Support/InitRevng.cpp new file mode 100644 index 000000000..7130448ec --- /dev/null +++ b/lib/Support/InitRevng.cpp @@ -0,0 +1,33 @@ +/// \file InitRevng.cpp + +// +// This file is distributed under the MIT License. See LICENSE.md for details. +// + +#include "llvm/CodeGen/CommandFlags.h" +#include "llvm/InitializePasses.h" +#include "llvm/PassRegistry.h" +#include "llvm/Support/TargetSelect.h" + +#include "revng/Support/InitRevng.h" + +static std::optional CodegenFlags; + +void revng::InitRevng::initializeLLVMLibraries() { + CodegenFlags.emplace(); + llvm::InitializeNativeTarget(); + llvm::InitializeNativeTargetAsmPrinter(); + llvm::InitializeNativeTargetAsmParser(); + + auto &Registry = *llvm::PassRegistry::getPassRegistry(); + llvm::initializeCore(Registry); + llvm::initializeTransformUtils(Registry); + llvm::initializeScalarOpts(Registry); + llvm::initializeVectorization(Registry); + llvm::initializeInstCombine(Registry); + llvm::initializeIPO(Registry); + llvm::initializeAnalysis(Registry); + llvm::initializeCodeGen(Registry); + llvm::initializeGlobalISel(Registry); + llvm::initializeTarget(Registry); +} diff --git a/lib/Yield/Assembly/DisassemblyHelper.cpp b/lib/Yield/Assembly/DisassemblyHelper.cpp index 7188e322e..cc6c0b566 100644 --- a/lib/Yield/Assembly/DisassemblyHelper.cpp +++ b/lib/Yield/Assembly/DisassemblyHelper.cpp @@ -108,15 +108,15 @@ static void analyzeBasicBlocks(yield::Function &Function, } } -yield::Function DH::disassemble(const model::Function &Function, - const efa::ControlFlowGraph &Metadata, - const RawBinaryView &BinaryView, - const model::Binary &Binary, - const model::AssemblyNameBuilder &NameBuilder) { +void DH::disassemble(const model::Function &Function, + const efa::ControlFlowGraph &Metadata, + const RawBinaryView &BinaryView, + const model::Binary &Binary, + const model::AssemblyNameBuilder &NameBuilder, + yield::Function &ResultFunction) { revng_log(Log, "Disassembling function at " << Function.Entry().toString()); LoggerIndent Indent(Log); - yield::Function ResultFunction; ResultFunction.Entry() = Function.Entry(); for (auto BasicBlockInserter = ResultFunction.Blocks().batch_insert(); const efa::BasicBlock &BasicBlock : Metadata.Blocks()) { @@ -208,8 +208,6 @@ yield::Function DH::disassemble(const model::Function &Function, } ResultFunction.verify(true); - - return ResultFunction; } LLVMDisassemblerInterface & diff --git a/lib/Yield/HexDump.cpp b/lib/Yield/HexDump.cpp index d9ed6b544..5752228b4 100644 --- a/lib/Yield/HexDump.cpp +++ b/lib/Yield/HexDump.cpp @@ -22,6 +22,7 @@ #include "revng/Pipes/Ranks.h" #include "revng/Support/IRHelpers.h" #include "revng/Support/MetaAddress/IntervalContainers.h" +#include "revng/Yield/HexDump.h" using namespace llvm; @@ -33,21 +34,15 @@ static FormattedNumber formatNumber(uint64_t Number, unsigned Width = 8) { return FormattedNumber(Number, 0, Width, true, false, false); }; -static void outputHexDump(const TupleTree &Binary, - const pipeline::LLVMContainer &ModuleContainer, - const CFGMap &CFGMap, - const BinaryFileContainer &SourceBinary, - StringRef OutputPath) { - auto BufferOrError = MemoryBuffer::getFileOrSTDIN(*SourceBinary.path()); - auto Buffer = cantFail(errorOrToExpected(std::move(BufferOrError))); - RawBinaryView BinaryView(*Binary.get(), Buffer->getBuffer()); +using CFG = efa::ControlFlowGraph; +using CFGGetter = std::function; - std::error_code ErrorCode; - raw_fd_ostream Output(OutputPath, ErrorCode, sys::fs::CD_CreateAlways); - - revng_assert(not ErrorCode, "Could not open file!"); - - ControlFlowGraphCache ControlFlowGraphCache(CFGMap); +static void outputHexDump(const model::Binary &Binary, + llvm::ArrayRef Functions, + CFGGetter CFGGetter, + llvm::StringRef BinaryBuffer, + llvm::raw_ostream &Output) { + RawBinaryView BinaryView(Binary, BinaryBuffer); using boost::icl::discrete_interval; using boost::icl::inplace_plus; @@ -71,10 +66,9 @@ static void outputHexDump(const TupleTree &Binary, return Tag; }; - for (const Function &F : - FunctionTags::Isolated.functions(&ModuleContainer.getModule())) { - const efa::ControlFlowGraph &Metadata = ControlFlowGraphCache - .getControlFlowGraph(&F); + for (const Function *F : Functions) { + MetaAddress Address = getMetaAddressOfIsolatedFunction(*F); + const efa::ControlFlowGraph &Metadata = CFGGetter(Address); MetaAddress EntryAddress = Metadata.Entry(); for (const Instruction &I : llvm::instructions(F)) { @@ -281,12 +275,31 @@ public: if (not CFGList.contains(kinds::CFG.allTargets(EC.getContext()))) return; + const model::Binary &Binary = *getModelFromContext(EC); + + std::vector Functions; + for (const llvm::Function &F : + FunctionTags::Isolated.functions(&ModuleContainer.getModule())) { + Functions.push_back(&F); + } + + ControlFlowGraphCache CFGCache(CFGMap); + auto CFGGetter = + [&CFGCache](const MetaAddress &Address) -> const efa::ControlFlowGraph & { + return CFGCache.getControlFlowGraph(Address); + }; + + auto Buffer = revng::cantFail(MemoryBuffer::getFile(*SourceBinary.path())); + + std::error_code ErrorCode; + raw_fd_ostream OutputOS(Output.getOrCreatePath(), + ErrorCode, + sys::fs::CD_CreateAlways); + + revng_assert(not ErrorCode, "Could not open file!"); + // Proceed with emission - outputHexDump(getModelFromContext(EC), - ModuleContainer, - CFGMap, - SourceBinary, - Output.getOrCreatePath()); + outputHexDump(Binary, Functions, CFGGetter, Buffer->getBuffer(), OutputOS); EC.commitUniqueTarget(Output); } @@ -295,3 +308,42 @@ public: } // namespace revng::pipes // static pipeline::RegisterPipe X; + +namespace revng::pypeline::piperuns { + +void HexDump::run(const class Model &Model, + llvm::StringRef Config, + llvm::StringRef DynamicConfig, + const BinariesContainer &BinaryContainer, + const LLVMFunctionContainer &ModuleContainer, + const CFGMap &CFG, + HexDumpContainer &Output) { + + const model::Binary &Binary = *Model.get().get(); + + auto Buffer = BinaryContainer.getFile(0); + auto OutputOS = Output.getOStream(ObjectID{}); + + std::vector Functions; + for (const model::Function &Function : Binary.Functions()) { + const llvm::Module &Module = ModuleContainer + .getModule(ObjectID(Function.Entry())); + for (const llvm::Function &LLVMFunction : + FunctionTags::Isolated.functions(&Module)) { + Functions.push_back(&LLVMFunction); + } + } + + auto CFGGetter = + [&CFG](const MetaAddress &Address) -> const efa::ControlFlowGraph & { + return *CFG.getElement(ObjectID(Address)); + }; + + ::revng::pipes::outputHexDump(Binary, + Functions, + CFGGetter, + { Buffer.data(), Buffer.size() }, + *OutputOS); +} + +} // namespace revng::pypeline::piperuns diff --git a/lib/Yield/Pipes/AssemblyPipes.cpp b/lib/Yield/Pipes/AssemblyPipes.cpp index e50a18b98..af7988b53 100644 --- a/lib/Yield/Pipes/AssemblyPipes.cpp +++ b/lib/Yield/Pipes/AssemblyPipes.cpp @@ -62,6 +62,45 @@ void ProcessAssembly::run(pipeline::ExecutionContext &Context, } } +} // namespace revng::pipes + +namespace revng::pypeline::piperuns { + +ProcessAssembly::ProcessAssembly(const class Model &Model, + llvm::StringRef Config, + llvm::StringRef DynamicConfig, + const BinariesContainer &BinariesContainer, + const CFGMap &CFG, + AssemblyInternalContainer &Output) : + Binary(*Model.get().get()), CFG(CFG), Output(Output), NameBuilder(Binary) { + Helper = std::make_unique(); + + auto BinaryBuffer = BinariesContainer.getFile(0); + BinaryView = std::make_unique(Binary, + llvm::StringRef{ + BinaryBuffer.data(), + BinaryBuffer.size() }); +}; + +ProcessAssembly::~ProcessAssembly() = default; + +void ProcessAssembly::runOnFunction(const model::Function &TheFunction) { + ObjectID Object(TheFunction.Entry()); + const auto &Metadata = CFG.getElement(Object); + + TupleTree &OutputFunction = Output.getElement(Object); + Helper->disassemble(TheFunction, + *Metadata, + *BinaryView, + Binary, + NameBuilder, + *OutputFunction); +} + +} // namespace revng::pypeline::piperuns + +namespace revng::pipes { + void YieldAssembly::run(pipeline::ExecutionContext &Context, const FunctionAssemblyStringMap &Input, FunctionAssemblyPTMLStringMap &Output) { @@ -108,3 +147,34 @@ static RegisterDefaultConstructibleContainer X2; static pipeline::RegisterPipe ProcessPipe; static pipeline::RegisterPipe YieldPipe; + +namespace revng::pypeline::piperuns { + +void YieldAssembly::runOnFunction(const model::Function &TheFunction) { + MetaAddress Address = TheFunction.Entry(); + ObjectID Object(Address); + const TupleTree &Function = Input.getElement(Object); + + revng_assert(Function.verify()); + revng_assert(Function->verify()); + revng_assert(Function->Entry() == Address); + + const model::Architecture::Values A = Model.Architecture(); + auto CommentIndicator = model::Architecture::getAssemblyCommentIndicator(A); + + const model::Configuration &Configuration = Model.Configuration(); + uint64_t LineWidth = Configuration.CommentLineWidth(); + + std::string R = ptml::functionComment(B, + TheFunction, + Model, + CommentIndicator, + 0, + LineWidth, + NameBuilder); + R += yield::ptml::functionAssembly(B, *Function, Model); + R = B.getTag(ptml::tags::Div, std::move(R)).toString(); + *Output.getOStream(Object) << R; +} + +} // namespace revng::pypeline::piperuns diff --git a/python/revng/internal/cli/revng2.py b/python/revng/internal/cli/revng2.py index 68d25a75a..a8afb1e28 100644 --- a/python/revng/internal/cli/revng2.py +++ b/python/revng/internal/cli/revng2.py @@ -12,8 +12,8 @@ import os from pathlib import Path import click -from xdg import xdg_cache_home +from revng.internal.support import cache_directory from revng.pypeline.cli.project import project from revng.pypeline.cli.utils import PypeGroup from revng.pypeline.main import pype, run @@ -53,10 +53,7 @@ def patch_pype(): "PIPELINE", Path(__file__).parent.parent / "pipeline.yml" ) elif param.name == "cache_dir": - param.default = os.environ.get( - "CACHE_DIR", - xdg_cache_home() / "revng", - ) + param.default = cache_directory() def main(): diff --git a/python/revng/internal/pipebox.py b/python/revng/internal/pipebox.py index 95c2f1b72..32aac3430 100644 --- a/python/revng/internal/pipebox.py +++ b/python/revng/internal/pipebox.py @@ -2,6 +2,7 @@ # This file is distributed under the MIT License. See LICENSE.md for details. # +import signal import tarfile from io import BytesIO @@ -18,6 +19,24 @@ from revng.support import get_root _module, _handles = import_pipebox([get_root() / "lib/librevngPipebox.so"]) +def initialize(argv: list[str] = []): + _module.initialize( + { + signal.SIGINT, + signal.SIGTERM, + signal.SIGHUP, + signal.SIGQUIT, + }, + { + signal.SIGCHLD, + signal.SIGPIPE, + signal.SIGUSR1, + signal.SIGUSR2, + }, + argv, + ) + + class ImportFiles(Pipe): @classmethod def signature(cls) -> tuple[TaskArgument, ...]: @@ -71,4 +90,7 @@ class ImportFiles(Pipe): containers[0].deserialize({root_object: buffer.getvalue()}) - return [[(root_object, f"/Binaries/{i}/Hash") for i in indexes]] + dependencies = [(root_object, f"/Binaries/{i}/Hash") for i in indexes] + dependencies.append((root_object, "/Binaries")) + + return [dependencies] diff --git a/python/revng/internal/pipeline.yml b/python/revng/internal/pipeline.yml index 5c48f664f..bb6cc8a1f 100644 --- a/python/revng/internal/pipeline.yml +++ b/python/revng/internal/pipeline.yml @@ -2,4 +2,202 @@ # This file is distributed under the MIT License. See LICENSE.md for details. # -# Placeholder for the revng pipeline +containers: + - name: binaries-container + type: BinariesContainer + - name: llvm-root + type: LLVMRootContainer + - name: model-header + type: CBytesContainer + - name: model-types + type: PTMLCTypeContainer + - name: cfg-map + type: CFGMap + - name: llvm-functions + type: LLVMFunctionContainer + - name: hexdump + type: HexDumpContainer + - name: assembly-internal + type: AssemblyInternalContainer + - name: assembly + type: AssemblyContainer + - name: object-file + type: ObjectFileContainer + - name: translated + type: TranslatedContainer + - name: llvm-root-with-functions + type: LLVMRootContainer + +branches: + files-imported: + tasks: + - pipe: ImportFiles + arguments: [binaries-container] + + lift: + from: files-imported + tasks: + - pipe: Lift + arguments: [binaries-container, llvm-root] + - pipe: PureLLVMPassesRootPipe + arguments: [llvm-root] + configuration: + Passes: + - globaldce + - savepoint: lifted + containers: [llvm-root] + artifacts: + - name: lift + container: llvm-root + + collect-cfg: + from: lift + tasks: + - pipe: CollectCFG + arguments: [llvm-root, cfg-map] + - savepoint: cfg-computed + containers: [cfg-map] + + isolate: + from: collect-cfg + tasks: + - pipe: Isolate + arguments: [cfg-map, llvm-root, llvm-functions] + - pipe: AttachDebugInfo + arguments: [cfg-map, llvm-functions] + - savepoint: isolate + containers: [llvm-functions] + artifacts: + - name: isolate + container: llvm-functions + + enforce-abi: + from: isolate + tasks: + - pipe: EnforceABI + arguments: [cfg-map, llvm-functions] + - pipe: PureLLVMPassesPipe + arguments: [llvm-functions] + configuration: + Passes: + - strip-debug-info-from-helpers + # Note: we're running promote-csvs twice: it is important to run + # it before inline-helpers so that mem2reg can constant + # propagate helper arguments that are constant. This enables + # us to inline less code, in particular for helpers such as + # `cc_compute_c` and `cc_compute_all`. + - pipe: PromoteCSVs + arguments: [llvm-functions] + - pipe: PureLLVMPassesPipe + arguments: [llvm-functions] + configuration: + Passes: + - mem2reg + - inline-helpers + - pipe: AttachDebugInfo + arguments: [cfg-map, llvm-functions] + - pipe: PromoteCSVs + arguments: [llvm-functions] + - pipe: PureLLVMPassesPipe + arguments: [llvm-functions] + configuration: + Passes: + - remove-exceptional-functions + - strip-dead-debug-info + - savepoint: enforce-abi + containers: [llvm-functions] + artifacts: + - name: enforce-abi + container: llvm-functions + + emit-model-header: + tasks: + - pipe: ModelToHeader + arguments: [model-header] + - savepoint: model-header + containers: [model-header] + artifacts: + - name: emit-model-header + container: model-header + + emit-type-definition: + tasks: + - pipe: GenerateModelTypeDefinition + arguments: [model-types] + - savepoint: model-types + containers: [model-types] + artifacts: + - name: emit-type-definitions + container: model-types + + hexdump: + from: isolate + tasks: + - pipe: HexDump + arguments: + - binaries-container + - llvm-functions + - cfg-map + - hexdump + - savepoint: hexdump + containers: [hexdump] + artifacts: + - name: hexdump + container: hexdump + + process-assembly: + from: collect-cfg + tasks: + - pipe: ProcessAssembly + arguments: [binaries-container, cfg-map, assembly-internal] + + disassemble: + from: process-assembly + tasks: + - pipe: YieldAssembly + arguments: [assembly-internal, assembly] + - savepoint: disassemble + containers: [assembly] + artifacts: + - name: disassemble + container: assembly + + recompile: + from: lift + tasks: + - pipe: LinkSupport + arguments: [llvm-root] + - pipe: PureLLVMPassesRootPipe + arguments: [llvm-root] + configuration: + Passes: [drop-opaque-return-address] + - pipe: CompileRootModule + arguments: [llvm-root, object-file] + - pipe: LinkForTranslation + arguments: [binaries-container, object-file, translated] + - savepoint: recompile + containers: [translated] + artifacts: + - name: recompile + container: translated + + recompile-isolated: + from: isolate + tasks: + - pipe: InvokeIsolatedFunctions + arguments: [llvm-root, llvm-functions, llvm-root-with-functions] + - pipe: LinkSupport + arguments: [llvm-root-with-functions] + - pipe: PureLLVMPassesRootPipe + arguments: [llvm-root-with-functions] + configuration: + Passes: [drop-opaque-return-address] + - pipe: CompileRootModule + arguments: [llvm-root-with-functions, object-file] + - pipe: LinkForTranslation + arguments: [binaries-container, object-file, translated] + - savepoint: recompile-isolated + containers: [translated] + artifacts: + - name: recompile-isolated + container: translated diff --git a/python/revng/pypeline/cli/pipeline/run_pipe.py b/python/revng/pypeline/cli/pipeline/run_pipe.py index f94494026..f2663e932 100644 --- a/python/revng/pypeline/cli/pipeline/run_pipe.py +++ b/python/revng/pypeline/cli/pipeline/run_pipe.py @@ -156,10 +156,7 @@ def build_pipe_command( pypeline_logger.debug_log(f"and kwargs: {kwargs}") # Create the pipe - pipe = pipe_type( - name=pipe_name, - static_configuration=static_configuration, - ) + pipe = pipe_type(static_configuration) # Load the model with open(model, "rb") as model_file: loaded_model = model_type.deserialize(model_file.read()) diff --git a/python/revng/pypeline/cli/project/artifact.py b/python/revng/pypeline/cli/project/artifact.py index 30b4b00ad..f23209bac 100644 --- a/python/revng/pypeline/cli/project/artifact.py +++ b/python/revng/pypeline/cli/project/artifact.py @@ -84,6 +84,7 @@ class ArtifactGroup(PypeGroup): "objects", type=str, default=None, + required=False, )(run_artifact_command) return run_artifact_command @@ -167,7 +168,7 @@ def build_artifact_command( ), ) @click.pass_context - def run_analysis_command( + def run_artifact_command( ctx: click.Context, configuration: str, project_id: str, @@ -197,7 +198,7 @@ def build_artifact_command( ) ) - return run_analysis_command + return run_artifact_command @click.group( diff --git a/python/revng/pypeline/container.py b/python/revng/pypeline/container.py index b234acda1..10e453606 100644 --- a/python/revng/pypeline/container.py +++ b/python/revng/pypeline/container.py @@ -41,7 +41,7 @@ Configuration = Annotated[ ] -@dataclass(slots=True, frozen=True) +@dataclass(slots=True, frozen=True, order=True) class ContainerDeclaration: """ A ContainerDeclaration represents a container when describing a pipeline. diff --git a/python/revng/pypeline/graph.py b/python/revng/pypeline/graph.py index 32ee4787b..0568d152c 100644 --- a/python/revng/pypeline/graph.py +++ b/python/revng/pypeline/graph.py @@ -13,7 +13,8 @@ def escape(string: str) -> str: @dataclass(slots=True) class Node: label: str - completed: bool = False + color: str = "" + bgcolor: str = "" entries: List[str] = field(default_factory=list) def name(self) -> str: @@ -26,23 +27,25 @@ class Node: result += """ \n""" cell_properties = "" - if self.completed: - cell_properties = ' bgcolor="lightgreen"' + if self.bgcolor: + cell_properties = f' bgcolor="{self.bgcolor}"' result += f""" {escape(self.label)}\n""" for index, entry in enumerate(self.entries): result += f""" \n""" result += "
{escape(entry)}
\n" - result += " >];\n" - result += "\n" + result += " >" + if self.color: + result += f', color="{self.color}"' + result += "];\n\n" return result def __hash__(self) -> int: return hash(self.name()) -@dataclass(slots=True) +@dataclass(frozen=True, slots=True) class Edge: source: Node destination: Node @@ -50,22 +53,37 @@ class Edge: destination_port: int = 0 head_label: str = "" tail_label: str = "" + style: str = "" + color: str = "" def to_graphviz(self) -> str: result = "" result += " " - result += f"{escape(self.source.name())}:entry_{self.source_port}" + if self.source_port >= 0: + result += f"{escape(self.source.name())}:entry_{self.source_port}" + else: + result += f"{escape(self.source.name())}" + result += " -> " - result += f"{escape(self.destination.name())}:entry_{self.destination_port}" - if self.head_label or self.tail_label: - result += " [" - if self.head_label: - result += f'headlabel="{escape(self.head_label)}"' - if self.tail_label: - if self.head_label: - result += "," - result += f'taillabel="{escape(self.tail_label)}"' - result += "]" + + if self.destination_port >= 0: + result += f"{escape(self.destination.name())}:entry_{self.destination_port}" + else: + result += f"{escape(self.destination.name())}" + + attributes = [] + if self.head_label: + attributes.append(f'headlabel="{escape(self.head_label)}"') + if self.tail_label: + attributes.append(f'taillabel="{escape(self.tail_label)}"') + if self.style: + attributes.append(f'style="{escape(self.style)}"') + if self.color: + attributes.append(f'color="{escape(self.color)}"') + + if len(attributes) > 0: + result += f" [{','.join(attributes)}]" + result += ";\n" return result diff --git a/python/revng/pypeline/object.py b/python/revng/pypeline/object.py index c475c754e..eb12c9cc3 100644 --- a/python/revng/pypeline/object.py +++ b/python/revng/pypeline/object.py @@ -10,7 +10,7 @@ from enum import Enum # Builtin since python 3.9 from graphlib import TopologicalSorter -from typing import Iterator, Optional, Sequence, Set, cast +from typing import Any, Callable, Iterator, Optional, Sequence, Set, cast from .graph import Graph from .utils.cabc import ABC, abstractmethod @@ -303,6 +303,7 @@ class ObjectSet(MutableSet[ObjectID]): return result def __post_init__(self): + assert isinstance(self.kind, Kind) for obj in self.objects: assert isinstance(obj, ObjectID), f"Expected ObjectID, got {obj}" assert obj.kind() == self.kind @@ -325,6 +326,35 @@ class ObjectSet(MutableSet[ObjectID]): return False return self.objects == other.objects + @classmethod + def _from_iterable(cls, it): + # This functions is called by various collections.abc.Set functions to + # compose the container for merge operations (e.g. `&`). These are + # overridden below. Throw an error in the offchance that we forgot to + # implement one of them. + raise NotImplementedError() + + def _merge( + self, + other: Any, + merge_fun: Callable[[set[ObjectID], set[ObjectID]], set[ObjectID]], + ) -> ObjectSet: + assert isinstance(other, ObjectSet) + assert other.kind == self.kind + return ObjectSet(self.kind, merge_fun(self.objects, other.objects)) + + def __and__(self, other: Any) -> ObjectSet: + return self._merge(other, lambda x, y: x & y) + + def __or__(self, other: Any) -> ObjectSet: + return self._merge(other, lambda x, y: x | y) + + def __sub__(self, other: Any) -> ObjectSet: + return self._merge(other, lambda x, y: x - y) + + def __xor__(self, other: Any) -> ObjectSet: + return self._merge(other, lambda x, y: x ^ y) + def __repr__(self) -> str: return f"ObjectSet(kind={self.kind.serialize()}, objects={self.objects})" diff --git a/python/revng/pypeline/pipeline.py b/python/revng/pypeline/pipeline.py index 6d156d9d0..971fae658 100644 --- a/python/revng/pypeline/pipeline.py +++ b/python/revng/pypeline/pipeline.py @@ -161,21 +161,19 @@ class Pipeline(Generic[C]): self, start: Optional[PipelineNode] = None, forward: bool = True, stable: bool = False ) -> Generator[PipelineNode, None, None]: """BFS walk of pipeline nodes""" + assert int(not forward) + int(stable) < 2, "forward=False,stable=True is unsupported" + to_visit: List[PipelineNode] = [start or self.root] visited: Set[PipelineNode] = set() if forward: def successors(node): - if not stable: - return node.successors - else: - return node.sorted_successors() + return node.sorted_successors() if stable else node.successors else: def successors(node): - assert not stable return node.predecessors while len(to_visit) > 0: @@ -186,40 +184,77 @@ class Pipeline(Generic[C]): if child_node not in visited: to_visit.append(child_node) - def graph(self) -> Graph: + def graph(self, container_edges=False) -> Graph: """A graph for debugging purposes.""" graph = Graph() nodes_map: Dict[PipelineNode, Graph.Node] = {} - def get_node(node: PipelineNode) -> Graph.Node: - if node not in nodes_map: - new_node = Graph.Node(node.task.name) - for argument in node.arguments: - new_node.entries.append(argument.name) - - nodes_map[node] = new_node - graph.nodes.add(new_node) - - return nodes_map[node] + access_to_str = { + TaskArgumentAccess.READ: "R", + TaskArgumentAccess.WRITE: "W", + TaskArgumentAccess.READ_WRITE: "RW", + } for node in self.walk_pipeline(): - graph_node = get_node(node) - node_inputs: List[ContainerDeclaration] = list(node.arguments) + if isinstance(node.task, Pipe): + new_node = Graph.Node(node.task.name) + for argument in node.arguments_with_access: + new_node.entries.append(f"{argument.name} [{access_to_str[argument.access]}]") + else: + new_node = Graph.Node(node.task.name, color="#2A52BE", bgcolor="#6C83BE") + for argument in node.arguments_with_access: + new_node.entries.append(argument.name) + + nodes_map[node] = new_node + graph.nodes.add(new_node) for predecessor in node.predecessors: - for source_index, argument in enumerate(predecessor.task.arguments): - if argument.access == TaskArgumentAccess.READ or argument not in node_inputs: + graph.edges.add( + Graph.Edge( + nodes_map[predecessor], + new_node, + source_port=-1, + destination_port=-1, + style="bold", + ) + ) + + if not container_edges: + continue + + node_inputs: list[ContainerDeclaration] = list(node.arguments) + taken_inputs: set[int] = set() + + for parent_node in self.walk_pipeline(node, forward=False): + if parent_node is node: + continue + + for source_index, parent_argument in enumerate(parent_node.arguments_with_access): + parent_container_decl = parent_argument.to_container_decl() + if ( + parent_argument.access == TaskArgumentAccess.READ + or parent_container_decl not in node_inputs + ): + continue + + destination_index = node_inputs.index(parent_container_decl) + if destination_index in taken_inputs: + continue + + taken_inputs.add(destination_index) + if parent_node in node.predecessors: continue - destination_index = node_inputs.index(argument) graph.edges.add( Graph.Edge( - get_node(predecessor), - graph_node, + nodes_map[parent_node], + new_node, source_port=source_index, destination_port=destination_index, + style="dashed", + color="#FD6D53", ) ) @@ -266,7 +301,7 @@ class Pipeline(Generic[C]): if not node.predecessors: assert node_ingoing_requests.empty(), ( f"Node {node} has no predecessors, but it still has " - f"requests: {node_ingoing_requests}" + f"requests: {node_ingoing_requests.minimize()}" ) break @@ -281,7 +316,59 @@ class Pipeline(Generic[C]): node = predecessor node_outgoing_requests = node_ingoing_requests - return Schedule(self.declarations, tasks[target_node], pipeline_configuration) + # Here the task list has been decided, these are now iterated to apply + # the following optimizations: + # * Reduce the set of container declarations to those that are actually + # necessary to run the schedule + # * Compact incoming and outgoing of each task to their reduced version + # * Skip tasks where nothing in outgoing is actually written by the task + # + # These could be derived by using a dataflow-based approach to the + # pipeline, where each pipe binds its inputs to the predecessor's pipe + # output, but in the general case (with RW pipes) this turns into + # having a mini-programming language and applying SSA analysis and the + # like to deduplicate the containers (that, in the general case, need + # to be duplicated at each node of the dataflow). + scheduled_task: ScheduledTask | None = tasks[target_node] + parent_scheduled_task: ScheduledTask | None = None + used_declatations: set[str] = set() + + while scheduled_task is not None: + # Assume that the schedule is a straight line, so a task has at + # most one dependency + assert len(scheduled_task.depends_on) in (0, 1) + + # Reduce incoming and outgoing + scheduled_task.incoming = scheduled_task.incoming.minimize() + scheduled_task.outgoing = scheduled_task.outgoing.minimize() + + # Figure out if a task will actually produce outputs + written_out = Requests() + for argument in scheduled_task.node.arguments_with_access: + if argument.access != TaskArgumentAccess.READ: + container_decl = argument.to_container_decl() + if container_decl in scheduled_task.outgoing: + written_out[container_decl] = scheduled_task.outgoing[container_decl] + + if written_out.empty() and parent_scheduled_task is not None: + # If here the task does not actually need to produce anything + # and can be skipped, by "glueing" its dependencies onto the parent + assert [scheduled_task] == parent_scheduled_task.depends_on + parent_scheduled_task.depends_on = scheduled_task.depends_on + else: + used_declatations.update(x.name for x in scheduled_task.node.arguments) + parent_scheduled_task = scheduled_task + + if len(scheduled_task.depends_on) == 1: + scheduled_task = scheduled_task.depends_on[0] + else: + scheduled_task = None + + return Schedule( + {v for v in self.declarations if v.name in used_declatations}, + tasks[target_node], + pipeline_configuration, + ) def get_artifact( self, @@ -350,10 +437,7 @@ class Pipeline(Generic[C]): analysis_info.analysis.run( model=new_model, containers=[all_containers[decl] for decl in analysis_info.bindings], - incoming=[ - requests.get(decl, ObjectSet(decl.container_type.kind, set())) - for decl in analysis_info.bindings - ], + incoming=[requests.get(decl) for decl in analysis_info.bindings], configuration=analysis_configuration, ) invalidated = storage_provider.invalidate(ReadOnlyModel(new_model).diff(model)) diff --git a/python/revng/pypeline/pipeline_node.py b/python/revng/pypeline/pipeline_node.py index 733337cbf..a5ef5194c 100644 --- a/python/revng/pypeline/pipeline_node.py +++ b/python/revng/pypeline/pipeline_node.py @@ -8,14 +8,16 @@ from hashlib import sha256 from typing import TYPE_CHECKING, Annotated, List, Mapping, Optional, Sequence, Set, Union from typing import overload -from .container import Configuration, ConfigurationId, ContainerDeclaration, ContainerSet +from .container import Configuration, ConfigurationId, Container, ContainerDeclaration +from .container import ContainerSet from .model import ReadOnlyModel from .object import ObjectSet +from .storage.file_provider import FileProvider from .storage.storage_provider import SavePointsRange, StorageProvider, StorageProviderFileProvider from .task.pipe import Pipe from .task.requests import Requests from .task.savepoint import SavePoint -from .task.task import ObjectDependencies, TaskArgumentAccess +from .task.task import ObjectDependencies, PipeObjectDependencies, TaskArgument, TaskArgumentAccess if TYPE_CHECKING: from _typeshed import SupportsRichComparison @@ -107,23 +109,27 @@ class PipelineNode: assert len(self.bindings) == len(task.arguments) @property - def arguments(self) -> list[ContainerDeclaration]: + def arguments_with_access(self) -> list[TaskArgument]: if isinstance(self.task, SavePoint): # SavePoints do not have bindings, so we return the task arguments directly - return [ - ContainerDeclaration( - name=x.name, - container_type=x.container_type, - ) - for x in self.task.arguments - ] + return self.task.arguments elif isinstance(self.task, Pipe): # For Pipes, we return the bindings, which are the pipeline declarations # that map to the task arguments - return self.bindings + result = [] + for index, argument in enumerate(self.bindings): + pipe_arguments = self.task.arguments[index] + result.append( + TaskArgument(argument.name, argument.container_type, pipe_arguments.access) + ) + return result else: raise TypeError(f"Unsupported task type: {type(self.task)}") + @property + def arguments(self) -> list[ContainerDeclaration]: + return [x.to_container_decl() for x in self.arguments_with_access] + def add_successor(self, node: PipelineNode) -> PipelineNode: node.predecessors.append(self) self.successors.append(node) @@ -161,10 +167,7 @@ class PipelineNode: ) elif isinstance(self.task, Pipe): # Use the bindings to map the requests to the pipe arguments - pipe_requests: list[ObjectSet] = [ - requests.get(decl, ObjectSet(decl.container_type.kind, set())) - for decl in self.bindings - ] + pipe_requests: list[ObjectSet] = [requests.get(decl) for decl in self.bindings] # Ask the pipe for its prerequisites outgoing = self.task.prerequisites_for( model=model, @@ -213,14 +216,8 @@ class PipelineNode: return None elif isinstance(self.task, Pipe): pipe_containers = [containers[decl] for decl in self.bindings] - pipe_incoming = [ - incoming.get(decl, ObjectSet(decl.container_type.kind, set())) - for decl in self.bindings - ] - pipe_outgoing = [ - outgoing.get(decl, ObjectSet(decl.container_type.kind, set())) - for decl in self.bindings - ] + pipe_incoming = [incoming.get(decl) for decl in self.bindings] + pipe_outgoing = [outgoing.get(decl) for decl in self.bindings] deps = self.task.run( file_provider=StorageProviderFileProvider(storage_provider), model=model, @@ -244,4 +241,32 @@ class PipelineNode: raise TypeError(f"Unsupported task type: {type(self.task)}") def __repr__(self): - return self.task.__repr__() + return f"" + + +class DummyPipelineNode(PipelineNode): + """Dummy pipeline node, to be used in cases where multiple PipelineNode + branches need to be merged into one""" + + class DummyPipe(Pipe): + def __init__(self, name: str): + self.name = name + self.static_configuration = "" + + @classmethod + def signature(cls) -> tuple[TaskArgument, ...]: + return () + + def run( + self, + file_provider: FileProvider, + model: ReadOnlyModel, + containers: list[Container], + incoming: list[ObjectSet], + outgoing: list[ObjectSet], + configuration: Configuration, + ) -> PipeObjectDependencies: + return [] + + def __init__(self, name: str): + super().__init__(self.__class__.DummyPipe(name), []) diff --git a/python/revng/pypeline/pipeline_parser.py b/python/revng/pypeline/pipeline_parser.py index 3d41954ff..96f135749 100644 --- a/python/revng/pypeline/pipeline_parser.py +++ b/python/revng/pypeline/pipeline_parser.py @@ -5,6 +5,7 @@ from __future__ import annotations from dataclasses import dataclass, field +from graphlib import TopologicalSorter from pathlib import Path from typing import Any, Optional @@ -14,7 +15,7 @@ import yaml from .analysis import Analysis, AnalysisBinding from .container import Container, ContainerDeclaration from .pipeline import Artifact, Pipeline -from .pipeline_node import PipelineNode +from .pipeline_node import DummyPipelineNode, PipelineNode from .task.pipe import Pipe from .task.savepoint import SavePoint from .utils.registry import get_registry @@ -55,14 +56,16 @@ def parse_pipe( ) bindings.append(container_decls[arg]) - name = task.get("name") - configuration = task.get("configuration", "") + configuration = task.get("configuration") + if configuration is None: + configuration_string = "" + elif isinstance(configuration, str): + configuration_string = configuration + else: + configuration_string = yaml.safe_dump(configuration) return PipelineNode( - task=pipe_type( - name=name, - static_configuration=configuration, - ), + task=pipe_type(configuration_string), bindings=bindings, ) @@ -206,7 +209,7 @@ def parse_branch( artifacts: set[Artifact], analyses: set[AnalysisBinding], container_decls: dict[str, ContainerDeclaration], -) -> PipelineNode: +) -> tuple[PipelineNode, PipelineNode]: """ Parse the branch and return the last node in the branch. """ @@ -219,6 +222,7 @@ def parse_branch( # Parse the nodes tasks = node.content.get("tasks", []) + head = None for task in tasks: if "pipe" not in task and "savepoint" not in task: raise ValueError("Task must have either a pipe or a savepoint") @@ -233,10 +237,13 @@ def parse_branch( container_decls=container_decls, parent=parent, ) + if head is None: + head = res parent = res assert parent is not None, "A branch cannot be empty and not have a parent" - return parent + assert head is not None, "A branch needs to have at least one task" + return (head, parent) def parse_branches( @@ -244,7 +251,7 @@ def parse_branches( artifacts: set[Artifact], analyses: set[AnalysisBinding], container_decls: dict[str, ContainerDeclaration], -) -> PipelineNode: +) -> list[PipelineNode]: """ Parse the branches from the JSON value. The JSON value should contain a dictionary of branches with their names and tasks. @@ -252,6 +259,7 @@ def parse_branches( # Find the root branch graph: dict[str, Node] = {} + roots: set[str] = set() for name, branch in branches.items(): node = Node(content=branch) graph[name] = node @@ -259,44 +267,31 @@ def parse_branches( node.is_root = False from_node = graph[branch["from"]] from_node.successors.add(name) + else: + roots.add(name) - # Check that there is only one root branch - roots = sorted(name for name, node in graph.items() if node.is_root) - if len(roots) != 1: - raise ValueError(f"There should be exactly one root branch but found {roots}") - root = roots[0] + sorter: TopologicalSorter[str] = TopologicalSorter() + for name, node in graph.items(): + sorter.add(name) + for successor in node.successors: + sorter.add(successor, name) # Parse the branches in the correct order (DFS) - result: dict[str, PipelineNode] = {} - queue: list[str] = [root] - root_node: PipelineNode | None = None - while queue: - name = queue.pop(0) - node = graph[name] - # Parse the branch - result[name] = parse_branch( - node=node, - graph=result, + node_tips: dict[str, PipelineNode] = {} + result: list[PipelineNode] = [] + for name in sorter.static_order(): + first_node, last_node = parse_branch( + node=graph[name], + graph=node_tips, container_decls=container_decls, artifacts=artifacts, analyses=analyses, ) - if root_node is None: - root_node = result[name] - # Result contains the END of each branch, so we need to find the root node - # by following the predecessors - while root_node.predecessors: - root_node = root_node.predecessors[0] - # Enqueue the successors - for successor in node.successors: - assert successor not in result, ( - f"The pipeline has to be a tree, not a DAG. Branch {successor} " - "is defined multiple times" - ) - queue.append(successor) - # Return the root node - assert root_node is not None, "The root node should be defined" - return root_node + node_tips[name] = last_node + if name in roots: + result.append(first_node) + + return result def parse_container_decls( @@ -352,12 +347,20 @@ def load_pipeline(values: Any) -> Pipeline: artifacts: set[Artifact] = set() analyses: set[AnalysisBinding] = set() - root: PipelineNode = parse_branches( + roots: list[PipelineNode] = parse_branches( branches=values["branches"], container_decls=container_decls, artifacts=artifacts, analyses=analyses, ) + + if len(roots) == 1: + root = roots[0] + else: + root = DummyPipelineNode("root") + for node in roots: + root.add_successor(node) + return Pipeline( declarations=set(container_decls.values()), root=root, diff --git a/python/revng/pypeline/schedule/schedule.py b/python/revng/pypeline/schedule/schedule.py index 7a61b9358..444e5ef48 100644 --- a/python/revng/pypeline/schedule/schedule.py +++ b/python/revng/pypeline/schedule/schedule.py @@ -59,7 +59,8 @@ class Schedule: def get_node(node: ScheduledTask) -> Graph.Node: if node not in nodes_map: new_node = Graph.Node(node.node.task.name) - new_node.completed = node.completed + if node.completed: + new_node.bgcolor = "lightgreen" for argument in node.node.arguments: new_node.entries.append(argument.name) @@ -104,6 +105,10 @@ class Schedule: model: ReadOnlyModel, storage_provider: StorageProvider, ) -> ContainerSet: + for task in self.tasks: + if isinstance(task.node.task, Pipe): + task.node.task.check_precondition(model) + # Produce a set of working containers working_containers: ContainerSet = { declaration: declaration.instance() for declaration in self.declarations @@ -181,8 +186,8 @@ class Schedule: args = [] for declaration in task.node.bindings: - incoming = [x.serialize() for x in task.incoming.get(declaration, [])] - outgoing = [x.serialize() for x in task.outgoing.get(declaration, [])] + incoming = [x.serialize() for x in task.incoming.get(declaration)] + outgoing = [x.serialize() for x in task.outgoing.get(declaration)] args.append( {"name": declaration.name, "incoming": incoming, "outgoing": outgoing} ) @@ -203,8 +208,8 @@ class Schedule: sp_containers = [] for declaration in self.declarations: - incoming = [x.serialize() for x in task.incoming.get(declaration, [])] - outgoing = [x.serialize() for x in task.outgoing.get(declaration, [])] + incoming = [x.serialize() for x in task.incoming.get(declaration)] + outgoing = [x.serialize() for x in task.outgoing.get(declaration)] if len(incoming) == 0 and len(outgoing) == 0: continue diff --git a/python/revng/pypeline/schedule/scheduled_task.py b/python/revng/pypeline/schedule/scheduled_task.py index cf0b5f5dd..de8835d30 100644 --- a/python/revng/pypeline/schedule/scheduled_task.py +++ b/python/revng/pypeline/schedule/scheduled_task.py @@ -51,7 +51,7 @@ class ScheduledTask: to check that the task can run. """ - depends_on: list[ScheduledTask] = field(default_factory=list) + depends_on: list[ScheduledTask] = field(default_factory=list, repr=False) """ These are the scheduled tasks that this task depends on, after the scheduling phase this list will contain only 0 or 1 elements as it's a path, but diff --git a/python/revng/pypeline/schema.yml b/python/revng/pypeline/schema.yml index db3a61087..715862622 100644 --- a/python/revng/pypeline/schema.yml +++ b/python/revng/pypeline/schema.yml @@ -19,8 +19,6 @@ $defs: $ref: "#/$defs/container_decl" branches: type: object - required: - - root patternProperties: ^[a-zA-Z0-9_]+$: type: object @@ -71,19 +69,16 @@ $defs: - pipe - arguments properties: - name: - type: string - description: The name of the pipe task. - example: my_pipe_task pipe: type: string description: The name of the pipe. example: my_pipe configuration: - type: string - description: The static configuration for the pipe, if any. - default: "" - example: intel_asm=1 + type: [string, object] + description: | + The static configuration for the pipe, if any. Can be either a string + or an object. If an object the content will be re-serialized in YAML + and passed to the pipe constructor as a string. arguments: $ref: "#/$defs/args" description: The names of the containers to pass to the pipe. diff --git a/python/revng/pypeline/task/pipe.py b/python/revng/pypeline/task/pipe.py index d4d6e6be5..ad5e23506 100644 --- a/python/revng/pypeline/task/pipe.py +++ b/python/revng/pypeline/task/pipe.py @@ -38,12 +38,8 @@ class Pipe(ABC): argument should not be added. """ - def __init__( - self, - static_configuration: str = "", - name: str | None = None, - ): - self.name: str = name or self.__class__.__name__ + def __init__(self, static_configuration: str = ""): + self.name = self.__class__.__name__ self.static_configuration: str = static_configuration @property @@ -87,15 +83,7 @@ class Pipe(ABC): if decl.access == TaskArgumentAccess.WRITE: continue - for ridx, object_list in enumerate(requests): - # The requests must be for the writeable containers, - # otherwise it's not possible to satisfy them - if self.arguments[ridx].access == TaskArgumentAccess.READ: - assert len(object_list) == 0, ( - f"Expected an empty request for {self.arguments[ridx].name}, " - f"but got {object_list}." - ) - + for object_list in requests: result[idx].update( model.move_to_kind( object_list, @@ -105,6 +93,14 @@ class Pipe(ABC): return result + def check_precondition(self, model: ReadOnlyModel): + """ + Checks that the pipe can be run successfully with the provided model. + Subclasses can optionally override this method if they wish to perform + checks before the `run` method. An exception should be thrown if some + property of the model would not allow running the pipe correctly. + """ + @abstractmethod def run( self, diff --git a/python/revng/pypeline/task/requests.py b/python/revng/pypeline/task/requests.py index 7559d6bdc..2c9afb21a 100644 --- a/python/revng/pypeline/task/requests.py +++ b/python/revng/pypeline/task/requests.py @@ -4,11 +4,14 @@ from __future__ import annotations -from typing import Dict, MutableMapping, Optional, Set +from typing import Dict, MutableMapping, Optional, Set, TypeVar, cast, overload from revng.pypeline.container import ContainerDeclaration, ContainerSet from revng.pypeline.object import ObjectSet +_DUMMY = object() +T = TypeVar("T") + class Requests(MutableMapping[ContainerDeclaration, ObjectSet]): """ @@ -76,6 +79,20 @@ class Requests(MutableMapping[ContainerDeclaration, ObjectSet]): def __getitem__(self, key: ContainerDeclaration) -> ObjectSet: return self.requests[key] + @overload + def get(self, key: ContainerDeclaration, /) -> ObjectSet: ... + + @overload + def get(self, key: ContainerDeclaration, default: T, /) -> ObjectSet | T: ... + + def get(self, key: ContainerDeclaration, default: T | object = _DUMMY, /) -> ObjectSet | T: + if key in self.requests: + return self.requests[key] + elif default is _DUMMY: + return ObjectSet(key.container_type.kind) + else: + return cast(T, default) + def __setitem__(self, key: ContainerDeclaration, value: ObjectSet) -> None: if not isinstance(value, ObjectSet): raise TypeError(f"Expected ObjectSet, got {type(value)}") @@ -104,3 +121,7 @@ class Requests(MutableMapping[ContainerDeclaration, ObjectSet]): def __eq__(self, other) -> bool: return self.requests == other.requests + + def minimize(self) -> Requests: + """Return the same request object, but with all the empty values removed""" + return Requests({k: v for k, v in self.requests.items() if len(v) > 0}) diff --git a/python/revng/pypeline/task/savepoint.py b/python/revng/pypeline/task/savepoint.py index 5853c6689..2f174c902 100644 --- a/python/revng/pypeline/task/savepoint.py +++ b/python/revng/pypeline/task/savepoint.py @@ -84,31 +84,20 @@ class SavePoint: set(incoming.keys()) | set(outgoing.keys()) ), "SavePoint containers must be a subset of incoming and outgoing requests." - # Cache the containers present for this configuration for decl in self.to_save: - if decl not in incoming: - continue - if len(incoming[decl]) == 0: - continue - storage_provider.put( - ContainerLocation( - savepoint_id=savepoint_range.start, - container_id=decl.name, - configuration_id=configuration_id, - ), - containers[decl].serialize(incoming[decl]), - ) - - # Fill the containers from our cache - for decl in self.to_save: - if decl not in outgoing: - continue - # Empty requests do not need to be restored - if len(outgoing[decl]) == 0: - continue location = ContainerLocation( savepoint_id=savepoint_range.start, container_id=decl.name, configuration_id=configuration_id, ) - containers[decl].deserialize(storage_provider.get(location, outgoing[decl])) + + container_incoming = incoming.get(decl) + if len(container_incoming) > 0: + storage_provider.put(location, containers[decl].serialize(container_incoming)) + + # Compute the actual set of objects to load, if an object has + # already been saved from incoming do not re-load it from storage + container_outgoing = outgoing.get(decl) - container_incoming + + if len(container_outgoing) > 0: + containers[decl].deserialize(storage_provider.get(location, container_outgoing)) diff --git a/share/revng/test/configuration/revng/pypeline-comparison.yml b/share/revng/test/configuration/revng/pypeline-comparison.yml new file mode 100644 index 000000000..40e6402ff --- /dev/null +++ b/share/revng/test/configuration/revng/pypeline-comparison.yml @@ -0,0 +1,61 @@ +# +# This file is distributed under the MIT License. See LICENSE.md for details. +# + +commands: + # Prepare a project directory to run the tests + - type: revng.test-pypeline-comparison-prepare + from: + - type: revng-qa.compiled + filter: example-executable-1 and with-debug-info + suffix: / + command: |- + "${SOURCES_ROOT}/share/revng/test/tests/pypeline-comparison/prepare.py" "$INPUT" "$OUTPUT"; + + # Run lift comparison + - type: revng.test-pypeline-comparison-lift + from: + - type: revng.test-pypeline-comparison-prepare + command: |- + cd "$INPUT"; + "${SOURCES_ROOT}/share/revng/test/tests/pypeline-comparison/compare.sh" model.yml input lift; + + # Run disassemble comparison + - type: revng.test-pypeline-comparison-disassemble + from: + - type: revng.test-pypeline-comparison-prepare + command: |- + cd "$INPUT"; + "${SOURCES_ROOT}/share/revng/test/tests/pypeline-comparison/compare.sh" model.yml input disassemble; + + # Run emit-model-header comparison + - type: revng.test-pypeline-comparison-emit-model-header + from: + - type: revng.test-pypeline-comparison-prepare + command: |- + cd "$INPUT"; + "${SOURCES_ROOT}/share/revng/test/tests/pypeline-comparison/compare.sh" model.yml input emit-model-header; + + # Run emit-type-definitions comparison + - type: revng.test-pypeline-comparison-emit-type-definitions + from: + - type: revng.test-pypeline-comparison-prepare + command: |- + cd "$INPUT"; + "${SOURCES_ROOT}/share/revng/test/tests/pypeline-comparison/compare.sh" model.yml input emit-type-definitions; + + # Run hexdump comparison + - type: revng.test-pypeline-comparison-hexdump + from: + - type: revng.test-pypeline-comparison-prepare + command: |- + cd "$INPUT"; + "${SOURCES_ROOT}/share/revng/test/tests/pypeline-comparison/compare.sh" model.yml input hexdump; + + # Run isolate comparison + - type: revng.test-pypeline-comparison-isolate + from: + - type: revng.test-pypeline-comparison-prepare + command: |- + cd "$INPUT"; + "${SOURCES_ROOT}/share/revng/test/tests/pypeline-comparison/compare_isolate.sh" model.yml input; diff --git a/share/revng/test/tests/abi/verify-abi-test-binaries.sh b/share/revng/test/tests/abi/verify-abi-test-binaries.sh index 4d5c75738..c5a4f0892 100755 --- a/share/revng/test/tests/abi/verify-abi-test-binaries.sh +++ b/share/revng/test/tests/abi/verify-abi-test-binaries.sh @@ -18,8 +18,7 @@ test -n "$OUTPUT_DIRECTORY" SCRIPT_DIRECTORY="$( dirname -- "$( readlink -f -- "$0"; )"; )" -orc shell \ - llvm-dwarfdump \ +llvm-dwarfdump \ "$BINARY" \ > "$OUTPUT_DIRECTORY/dwarf.dump" diff --git a/share/revng/test/tests/pypeline-comparison/compare.sh b/share/revng/test/tests/pypeline-comparison/compare.sh new file mode 100755 index 000000000..017242a61 --- /dev/null +++ b/share/revng/test/tests/pypeline-comparison/compare.sh @@ -0,0 +1,137 @@ +#!/usr/bin/env bash + +# +# This file is distributed under the MIT License. See LICENSE.md for details. +# + +set -euo pipefail + +WORKDIR=$(mktemp --tmpdir -d tmp.revng-pypeline-compare.XXXXXXXXXX) +trap 'rm -r "$WORKDIR"' EXIT + +MODEL="$1" +BINARY="$2" +ARTIFACT="$3" + +OLD_PATH="$WORKDIR/old" +NEW_PATH="$WORKDIR/new" +mkdir "$WORKDIR/cache" +export REVNG_CACHE_DIR="$WORKDIR/cache" + + +function files_in_dir() { + find "$1" -type f -printf '%f\n' +} + +# JQ appends a newline '\n' to the output, this command runs it and strips it +function jq_exact() { + jq "$@" | head -c -1 +} + +# Comparison between multiple artifact objects (e.g. function, type-definition) +function compare() { + local PREFIX="$1" + local OLD_EXTRACTED NEW_EXTRACTED OLD_FILES + OLD_EXTRACTED="$WORKDIR/old_extracted" + NEW_EXTRACTED="$WORKDIR/new_extracted" + mkdir "$OLD_EXTRACTED" "$NEW_EXTRACTED" + + # Extract all the files from the tar (old format), rename them without all + # the extensions + tar -C"$OLD_EXTRACTED" -xf "$OLD_PATH" + readarray -t OLD_FILES < <(files_in_dir "$OLD_EXTRACTED") + + local FILE + for FILE in "${OLD_FILES[@]}"; do + local KEY_EXTRACTED + KEY_EXTRACTED=$(cut -d. -f1 <<< "$FILE") + mv "$OLD_EXTRACTED/$FILE" "$OLD_EXTRACTED/$KEY_EXTRACTED" + done + + # Extract all the files from the JSON (new format), strip away the prefix + # from each entry + local KEY + jq -r '. | keys | .[]' "$NEW_PATH" | \ + while IFS= read -r KEY; do + local KEY_EXTRACTED="${KEY/#$PREFIX/}" + jq_exact -r ".[\"$KEY\"]" "$NEW_PATH" > "$NEW_EXTRACTED/$KEY_EXTRACTED" + done + + if ! diff <(files_in_dir "$OLD_EXTRACTED" | sort) <(files_in_dir "$NEW_EXTRACTED" | sort); then + echo "File list mismatch!" + return 1 + fi + + # Compare the files for each element + local OK=0 + while IFS= read -r KEY; do + local DIFF_OUTPUT RC=0 + DIFF_OUTPUT=$(mktemp --tmpdir="$WORKDIR") + + diff -u "$OLD_EXTRACTED/$KEY" "$NEW_EXTRACTED/$KEY" > "$DIFF_OUTPUT" || RC=$? + if [[ "$RC" -ne 0 ]]; then + OK=1 + echo "Comparison for $ARTIFACT: $KEY failed" + cat "$DIFF_OUTPUT" + fi + done < <(files_in_dir "$OLD_EXTRACTED") + + return "$OK" +} + + +# Comparison of the `lift` artifact, this is special because the two formats +# are quite different and need to be converted to `.ll` before comparing +function compare_lift() { + # Convert old + zstdcat "$OLD_PATH" | revng opt -S > "$WORKDIR/old.ll" + # Convert new + jq_exact -r '.["/binary"]' "$NEW_PATH" | base64 -d | revng opt -S > "$WORKDIR/new.ll" + + local DIFF_OUTPUT RC=0 + DIFF_OUTPUT=$(mktemp --tmpdir="$WORKDIR") + diff -u "$WORKDIR/old.ll" "$WORKDIR/new.ll" > "$DIFF_OUTPUT" || RC=$? + if [[ "$RC" -ne 0 ]]; then + echo "Comparison for $ARTIFACT failed!" + cat "$DIFF_OUTPUT" + fi + + return "$RC" +} + +# Comparison between single-valued artifacts +function compare_binary() { + local DIFF_OUTPUT RC=0 + DIFF_OUTPUT=$(mktemp --tmpdir="$WORKDIR") + + diff -u "$OLD_PATH" <(jq_exact -r '.["/binary"]' "$NEW_PATH") > "$DIFF_OUTPUT" || RC=$? + if [[ "$RC" -ne 0 ]]; then + echo "Comparison for $ARTIFACT failed!" + cat "$DIFF_OUTPUT" + fi + + return "$RC" +} + +# Compute the artifacts from both the old and new pipeline +revng artifact "$ARTIFACT" --model="$MODEL" "$BINARY" -o "$OLD_PATH" +revng2 project artifact "$ARTIFACT" -o "$NEW_PATH" 2>/dev/null + +RC=0 +# NOTE: this could be derived from `pipeline-description.yml`, but since this +# is temporary they are hardcorded here. +if [[ "$ARTIFACT" = "disassemble" ]]; then + compare "/function/" || RC=$? +elif [[ "$ARTIFACT" = "emit-type-definitions" ]]; then + compare "/type-definition/" || RC=$? +elif [[ "$ARTIFACT" = "lift" ]]; then + compare_lift || RC=$? +else + compare_binary || RC=$? +fi + +if [[ "$RC" -ne 0 ]]; then + echo "Comparison for $ARTIFACT failed!" +fi + +exit "$RC" diff --git a/share/revng/test/tests/pypeline-comparison/compare_isolate.sh b/share/revng/test/tests/pypeline-comparison/compare_isolate.sh new file mode 100755 index 000000000..1cb73c1d9 --- /dev/null +++ b/share/revng/test/tests/pypeline-comparison/compare_isolate.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash + +# +# This file is distributed under the MIT License. See LICENSE.md for details. +# + +set -euo pipefail + +WORKDIR=$(mktemp --tmpdir -d tmp.revng-pypeline-compare.XXXXXXXXXX) +trap 'rm -r "$WORKDIR"' EXIT + +MODEL="$1" +BINARY="$2" + +mkdir "$WORKDIR/cache" "$WORKDIR/old" "$WORKDIR/new" +export REVNG_CACHE_DIR="$WORKDIR/cache" + +# This function takes a bitcode file and normalizes it a bit, with the following: +# * Run it through globaldce and convert it to textual IR +# * All metadata IDs are blanked to '!0' +# * Metadata declarations are dropped +# * Empty lines and lines starting with comments are dropped +function normalize() { + revng opt -globaldce -S | sed 's;![0-9]\+;!0;g' | grep -v -e '^!0 = ' -e '^\s*$' -e '^;' +} + +OK=0 +while IFS= read -r FUNCTION; do + # When working with an LLVMContext, types are pooled into the context. This + # poses a problem when adding a type with the same name twice; to fix this + # the context automatically appends a `.[0-9]+` suffix to these to + # disambiguate. Since this test relies on the diff being identical, both + # old and new pipeline commands need to be run individually on each + # function, otherwise the types will have the numeric suffix and the + # comparison will fail. + + revng artifact isolate --model="$MODEL" "$BINARY" "$FUNCTION" | \ + zstdcat | normalize > "$WORKDIR/old/$FUNCTION.ll" + + OBJECT_ID="/function/$FUNCTION" + revng2 project artifact isolate "$OBJECT_ID" 2>/dev/null | \ + jq -r ".[\"/function/$FUNCTION\"]" | base64 -d | normalize > "$WORKDIR/new/$FUNCTION.ll" + + DIFF_OUTPUT="$WORKDIR/diff_output_$FUNCTION" + RC=0 + diff -u "$WORKDIR/old/$FUNCTION.ll" "$WORKDIR/new/$FUNCTION.ll" > "$DIFF_OUTPUT" || RC=$? + if [[ "$RC" -ne 0 ]]; then + echo "Comparison failed for $FUNCTION" + cat "$DIFF_OUTPUT" + OK=1 + fi +done < <(yq -r '.Functions[].Entry' "$MODEL") + +exit "$OK" diff --git a/share/revng/test/tests/pypeline-comparison/prepare.py b/share/revng/test/tests/pypeline-comparison/prepare.py new file mode 100755 index 000000000..0bf9bed33 --- /dev/null +++ b/share/revng/test/tests/pypeline-comparison/prepare.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 + +# +# This file is distributed under the MIT License. See LICENSE.md for details. +# + +import os +import sys +from hashlib import file_digest +from pathlib import Path +from shutil import copy +from subprocess import run +from tempfile import TemporaryFile + +import yaml + + +# This script prepares the output directory for `revng project ...` commands to +# be run. Eventually it will be replaced by `revng project init` once that's +# working. +def main(): + input_binary = Path(sys.argv[1]) + output_dir = Path(sys.argv[2]) + + with TemporaryFile("wb+") as temp_model: + # Generate a standard model file from the binary, via initial auto-analysis + run(["revng", "analyze", "revng-initial-auto-analysis", input_binary], stdout=temp_model) + temp_model.seek(0) + model = yaml.safe_load(temp_model) + + with open(input_binary, "rb") as f: + hash_ = file_digest(f, "sha256").hexdigest() + size = f.seek(0, os.SEEK_END) + + # Patch the `Binaries` entry with the input file + model["Binaries"] = [{"Index": 0, "Hash": hash_, "Size": size, "Name": "input"}] + # Add a reference to the newly-added binary to all the segments + for segment in model["Segments"]: + segment["Binary"] = "/Binaries/0" + + with open(output_dir / "model.yml", "w") as model_out: + yaml.safe_dump(model, model_out) + + copy(input_binary, output_dir / "input") + + +if __name__ == "__main__": + main() diff --git a/tests/pypeline/MockPipeboxImpl.h b/tests/pypeline/MockPipeboxImpl.h index 365611535..292a81a85 100644 --- a/tests/pypeline/MockPipeboxImpl.h +++ b/tests/pypeline/MockPipeboxImpl.h @@ -16,6 +16,8 @@ private: public: static constexpr llvm::StringRef Name = "StringContainer"; static constexpr Kind Kind = Kinds::Function; + static constexpr llvm::StringRef MimeType = "application/x-unknown"; + std::set objects() const; void deserialize(const std::map> Data); @@ -28,10 +30,12 @@ public: }; class AppendFooPipe { +private: + using Access = revng::pypeline::Access; + public: static constexpr llvm::StringRef Name = "AppendFooPipe"; - using ArgumentsDocumentation = TypeList< - revng::pypeline::PipeArgumentDocumentation<"Container", "">>; + using Arguments = TypeList>; const std::string StaticConfiguration; diff --git a/tests/pypeline/pipebox.py b/tests/pypeline/pipebox.py index 6e32ed98d..063aea413 100644 --- a/tests/pypeline/pipebox.py +++ b/tests/pypeline/pipebox.py @@ -485,15 +485,8 @@ class GeneratorPipe(Pipe): ), ) - def __init__( - self, - static_configuration: str = "", - name: str | None = None, - ): - super().__init__( - name=name, - static_configuration=static_configuration, - ) + def __init__(self, static_configuration: str = ""): + super().__init__(static_configuration) def run( self, diff --git a/tests/pypeline/python_cpp_test.py b/tests/pypeline/python_cpp_test.py index 8e6c768c4..f73afbb3f 100755 --- a/tests/pypeline/python_cpp_test.py +++ b/tests/pypeline/python_cpp_test.py @@ -30,7 +30,7 @@ def check_names(ext): """Check that _pipebox has all the classes we expect it to have""" # Names that we know are always present in `_pipebox` - known_names = ("Buffer",) + known_names = ("Buffer", "initialize") names = [ x for x in dir(ext) if not ((x.startswith("__") and x.endswith("__")) or x in known_names) @@ -40,6 +40,8 @@ def check_names(ext): for type_ in (Analysis, Container, Kind, Model, ObjectID, Pipe): registry = get_registry(type_) for key, value in registry.items(): + if key == "DummyPipe": + continue assert issubclass(value, type_) assert key in names assert getattr(ext, key) is value @@ -207,6 +209,7 @@ def check_simple_pipeline(): def main(): ext, _ = import_pipebox(sys.argv[1:]) + ext.initialize(set(), set(), []) initialize_pypeline() check_names(ext) check_pipeline()