diff --git a/CMakeLists.txt b/CMakeLists.txt index c59b6cd37..f739e7998 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -282,7 +282,16 @@ set(PYTHON_GENERATED_MODEL_PATH revng/model/_generated.py) # enable_testing() -find_package(Python REQUIRED) +# +# Find Python using the FindPython module. We request two components: * +# Interpreter: needed for some variables, such as $Python_SITELIB * Development: +# needed to compile nanobind module and friends, provides the +# $Python_INCLUDE_DIRS and $Python_LIBRARIES variables +# +find_package( + Python + COMPONENTS Interpreter Development + REQUIRED) file(RELATIVE_PATH PYTHON_INSTALL_PATH "${CMAKE_INSTALL_PREFIX}" "${Python_SITELIB}") diff --git a/include/revng/ADT/CompilationTime.h b/include/revng/ADT/CompilationTime.h index 8a2d23d77..0567e3de2 100644 --- a/include/revng/ADT/CompilationTime.h +++ b/include/revng/ADT/CompilationTime.h @@ -11,6 +11,8 @@ #include "llvm/ADT/StringRef.h" +#include "revng/ADT/Concepts.h" + namespace compile_time { namespace detail { @@ -94,6 +96,15 @@ constexpr std::optional select(CallableType &&Callable) { std::forward(Callable)); } +/// Calls \ref Callable on each element of \ref TupleType. Each time the element +/// type and index will be provided as template parameters. +template TupleType, typename CallableType> +constexpr void forEach(CallableType &&Callable) { + repeat>([&Callable]() { + Callable.template operator(), I>(); + }); +} + namespace detail { template @@ -134,4 +145,28 @@ split(llvm::StringRef Separator, llvm::StringRef Input) { return std::nullopt; } +namespace detail { + +template +struct ArrayTraits {}; + +template +struct ArrayTraits { + using value_type = T; + static constexpr size_t Size = N; +}; + +template +struct ArrayTraits> { + using value_type = T; + static constexpr size_t Size = N; +}; + +} // namespace detail + +/// Helper struct that reports the value_type and Size of an array at +/// compile-time +template +using ArrayTraits = detail::ArrayTraits>; + } // namespace compile_time diff --git a/include/revng/PipeboxCommon/Common.h b/include/revng/PipeboxCommon/Common.h new file mode 100644 index 000000000..342ed0ecd --- /dev/null +++ b/include/revng/PipeboxCommon/Common.h @@ -0,0 +1,50 @@ +#pragma once + +// +// This file is distributed under the MIT License. See LICENSE.md for details. +// + +#include + +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringRef.h" + +#include "revng/PipeboxCommon/ObjectID.h" + +namespace revng::pypeline { + +/// Type used for both incoming and outgoing requests +/// Each index maps to the i-th container passed to the 'run' function +using Request = std::vector>; +/// Type representing a path in the model, returned in a set when an analysis +/// runs +using ModelPath = std::string; +/// Type used to return the dependencies of objects produced by a pipe +/// The first index maps to the i-th container of the pipe +using ObjectDependencies = std::vector< + std::vector>>; + +/// Description of a single Pipe argument +struct PipeArgumentDocumentation { + /// Pretty-name for the argument, e.g. to show in the CLI + llvm::StringRef Name; + /// Long description of the container, to be used in help texts + llvm::StringRef HelpText; +}; + +class Buffer { +private: + llvm::SmallVector Vector; + +public: + template + Buffer(T... Args) : Vector(std::forward(Args)...) {} + + llvm::SmallVector &data() { return Vector; } + + // Read the contents + llvm::ArrayRef data() const { return { Vector.data(), Vector.size() }; } +}; + +} // namespace revng::pypeline diff --git a/include/revng/PipeboxCommon/Concepts.h b/include/revng/PipeboxCommon/Concepts.h new file mode 100644 index 000000000..af3d52a6d --- /dev/null +++ b/include/revng/PipeboxCommon/Concepts.h @@ -0,0 +1,123 @@ +#pragma once + +// +// This file is distributed under the MIT License. See LICENSE.md for details. +// + +#include + +#include "llvm/ADT/StringRef.h" + +#include "revng/PipeboxCommon/Common.h" +#include "revng/PipeboxCommon/Model.h" +#include "revng/PipeboxCommon/ObjectID.h" + +template +concept HasName = requires { + { T::Name } -> std::same_as; + requires not T::Name.empty(); +}; + +// +// IsContainer +// + +template +concept IsContainer = requires(T &A, const T &AConst) { + requires HasName; + { T() } -> std::same_as; + { T::Kind } -> std::same_as; + { AConst.objects() } -> std::same_as>; + { AConst.verify() } -> std::same_as; + { + A.deserialize(std::declval>>()) + } -> std::same_as; + { + AConst.serialize(std::declval>()) + } -> std::same_as>; +}; + +namespace detail { + +template +constexpr bool IsContainerReference = false; + +template +constexpr bool IsContainerReference = IsContainer; + +template +constexpr bool IsContainerReference = IsContainer; + +} // namespace detail + +// +// IsAnalysis +// + +namespace detail { + +template +struct AnalysisRunTraits {}; + +template + requires(IsContainerReference and ...) +struct AnalysisRunTraits { + using ContainerTypes = std::tuple...>; + static constexpr size_t Size = sizeof...(Args); +}; + +} // namespace detail + +template +using AnalysisRunTraits = detail::AnalysisRunTraits; + +template +concept IsAnalysis = requires(T &A) { + requires HasName; + { T() } -> std::same_as; + requires AnalysisRunTraits::Size >= 0; +}; + +// +// IsPipe +// + +namespace detail { + +template +struct PipeRunTraits {}; + +template + requires(IsContainerReference and ...) +struct PipeRunTraits< + revng::pypeline::ObjectDependencies (C::*)(const Model &, + const revng::pypeline::Request &, + const revng::pypeline::Request &, + llvm::StringRef, + Args...)> { + using ContainerTypes = std::tuple...>; + static constexpr size_t Size = sizeof...(Args); +}; + +template +using DocTraits = compile_time::ArrayTraits; + +} // namespace detail + +template +using PipeRunTraits = detail::PipeRunTraits; + +template +concept IsPipe = requires(T &A, llvm::StringRef StaticConfig) { + requires HasName; + { T(StaticConfig) } -> std::same_as; + { A.StaticConfiguration } -> std::same_as; + requires std::is_array_v; + requires std::is_same_v::value_type, + const revng::pypeline::PipeArgumentDocumentation>; + requires PipeRunTraits::Size == detail::DocTraits::Size; +}; diff --git a/include/revng/PipeboxCommon/Helpers/AnalysisRunner.h b/include/revng/PipeboxCommon/Helpers/AnalysisRunner.h new file mode 100644 index 000000000..1c7f763c8 --- /dev/null +++ b/include/revng/PipeboxCommon/Helpers/AnalysisRunner.h @@ -0,0 +1,57 @@ +#pragma once + +// +// This file is distributed under the MIT License. See LICENSE.md for details. +// + +#include + +#include "llvm/Support/Error.h" + +#include "revng/PipeboxCommon/Common.h" +#include "revng/PipeboxCommon/Concepts.h" +#include "revng/PipeboxCommon/Model.h" + +namespace revng::pypeline::helpers { + +/// Helper class that allows running an analysis +// The main hiccup in doing so is to unpack the arguments which are in some +// kind of sequence (e.g. std::vector) into arguments for the Analysis::run +// method. The type of the Analysis, the type of the sequence and how to unpack +// them are conveyed through the Info type, which cannot be a function pointer +// due to the unpacking function requiring template parameters. +template +struct AnalysisRunner { +private: + template + using integer_sequence = std::integer_sequence; + + using ListType = ContainerListUnwrapper::ListType; + +public: + template + static llvm::Error run(T &Analysis, + llvm::Error (T::*RunMethod)(Model &, + const pypeline::Request &, + llvm::StringRef, + ContainersT...), + Model &TheModel, + const pypeline::Request &Incoming, + llvm::StringRef Configuration, + ListType Containers) { + revng_assert(Incoming.size() == sizeof...(ContainersT)); + + auto Runner = ([&](const integer_sequence< + ContainerIndexes...> &) { + return (Analysis.*RunMethod)(TheModel, + Incoming, + Configuration, + ContainerListUnwrapper::template unwrap< + ContainersT, + ContainerIndexes>(Containers)...); + }); + return Runner(std::make_integer_sequence()); + } +}; + +} // namespace revng::pypeline::helpers diff --git a/include/revng/PipeboxCommon/Helpers/PipeRunner.h b/include/revng/PipeboxCommon/Helpers/PipeRunner.h new file mode 100644 index 000000000..569478c43 --- /dev/null +++ b/include/revng/PipeboxCommon/Helpers/PipeRunner.h @@ -0,0 +1,60 @@ +#pragma once + +// +// This file is distributed under the MIT License. See LICENSE.md for details. +// + +#include + +#include "revng/PipeboxCommon/Common.h" +#include "revng/PipeboxCommon/Concepts.h" +#include "revng/PipeboxCommon/Model.h" + +namespace revng::pypeline::helpers { + +/// Helper class that allows running a pipe +// The main hiccup in doing so is to unpack the arguments which are in some +// kind of sequence (e.g. std::vector) into arguments for the Pipe::run +// method. The type of the Pipe, the type of the sequence and how to unpack +// them are conveyed through the Info type, which cannot be a function pointer +// due to the unpacking function requiring template parameters. +template +struct PipeRunner { +private: + template + using integer_sequence = std::integer_sequence; + using ObjectDeps = pypeline::ObjectDependencies; + + using ListType = ContainerListUnwrapper::ListType; + +public: + template + static ObjectDeps run(T &Pipe, + ObjectDeps (T::*RunMethod)(const Model &, + const pypeline::Request &, + const pypeline::Request &, + llvm::StringRef, + ContainersT...), + const Model &TheModel, + const pypeline::Request &Incoming, + const pypeline::Request &Outgoing, + llvm::StringRef Configuration, + ListType Containers) { + revng_assert(Incoming.size() == sizeof...(ContainersT)); + revng_assert(Outgoing.size() == sizeof...(ContainersT)); + + auto Runner = ([&](const integer_sequence< + ContainerIndexes...> &) { + return (Pipe.*RunMethod)(TheModel, + Incoming, + Outgoing, + Configuration, + ContainerListUnwrapper::template unwrap< + ContainersT, + ContainerIndexes>(Containers)...); + }); + return Runner(std::make_integer_sequence()); + } +}; + +} // namespace revng::pypeline::helpers diff --git a/include/revng/PipeboxCommon/Helpers/Python/Casters.h b/include/revng/PipeboxCommon/Helpers/Python/Casters.h new file mode 100644 index 000000000..80f68d3ca --- /dev/null +++ b/include/revng/PipeboxCommon/Helpers/Python/Casters.h @@ -0,0 +1,123 @@ +#pragma once + +// +// This file is distributed under the MIT License. See LICENSE.md for details. +// + +#include "nanobind/nanobind.h" + +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/Error.h" + +#include "revng/PipeboxCommon/Common.h" +#include "revng/PipeboxCommon/Helpers/Python/Helpers.h" + +NAMESPACE_BEGIN(NB_NAMESPACE) +NAMESPACE_BEGIN(detail) + +template<> +struct type_caster { + NB_TYPE_CASTER(revng::pypeline::Request, + const_name("list[revng.pypeline.object.ObjectSet]")) + + bool from_python(handle Source, uint8_t, cleanup_list *) { + using namespace revng::pypeline::helpers::python; + nanobind::object ObjectSet = importObject("revng.pypeline.object." + "ObjectSet"); + + revng_assert(nanobind::isinstance(Source)); + for (const auto &OuterElement : Source) { + revng_assert(nanobind::isinstance(OuterElement, ObjectSet)); + nanobind::object OuterSet = nanobind::getattr(OuterElement, "objects"); + + std::vector Chunk; + for (const auto &Element : OuterSet) + Chunk.push_back(nanobind::cast(Element)); + + value.push_back(Chunk); + } + + return true; + } + + // from_cpp is not implemented since we don't support returning Request + // from C++ +}; + +/// This is a caster class, it allows nanobind to automatically convert python +/// strings to `llvm::StringRef`s and back +/// +/// Copied from nanobind's caster implementation for std::string_view +/// See the header "nanobind/stl/string_view.h" +template<> +struct type_caster { + NB_TYPE_CASTER(llvm::StringRef, const_name("str")) + + bool from_python(handle Source, uint8_t, cleanup_list *) { + Py_ssize_t Size = 0; + const char *String = PyUnicode_AsUTF8AndSize(Source.ptr(), &Size); + if (String == NULL) { + PyErr_Clear(); + return false; + } + value = llvm::StringRef(String, Size); + return true; + } + + static handle from_cpp(llvm::StringRef Value, rv_policy, cleanup_list *) { + return PyUnicode_FromStringAndSize(Value.data(), Value.size()); + } +}; + +namespace detail { + +inline void llvmErrorToPythonException(llvm::Error &&Error) { + std::string Message = llvm::toString(std::move(Error)); + // TODO: specialize the exception based on the returned llvm::Error + PyErr_SetString(PyExc_RuntimeError, Message.c_str()); +} + +} // namespace detail + +/// This is a caster class for llvm::Error, it allows converting functions +/// returning it into a thrown python exception +template<> +struct type_caster { + NB_TYPE_CASTER(llvm::Error, const_name("None")) + + // from_python is intentionally not defined because we do not support passing + // `llvm::Error`s as arguments + + static handle from_cpp(llvm::Error &&Error, rv_policy, cleanup_list *) { + if (Error) { + detail::llvmErrorToPythonException(std::move(Error)); + return {}; + } + return nanobind::none().release(); + } +}; + +/// This is a caster class for llvm::Expected, it allows converting functions +/// returning it to either the wrapped T or a thrown python exception +/// Inspired by "nanobind/stl/detail/optional.h" +template +struct type_caster> { + using Caster = make_caster; + NB_TYPE_CASTER(llvm::Expected, Caster::Name) + + // from_python is intentionally not defined because we do not support passing + // `llvm::Expected`s as arguments + + static handle + from_cpp(llvm::Expected &&Error, rv_policy Policy, cleanup_list *Cleanup) { + if (not Error) { + detail::llvmErrorToPythonException(Error.takeError()); + return {}; + } + + return Caster::from_cpp(std::move(Error.get()), Policy, Cleanup); + } +}; + +NAMESPACE_END(detail) +NAMESPACE_END(NB_NAMESPACE) diff --git a/include/revng/PipeboxCommon/Helpers/Python/ContainerIO.h b/include/revng/PipeboxCommon/Helpers/Python/ContainerIO.h new file mode 100644 index 000000000..1f34f0660 --- /dev/null +++ b/include/revng/PipeboxCommon/Helpers/Python/ContainerIO.h @@ -0,0 +1,94 @@ +#pragma once + +// +// This file is distributed under the MIT License. See LICENSE.md for details. +// + +#include "nanobind/nanobind.h" + +#include "revng/PipeboxCommon/Common.h" +#include "revng/PipeboxCommon/Concepts.h" +#include "revng/PipeboxCommon/Helpers/Python/Helpers.h" +#include "revng/PipeboxCommon/ObjectID.h" + +namespace revng::pypeline::helpers::python { + +template +struct ContainerIO { + static nanobind::object objects(T &Handle) { + nanobind::object ObjectSet = importObject("revng.pypeline.object." + "ObjectSet"); + std::set Objects = Handle.objects(); + + nanobind::list Result; + for (const auto &Object : Objects) + Result.append(nanobind::cast(Object)); + + return ObjectSet(nanobind::cast(T::Kind), nanobind::set(Result)); + } + + static llvm::Error deserialize(T &Handle, nanobind::dict &Data) { + // Input map that will be fed to the Container's deserialize method + std::map> Input; + // Temporary PyBuffer-s that are needed to read the contents of Data's + // values + std::vector Buffers; + + for (const auto &[Key, Value] : Data) { + ObjectID *First = nanobind::cast(Key); + + // Retrieve the value as a PyBuffer, this allows both bytes, + // revng::pypeline::Buffer-s and any other container implementing the + // buffer protocol to be used + revng_assert(PyObject_CheckBuffer(Value.ptr()) == 1); + auto MaybeBuffer = ManagedPyBuffer::make(Value.ptr(), PyBUF_SIMPLE); + if (std::holds_alternative(MaybeBuffer)) { + // Return a success because the `PyObject_GetBuffer` already set the + // correct PyError + return llvm::Error::success(); + } + Buffers.push_back(std::move(std::get(MaybeBuffer))); + ManagedPyBuffer &Buffer = Buffers.back(); + + // Buffer support 2D arrays, we requested PyBUF_SIMPLE which should be + // equivalent to void[], but check that is 1D anyways + if (Buffer->itemsize != 1 or Buffer->ndim != 1) + return revng::createError("Invalid buffer shape"); + + // Convert the buffer to ArrayRef + const char *DataPtr = static_cast(Buffer->buf); + Input[First] = llvm::ArrayRef(DataPtr, Buffer->len); + } + + Handle.deserialize(Input); + return llvm::Error::success(); + } + + static nanobind::dict serialize(T &Handle, nanobind::object ObjectSet) { + nanobind::object ObjectSetCls = importObject("revng.pypeline.object." + "ObjectSet"); + revng_assert(nanobind::isinstance(ObjectSet, ObjectSetCls)); + nanobind::object Objects = nanobind::getattr(ObjectSet, "objects"); + + // Convert the input Objects set into a vector of ObjectIDs + std::vector CppObjects; + for (const auto &Object : Objects) + CppObjects.push_back(nanobind::cast(Object)); + + std::map Result = Handle.serialize(CppObjects); + // Manually convert the map above to a Python dict + nanobind::dict Return; + for (auto &Entry : Result) { + // Here the key is copied on purpose because it needs to be moved. + // We use the special syntax of nanobind::cast to move-construct the + // python object, so that the amount of memory copying is minimized + ObjectID KeyCopy = Entry.first; + nanobind::object Key = nanobind::cast(std::move(KeyCopy)); + nanobind::object Value = nanobind::cast(std::move(Entry.second)); + Return[Key] = Value; + } + return Return; + }; +}; + +} // namespace revng::pypeline::helpers::python diff --git a/include/revng/PipeboxCommon/Helpers/Python/Helpers.h b/include/revng/PipeboxCommon/Helpers/Python/Helpers.h new file mode 100644 index 000000000..2fcf0d728 --- /dev/null +++ b/include/revng/PipeboxCommon/Helpers/Python/Helpers.h @@ -0,0 +1,75 @@ +#pragma once + +// +// This file is distributed under the MIT License. See LICENSE.md for details. +// + +#include + +#include "nanobind/nanobind.h" + +#include "llvm/ADT/StringRef.h" + +#include "revng/PipeboxCommon/Common.h" +#include "revng/PipeboxCommon/ObjectID.h" + +namespace revng::pypeline::helpers::python { + +inline nanobind::object importObject(llvm::StringRef String) { + auto [ModulePath, ObjectName] = String.rsplit('.'); + nanobind::module_ Module = nanobind::module_::import_(ModulePath.str() + .c_str()); + return Module.attr(ObjectName.str().c_str()); +} + +// Helper class to unpack containers from a nanobind::list. +// To be used in conjunction with PipeRunner or AnalysisRunner +class ContainerListUnwrapper { +public: + using ListType = nanobind::list &; + + template + static C unwrap(ListType ContainerList) { + using C_ref_removed = std::remove_reference_t; + return *nanobind::cast(ContainerList[I]); + } +}; + +class ManagedPyBuffer { +private: + Py_buffer Buffer; + + ManagedPyBuffer() { Buffer.obj = NULL; } + +public: + ~ManagedPyBuffer() { + if (Buffer.obj != NULL) + PyBuffer_Release(&Buffer); + } + + ManagedPyBuffer(const ManagedPyBuffer &) = delete; + ManagedPyBuffer &operator=(const ManagedPyBuffer &) = delete; + + ManagedPyBuffer(ManagedPyBuffer &&Other) { *this = std::move(Other); } + ManagedPyBuffer &operator=(ManagedPyBuffer &&Other) { + if (this == &Other) + return *this; + + Buffer = std::move(Other.Buffer); + Other.Buffer.obj = NULL; + return *this; + } + + static std::variant make(PyObject *Obj, int Flags) { + ManagedPyBuffer Result; + int RC = PyObject_GetBuffer(Obj, &Result.Buffer, Flags); + if (RC != 0) + return RC; + return Result; + } + + const Py_buffer &operator*() { return Buffer; } + const Py_buffer *operator->() { return &Buffer; } +}; + +} // namespace revng::pypeline::helpers::python diff --git a/include/revng/PipeboxCommon/Helpers/Python/Registry.h b/include/revng/PipeboxCommon/Helpers/Python/Registry.h new file mode 100644 index 000000000..6d507c9dd --- /dev/null +++ b/include/revng/PipeboxCommon/Helpers/Python/Registry.h @@ -0,0 +1,47 @@ +#pragma once + +// +// This file is distributed under the MIT License. See LICENSE.md for details. +// + +#include + +#include "nanobind/nanobind.h" + +namespace revng::pypeline::helpers::python { + +struct BaseClasses { + nanobind::object BaseContainer; + nanobind::object BaseAnalysis; + nanobind::object BasePipe; +}; + +/// Registry class that allows registering functions that will modify the +/// provided Python module by adding Container, Analysis and Pipes subclasses. +class RegistryImpl { +private: + using PythonModuleInitializer = void (*)(nanobind::module_ &, BaseClasses &); + std::vector ModuleInitializers; + +public: + RegistryImpl() = default; + ~RegistryImpl() = default; + RegistryImpl(const RegistryImpl &) = delete; + RegistryImpl &operator=(const RegistryImpl &) = delete; + RegistryImpl(RegistryImpl &&) = delete; + RegistryImpl &operator=(RegistryImpl &&) = delete; + +public: + void registerModuleInitializer(PythonModuleInitializer PMI) { + ModuleInitializers.push_back(PMI); + } + + void callAll(nanobind::module_ &M, BaseClasses &BC) { + for (auto &Element : ModuleInitializers) + Element(M, BC); + } +}; + +inline RegistryImpl Registry; + +} // namespace revng::pypeline::helpers::python diff --git a/include/revng/PipeboxCommon/Helpers/Python/RunAnalysis.h b/include/revng/PipeboxCommon/Helpers/Python/RunAnalysis.h new file mode 100644 index 000000000..502dcfef5 --- /dev/null +++ b/include/revng/PipeboxCommon/Helpers/Python/RunAnalysis.h @@ -0,0 +1,32 @@ +#pragma once + +// +// This file is distributed under the MIT License. See LICENSE.md for details. +// + +#include "nanobind/nanobind.h" + +#include "revng/PipeboxCommon/Helpers/AnalysisRunner.h" +#include "revng/PipeboxCommon/Helpers/Python/Helpers.h" +#include "revng/PipeboxCommon/Model.h" + +namespace revng::pypeline::helpers::python { + +template +inline llvm::Error runAnalysis(T &Handle, + nanobind::handle_t TheModel, + nanobind::list Containers, + Request Incoming, + llvm::StringRef Configuration) { + using namespace revng::pypeline::helpers::python; + Model *CppModel = nanobind::cast(TheModel); + + return AnalysisRunner::run(Handle, + &T::run, + *CppModel, + Incoming, + Configuration, + Containers); +} + +} // namespace revng::pypeline::helpers::python diff --git a/include/revng/PipeboxCommon/Helpers/Python/RunPipe.h b/include/revng/PipeboxCommon/Helpers/Python/RunPipe.h new file mode 100644 index 000000000..94f619b53 --- /dev/null +++ b/include/revng/PipeboxCommon/Helpers/Python/RunPipe.h @@ -0,0 +1,38 @@ +#pragma once + +// +// This file is distributed under the MIT License. See LICENSE.md for details. +// + +#include "nanobind/nanobind.h" + +#include "revng/PipeboxCommon/Common.h" +#include "revng/PipeboxCommon/Helpers/PipeRunner.h" +#include "revng/PipeboxCommon/Helpers/Python/Helpers.h" +#include "revng/PipeboxCommon/Model.h" + +namespace revng::pypeline::helpers::python { + +template +inline ObjectDependencies runPipe(T &Handle, + nanobind::object TheModel, + nanobind::list Containers, + Request Incoming, + Request Outgoing, + llvm::StringRef Configuration) { + nanobind::object ReadOnlyModel = importObject("revng.pypeline.model." + "ReadOnlyModel"); + revng_assert(nanobind::isinstance(TheModel, ReadOnlyModel)); + nanobind::object ActualModel = nanobind::getattr(TheModel, "downcast")(); + const Model *CppModel = nanobind::cast(ActualModel); + + return PipeRunner::run(Handle, + &T::run, + *CppModel, + Incoming, + Outgoing, + Configuration, + Containers); +} + +} // namespace revng::pypeline::helpers::python diff --git a/include/revng/PipeboxCommon/Helpers/Python/SignatureHelper.h b/include/revng/PipeboxCommon/Helpers/Python/SignatureHelper.h new file mode 100644 index 000000000..cd8c1f019 --- /dev/null +++ b/include/revng/PipeboxCommon/Helpers/Python/SignatureHelper.h @@ -0,0 +1,108 @@ +#pragma once + +// +// This file is distributed under the MIT License. See LICENSE.md for details. +// + +#include + +#include "nanobind/nanobind.h" + +#include "revng/PipeboxCommon/Common.h" +#include "revng/PipeboxCommon/Concepts.h" +#include "revng/PipeboxCommon/Helpers/Python/Helpers.h" + +namespace revng::pypeline::helpers::python { + +template + requires IsPipe or IsAnalysis +class SignatureHelper { +public: + /// Function that gets the signature of a given Pipe or Analysis type. Since + /// the signature might be requested many times, the result is cached in the + /// `_signature` attribute inside the class. + static nanobind::object getSignature() { + // Get the PyObject of the specified class T. + // We need to `borrow` it since we want to increase the refcount for the + // duration of this function. + nanobind::object Class = nanobind::borrow(nanobind::type()); + // Check that it's actually not NULL (e.g. not registered through nanobind) + revng_assert(Class.is_valid()); + // Check if we previously computed the signature, if so return that + nanobind::object Signature = nanobind::getattr(Class, + "_signature", + nanobind::none()); + + if (not Signature.is_none()) + return Signature; + + // If here we've yet to compute the signature, do that and save it as an + // attribute inside the class + if constexpr (IsAnalysis) + Signature = parseAnalysisSignature(); + else + Signature = parsePipeSignature(); + nanobind::setattr(Class, "_signature", Signature); + return Signature; + } + +private: + /// Given a pipe type T, compute its signature + static nanobind::object parsePipeSignature() { + nanobind::object TaskArgument = importObject("revng.pypeline.task.task." + "TaskArgument"); + nanobind::object TaskArgumentAccess = importObject("revng.pypeline.task." + "task." + "TaskArgumentAccess"); + // Result list, this will be appended with a TaskArgument object for each + // argument of the pipe + nanobind::list Result; + + using CT = PipeRunTraits::ContainerTypes; + compile_time::forEach([&Result, + &TaskArgument, + &TaskArgumentAccess]() { + const PipeArgumentDocumentation &Argument = T::ArgumentsDocumentation[I]; + // Create a Kwargs dictionary, this will be passed to the constructor of + // TaskArgument + nanobind::dict Kwargs; + + Kwargs["name"] = nanobind::str(Argument.Name.data(), + Argument.Name.size()); + Kwargs["help_text"] = nanobind::str(Argument.HelpText.data(), + Argument.HelpText.size()); + // nanobind::type returns a reference, so we need to borrow it and + // increase the reference count + Kwargs["container_type"] = nanobind::borrow(nanobind::type()); + // If the argument is const then the access is READ, otherwise READ_WRITE + if constexpr (std::is_const_v) + Kwargs["access"] = nanobind::getattr(TaskArgumentAccess, "READ"); + else + Kwargs["access"] = nanobind::getattr(TaskArgumentAccess, "READ_WRITE"); + + // kwargs_proxy is a special nanobind class that allows passing a + // nanobind::dict as kwargs, this is equivalent to doing + // `TaskArgument(**Kwargs)`. + Result.append(TaskArgument(nanobind::detail::kwargs_proxy(Kwargs))); + }); + + // This version of the nanobind::tuple constructor automatically converts + // the nanobind::list to a tuple. + return nanobind::tuple(Result); + } + + /// Given an analysis type T, compute its signature + static nanobind::object parseAnalysisSignature() { + nanobind::list Result; + + using CT = AnalysisRunTraits::ContainerTypes; + compile_time::forEach([&Result]() { + // For each tuple element, add the python type to the Result list + Result.append(nanobind::borrow(nanobind::type())); + }); + + return nanobind::tuple(Result); + } +}; + +} // namespace revng::pypeline::helpers::python diff --git a/include/revng/PipeboxCommon/Helpers/Registrars.h b/include/revng/PipeboxCommon/Helpers/Registrars.h new file mode 100644 index 000000000..973d85609 --- /dev/null +++ b/include/revng/PipeboxCommon/Helpers/Registrars.h @@ -0,0 +1,96 @@ +#pragma once + +// +// This file is distributed under the MIT License. See LICENSE.md for details. +// + +#include "nanobind/nanobind.h" +#include "nanobind/stl/optional.h" +#include "nanobind/stl/pair.h" +#include "nanobind/stl/set.h" +#include "nanobind/stl/string.h" +#include "nanobind/stl/vector.h" + +#include "revng/PipeboxCommon/Concepts.h" +#include "revng/PipeboxCommon/Helpers/Python/Casters.h" +#include "revng/PipeboxCommon/Helpers/Python/ContainerIO.h" +#include "revng/PipeboxCommon/Helpers/Python/Registry.h" +#include "revng/PipeboxCommon/Helpers/Python/RunAnalysis.h" +#include "revng/PipeboxCommon/Helpers/Python/RunPipe.h" +#include "revng/PipeboxCommon/Helpers/Python/SignatureHelper.h" + +template +struct RegisterAnalysis { + RegisterAnalysis() { + using namespace nanobind::literals; + using namespace revng::pypeline::helpers; + + // Python + python::Registry.registerModuleInitializer([](nanobind::module_ &M, + python::BaseClasses &BC) { + nanobind::class_(M, T::Name.data(), BC.BaseAnalysis) + .def_ro_static("name", &T::Name) + .def(nanobind::init<>()) + .def_static("signature", + &python::SignatureHelper::getSignature, + nanobind::sig("def signature() -> " + "tuple[type[revng.pypeline.container." + "Container], ...]")) + .def("run", + &python::runAnalysis, + "model"_a, + "containers"_a, + "incoming"_a, + "configuration"_a); + }); + } +}; + +template +struct RegisterContainer { + RegisterContainer() { + using namespace revng::pypeline::helpers; + + // Python + python::Registry.registerModuleInitializer([](nanobind::module_ &M, + python::BaseClasses &BC) { + nanobind::class_(M, T::Name.data(), BC.BaseContainer) + .def_ro_static("kind", &T::Kind) + .def(nanobind::init<>()) + .def("objects", &python::ContainerIO::objects) + .def("verify", &T::verify) + .def("deserialize", &python::ContainerIO::deserialize) + .def("serialize", &python::ContainerIO::serialize); + }); + } +}; + +template +struct RegisterPipe { + RegisterPipe() { + using namespace nanobind::literals; + using namespace revng::pypeline::helpers; + + // Python + python::Registry.registerModuleInitializer([](nanobind::module_ &M, + python::BaseClasses &BC) { + nanobind::class_(M, T::Name.data(), BC.BasePipe) + .def_ro_static("name", &T::Name) + .def_static("signature", + &python::SignatureHelper::getSignature, + nanobind::sig("def signature() -> " + "tuple[revng.pypeline.task.task.TaskArgument," + " ...]")) + .def(nanobind::init()) + .def_prop_ro("static_configuration", + [](T &Handle) { return Handle.StaticConfiguration; }) + .def("run", + &python::runPipe, + "model"_a, + "containers"_a, + "incoming"_a, + "outgoing"_a, + "configuration"_a); + }); + } +}; diff --git a/include/revng/PipeboxCommon/Model.h b/include/revng/PipeboxCommon/Model.h new file mode 100644 index 000000000..674475866 --- /dev/null +++ b/include/revng/PipeboxCommon/Model.h @@ -0,0 +1,65 @@ +#pragma once + +// +// This file is distributed under the MIT License. See LICENSE.md for details. +// + +#include "revng/Model/Binary.h" +#include "revng/PipeboxCommon/Common.h" +#include "revng/Support/MetaAddress.h" + +class Model { +private: + TupleTree TheModel; + +public: + std::set diff(const Model &Other) const { + auto Diff = ::diff(*TheModel.get(), *Other.TheModel.get()); + std::set Result; + for (auto &Entry : Diff.Changes) { + auto MaybePath = pathAsString(Entry.Path); + Result.insert(MaybePath.value()); + } + return Result; + } + + Model clone() const { return *this; } + + std::set children(const ObjectID &Obj, Kind Kind) const { + if (Obj.kind() == Kinds::Binary and Kind == Kinds::Function) { + std::set Result; + for (const model::Function &F : TheModel->Functions()) + Result.insert(ObjectID(F.Entry())); + return Result; + } + + if (Obj.kind() == Kinds::Binary and Kind == Kinds::TypeDefinition) { + std::set Result; + for (const UpcastablePointer &TD : + TheModel->TypeDefinitions()) + Result.insert(ObjectID(TD->key())); + return Result; + } + revng_abort(); + } + + revng::pypeline::Buffer serialize() const { + revng::pypeline::Buffer Out; + llvm::raw_svector_ostream OS(Out.data()); + TheModel.serialize(OS); + return Out; + } + + llvm::Error deserialize(llvm::StringRef Input) { + auto MaybeModel = TupleTree::fromString(Input); + if (not MaybeModel) + return MaybeModel.takeError(); + + TheModel = std::move(*MaybeModel); + return llvm::Error::success(); + } + +public: + TupleTree &get() { return TheModel; } + const TupleTree &get() const { return TheModel; } +}; diff --git a/include/revng/PipeboxCommon/ObjectID.h b/include/revng/PipeboxCommon/ObjectID.h new file mode 100644 index 000000000..645154164 --- /dev/null +++ b/include/revng/PipeboxCommon/ObjectID.h @@ -0,0 +1,184 @@ +#pragma once + +// +// This file is distributed under the MIT License. See LICENSE.md for details. +// + +#include + +#include "llvm/ADT/StringRef.h" + +#include "revng/Model/TypeDefinition.h" +#include "revng/Pipeline/Location.h" +#include "revng/Pipes/Ranks.h" +#include "revng/Support/MetaAddress.h" + +struct Kinds; + +class Kind { +private: + enum class ValueType { + Binary = 0, + Function, + TypeDefinition, + }; + +public: + ValueType Value; + +private: + constexpr Kind(ValueType Value) : Value(Value){}; + friend std::hash; + friend Kinds; + +public: + static std::vector kinds() { + return { Kind{ ValueType::Binary }, + Kind{ ValueType::Function }, + Kind{ ValueType::TypeDefinition } }; + }; + + std::optional parent() { + if (Value == ValueType::Binary) + return std::nullopt; + else if (Value == ValueType::Function or Value == ValueType::TypeDefinition) + return Kind{ ValueType::Binary }; + else + revng_abort(); + } + + static Kind deserialize(llvm::StringRef Value) { + if (Value == "binary") { + return Kind{ ValueType::Binary }; + } else if (Value == "function") { + return Kind{ ValueType::Function }; + } else if (Value == "type-definition") { + return Kind{ ValueType::TypeDefinition }; + } else { + revng_abort(); + } + } + + std::string serlialize() { + if (Value == ValueType::Binary) { + return "binary"; + } else if (Value == ValueType::Function) { + return "function"; + } else if (Value == ValueType::TypeDefinition) { + return "type-definition"; + } else { + revng_abort(); + } + } + + std::strong_ordering operator<=>(const Kind &) const = default; +}; + +template<> +struct std::hash { + uint64_t operator()(const Kind &Kind) const { + return std::hash{}(Kind.Value); + } +}; + +template<> +struct std::hash : std::hash {}; + +struct Kinds { + static inline constexpr Kind Binary{ Kind::ValueType::Binary }; + static inline constexpr Kind Function{ Kind::ValueType::Function }; + static inline constexpr Kind TypeDefinition{ + Kind::ValueType::TypeDefinition + }; +}; + +class ObjectID { +private: + using TypeDefinitionKey = model::TypeDefinition::Key; + std::variant Key; + +public: + // Create a root ObjectID + ObjectID() = default; + // Create a Function ObjectID + ObjectID(const MetaAddress &Addr) : Key(Addr) {} + // Create a TypeDefinition ObjectID + ObjectID(const TypeDefinitionKey &Key) : Key(Key) {} + + Kind kind() const { + auto Visitor = [](const T &) { + if constexpr (std::is_same_v) + return Kinds::Binary; + else if constexpr (std::is_same_v) + return Kinds::Function; + else if constexpr (std::is_same_v) + return Kinds::TypeDefinition; + else + revng_abort(); + }; + return std::visit(Visitor, Key); + } + + std::optional parent() const { + const Kind &TheKind = kind(); + if (TheKind == Kinds::Binary) + return std::nullopt; + else if (TheKind == Kinds::Function or TheKind == Kinds::TypeDefinition) + return ObjectID(); + else + revng_abort(); + } + + static ObjectID root() { return ObjectID(); } + + std::string serialize() const { + using namespace revng::ranks; + using pipeline::locationString; + const Kind &TheKind = kind(); + + if (TheKind == Kinds::Binary) + return locationString(Binary); + else if (TheKind == Kinds::Function) + return locationString(Function, std::get(Key)); + else if (TheKind == Kinds::TypeDefinition) + return locationString(TypeDefinition, std::get(Key)); + else + revng_abort(); + } + + static llvm::Expected deserialize(llvm::StringRef Input) { + using namespace revng::ranks; + using pipeline::locationFromString; + + if (Input == "/binary") { + return ObjectID(); + } else if (Input.starts_with("/function/")) { + auto MaybeKey = locationFromString(Function, Input); + if (not MaybeKey.has_value()) + return revng::createError("Failed deserializing ObjectID"); + MetaAddress Key = std::get<0>(std::get<0>(MaybeKey->tuple())); + return ObjectID(Key); + } else if (Input.starts_with("/type-definition/")) { + auto MaybeKey = locationFromString(TypeDefinition, Input); + if (not MaybeKey.has_value()) + return revng::createError("Failed deserializing ObjectID"); + TypeDefinitionKey Key = std::get<0>(MaybeKey->tuple()); + return ObjectID(Key); + } else { + return revng::createError("Failed deserializing ObjectID"); + } + } + + std::strong_ordering operator<=>(const ObjectID &) const = default; + friend std::hash; +}; + +template<> +struct std::hash { + uint64_t operator()(const ObjectID &Obj) const { + return std::hash{}(Obj.Key); + } +}; + +template<> +struct std::hash : std::hash {}; diff --git a/lib/PipeboxCommon/Python/_pipebox.cpp b/lib/PipeboxCommon/Python/_pipebox.cpp new file mode 100644 index 000000000..4e31d13aa --- /dev/null +++ b/lib/PipeboxCommon/Python/_pipebox.cpp @@ -0,0 +1,99 @@ +// +// This file is distributed under the MIT License. See LICENSE.md for details. +// + +#include "nanobind/nanobind.h" +#include "nanobind/stl/optional.h" +#include "nanobind/stl/set.h" +#include "nanobind/stl/string.h" +#include "nanobind/stl/vector.h" + +#include "revng/PipeboxCommon/Helpers/Python/Casters.h" +#include "revng/PipeboxCommon/Helpers/Python/Helpers.h" +#include "revng/PipeboxCommon/Helpers/Python/Registry.h" +#include "revng/PipeboxCommon/Model.h" +#include "revng/Support/InitRevng.h" + +NB_MODULE(_pipebox, m) { + { + // Do InitRevng, use a capsule to call the destructor when the Python module + // is unloaded + int Argc = 1; + const char *Argv[] = { "" }; + const char **ArgvPtr = Argv; + nanobind::capsule Capsule(new revng::InitRevng(Argc, ArgvPtr, "", {}), + [](void *Ptr) noexcept { + delete static_cast(Ptr); + }); + nanobind::setattr(m, "__init_revng__", Capsule); + } + + using namespace revng::pypeline::helpers::python; + + // Register the Buffer class + nanobind::class_(m, "Buffer") + .def(nanobind::init<>()) + .def("__buffer__", [](revng::pypeline::Buffer &Handle, int Flags) { + // PyMemoryView_FromMemory returns a new reference, hence the need to + // `steal` it with nanobind. + return nanobind::steal(PyMemoryView_FromMemory(Handle.data().data(), + Handle.data().size(), + Flags)); + }); + + // Register ObjectID + nanobind::object ObjectIDBaseClass = importObject("revng.pypeline.object." + "ObjectID"); + nanobind::class_(m, "ObjectID", ObjectIDBaseClass) + .def(nanobind::init<>()) + .def("kind", &ObjectID::kind) + .def("parent", &ObjectID::parent) + .def_static("root", &ObjectID::root) + .def("serialize", &ObjectID::serialize) + .def_static("deserialize", &ObjectID::deserialize) + .def("__eq__", + [](ObjectID &Handle, nanobind::object Other) { + ObjectID *OtherHandle; + if (not nanobind::try_cast(Other, OtherHandle)) + return false; + return Handle == *OtherHandle; + }) + .def("__hash__", + [](ObjectID &Handle) { return std::hash{}(Handle); }); + + // Register Kind + nanobind::object KindBaseClass = importObject("revng.pypeline.object.Kind"); + nanobind::class_(m, "Kind", KindBaseClass) + .def_static("kinds", &Kind::kinds) + .def("parent", &Kind::parent) + .def_static("deserialize", &Kind::deserialize) + .def("serialize", &Kind::serlialize) + .def("__eq__", + [](Kind &Handle, nanobind::object Other) { + Kind *OtherHandle; + if (not nanobind::try_cast(Other, OtherHandle)) + return false; + return Handle == *OtherHandle; + }) + .def("__hash__", [](Kind &Handle) { return std::hash{}(Handle); }); + + // Register Model + nanobind::object ModelBaseClass = importObject("revng.pypeline.model.Model"); + nanobind::class_(m, "Model", ModelBaseClass) + .def(nanobind::init<>()) + .def("diff", + [](Model &Handle, nanobind::handle_t Other) { + return Handle.diff(*nanobind::cast(Other)); + }) + .def("clone", &Model::clone) + .def("serialize", &Model::serialize) + .def("deserialize", &Model::deserialize); + + // Register all Pipes, Analyses and Containers + BaseClasses BC{ + .BaseContainer = importObject("revng.pypeline.container.Container"), + .BaseAnalysis = importObject("revng.pypeline.analysis.Analysis"), + .BasePipe = importObject("revng.pypeline.task.pipe.Pipe"), + }; + Registry.callAll(m, BC); +} diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index 7d37c12c0..5703c7c8c 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -154,8 +154,7 @@ python_module( MODULE_INIT revng/internal/__init__.py MODULE_FILES - revng/internal/py.typed - revng/internal/pipebox.py) + revng/internal/py.typed) # # Install revng.model (including autogenerated classes) @@ -416,6 +415,7 @@ set(REVNG_PYPELINE_MODULE_FILES revng/pypeline/utils/registry.py revng/pypeline/utils/cabc.py revng/pypeline/utils/__init__.py) + python_module( TARGET_NAME revng-python-pypeline @@ -426,6 +426,35 @@ python_module( MODULE_FILES ${REVNG_PYPELINE_MODULE_FILES}) +# +# Nanobind module generation and creation of revng.internal.{,_}cpp_pypeline +# +llvm_map_components_to_libnames(LLVM_LIBRARIES Support Core) + +set(CPP_MODULE_NAME _pipebox) +add_library("${CPP_MODULE_NAME}" MODULE + "${CMAKE_SOURCE_DIR}/lib/PipeboxCommon/Python/_pipebox.cpp") +set_target_properties( + "${CPP_MODULE_NAME}" + PROPERTIES LINKER_LANGUAGE CXX + PREFIX "" + LIBRARY_OUTPUT_DIRECTORY + "${CMAKE_BINARY_DIR}/${PYTHON_INSTALL_PATH}/revng/internal") +target_link_libraries( + "${CPP_MODULE_NAME}" PUBLIC ${Python_LIBRARIES} ${LLVM_LIBRARIES} nanobind + revngSupport revngModel) +target_include_directories("${CPP_MODULE_NAME}" PUBLIC ${Python_INCLUDE_DIRS}) + +python_module( + TARGET_NAME + python-cpp-pipebox + WHEEL + revng_internal + MODULE_FILES + "revng/internal/pipebox.py" + MODULE_GENERATED_FILES + "revng/internal/${CPP_MODULE_NAME}.so") + # # Generate actual wheel rules # diff --git a/python/revng/internal/pyproject.toml b/python/revng/internal/pyproject.toml index 6b0246fa6..5cdb65b3b 100644 --- a/python/revng/internal/pyproject.toml +++ b/python/revng/internal/pyproject.toml @@ -42,6 +42,7 @@ build-backend = "setuptools.build_meta" [tool.setuptools.package-data] revng = [ "internal/py.typed", + "internal/_pipebox.so", "internal/daemon/schema.graphql", "internal/cli/_commands/generate_migrations/metaschema.yml", "internal/cli/_commands/generate_migrations/migration.py.tpl", diff --git a/python/revng/internal/support/__init__.py b/python/revng/internal/support/__init__.py index 53d665b03..bfff78869 100644 --- a/python/revng/internal/support/__init__.py +++ b/python/revng/internal/support/__init__.py @@ -3,9 +3,11 @@ # import os +import sys from collections.abc import Iterable as CIterable +from ctypes import CDLL from pathlib import Path -from typing import Iterable, TypeVar +from typing import Any, Iterable, TypeVar from xdg import xdg_cache_home @@ -25,3 +27,27 @@ def cache_directory() -> Path: return Path(os.environ["REVNG_CACHE_DIR"]) else: return xdg_cache_home() / "revng" + + +def import_pipebox(libraries: Iterable[str | Path]) -> tuple[Any, list[Any]]: + """Load libraries and return the pipebox module""" + + # Load the provided shared libraries, these contain static initializers + # like RegisterPipe which allows registering Pipes, Analyses and + # Containers to be later created as python classes when the `_pipeline` + # module is imported. + # + # RTLD_GLOBAL is needed due to nanobind using weak symbols to derive the TypeID + handles = [CDLL(path, os.RTLD_NOW | os.RTLD_GLOBAL) for path in libraries] + + # Check that the module hasn't been previously imported, otherwise result + # might be unpredictable + assert "revng.internal._pipebox" not in sys.modules + + # Import the actual module, this will do the following: + # 1. Create actual classes for Model, ObjectID and Kind + # 2. Create, on the fly, classes that have been previously registered by + # the static initializers in the imported libraries above + import revng.internal._pipebox as ext + + return (ext, handles) diff --git a/python/revng/pypeline/analysis.py b/python/revng/pypeline/analysis.py index 2221f08dd..f9bec6ad9 100644 --- a/python/revng/pypeline/analysis.py +++ b/python/revng/pypeline/analysis.py @@ -21,8 +21,6 @@ class Analysis(ABC): objects in save points. """ - __slots__: tuple = ("name",) - def __init__(self, name: str): """ Initialize the analysis with a name. diff --git a/python/revng/pypeline/container.py b/python/revng/pypeline/container.py index b546032cc..12ab1f763 100644 --- a/python/revng/pypeline/container.py +++ b/python/revng/pypeline/container.py @@ -7,7 +7,7 @@ from __future__ import annotations import json from collections.abc import Mapping from dataclasses import dataclass -from typing import Annotated, Dict, Generator, Optional, Tuple, Type +from typing import Annotated, Dict, Generator, Tuple, Type from .object import Kind, ObjectID, ObjectSet from .utils.cabc import ABC, abstractmethod @@ -109,11 +109,9 @@ class Container(ABC): """ @abstractmethod - def serialize(self, objects: Optional[ObjectSet] = None) -> Mapping[ObjectID, bytes]: + def serialize(self, objects: ObjectSet) -> Mapping[ObjectID, bytes]: """ Dump objects from this container into a serialized format. - If objects is provided, only those objects will be dumped. - If not, all objects in the container will be dumped. """ @classmethod @@ -182,7 +180,7 @@ def dump_container( """ with open(path, "w", encoding="utf-8") as f: json.dump( - {k.serialize(): v.hex() for k, v in container.serialize().items()}, + {k.serialize(): v.hex() for k, v in container.serialize(container.objects()).items()}, f, indent=4, sort_keys=True, diff --git a/python/revng/pypeline/task/pipe.py b/python/revng/pypeline/task/pipe.py index 1f9430239..d4e5ceb71 100644 --- a/python/revng/pypeline/task/pipe.py +++ b/python/revng/pypeline/task/pipe.py @@ -37,8 +37,6 @@ class Pipe(ABC): argument should not be added. """ - __slots__: tuple = ("name", "static_configuration") - def __init__( self, static_configuration: str = "", diff --git a/scripts/nanobind_generate_stubs.py b/scripts/nanobind_generate_stubs.py new file mode 100755 index 000000000..656f1b450 --- /dev/null +++ b/scripts/nanobind_generate_stubs.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python3 + +# +# This file is distributed under the MIT License. See LICENSE.md for details. +# + +import argparse +import sys + +from nanobind.stubgen import StubGen + +from revng.internal.support import import_pipebox + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("-o", "--output", help="Output file (stdout if omitted)") + parser.add_argument( + "-l", + "--library", + default=[], + action="append", + help="Shared library to preload before generating stubs", + ) + args = parser.parse_args() + + module, handles = import_pipebox(args.library) + stub_gen = StubGen(module, quiet=False) + stub_gen.put(module) + stub_text = stub_gen.get() + + if args.output is None: + sys.stdout.write(stub_text) + sys.stdout.flush() + else: + with open(args.output, "w") as f: + f.write(stub_text) + + +if __name__ == "__main__": + main() diff --git a/share/revng/clang-format-style-file.yml b/share/revng/clang-format-style-file.yml index 9bb14ae3b..55c2eda7f 100644 --- a/share/revng/clang-format-style-file.yml +++ b/share/revng/clang-format-style-file.yml @@ -81,7 +81,7 @@ { Regex: '^"clang/', Priority: -3 }, { Regex: '^"mlir/', Priority: -4 }, { Regex: '^"llvm/', Priority: -5 }, - { Regex: '^"aws/', Priority: -6 }, + { Regex: '^"nanobind/', Priority: -6 }, { Regex: '^"boost/', Priority: -7 }, { Regex: "^<", Priority: -8 }, ], diff --git a/tests/pypeline/CMakeLists.txt b/tests/pypeline/CMakeLists.txt index 6c050c01c..b83faac17 100644 --- a/tests/pypeline/CMakeLists.txt +++ b/tests/pypeline/CMakeLists.txt @@ -19,3 +19,41 @@ set_tests_properties( test_pypeline PROPERTIES LABELS "pypeline" ENVIRONMENT "PYTHONPATH=${CMAKE_BINARY_DIR}/${PYTHON_INSTALL_PATH}") + +# +# Generate libmockPipeboxImpl and libmockPipebox +# +add_library(mockPipeboxImpl SHARED MockPipeboxImpl.cpp) +add_library(mockPipebox SHARED MockPipebox.cpp) + +llvm_map_components_to_libnames(LLVM_LIBRARIES Support Core) +target_link_libraries(mockPipeboxImpl ${LLVM_LIBRARIES} revngSupport revngModel) +target_link_libraries(mockPipebox mockPipeboxImpl ${Python_LIBRARIES} + ${LLVM_LIBRARIES} nanobind revngSupport) + +target_include_directories(mockPipebox PUBLIC ${Python_INCLUDE_DIRS}) + +# +# Add tests that use libmockPipebox +# +revng_add_test( + NAME pypeline-cpp-test COMMAND + "${CMAKE_CURRENT_SOURCE_DIR}/python_cpp_test.py" + "${CMAKE_CURRENT_BINARY_DIR}/libmockPipebox.so") +set_tests_properties( + pypeline-cpp-test + PROPERTIES LABELS "pypeline;test" ENVIRONMENT + "PYTHONPATH=${CMAKE_BINARY_DIR}/${PYTHON_INSTALL_PATH}") + +revng_add_test( + NAME + pypeline-annotations-test + COMMAND + "${CMAKE_CURRENT_SOURCE_DIR}/nanobind_test_annotations.sh" + "${CMAKE_SOURCE_DIR}" + -l + "${CMAKE_CURRENT_BINARY_DIR}/libmockPipebox.so") +set_tests_properties( + pypeline-annotations-test + PROPERTIES LABELS "pypeline;test" ENVIRONMENT + "PYTHONPATH=${CMAKE_BINARY_DIR}/${PYTHON_INSTALL_PATH}") diff --git a/tests/pypeline/MockPipebox.cpp b/tests/pypeline/MockPipebox.cpp new file mode 100644 index 000000000..67fd3b89f --- /dev/null +++ b/tests/pypeline/MockPipebox.cpp @@ -0,0 +1,11 @@ +// +// This file is distributed under the MIT License. See LICENSE.md for details. +// + +#include "revng/PipeboxCommon/Helpers/Registrars.h" + +#include "MockPipeboxImpl.h" + +static RegisterContainer X; +static RegisterPipe Y; +static RegisterAnalysis Z; diff --git a/tests/pypeline/MockPipeboxImpl.cpp b/tests/pypeline/MockPipeboxImpl.cpp new file mode 100644 index 000000000..5b6debd70 --- /dev/null +++ b/tests/pypeline/MockPipeboxImpl.cpp @@ -0,0 +1,86 @@ +// +// This file is distributed under the MIT License. See LICENSE.md for details. +// + +#include +#include +#include + +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/StringRef.h" + +#include "revng/PipeboxCommon/Model.h" +#include "revng/PipeboxCommon/ObjectID.h" + +#include "MockPipeboxImpl.h" + +// +// StringContainer +// + +std::set StringContainer::objects() const { + return std::views::keys(Storage) | revng::to>(); +} + +void StringContainer::deserialize(const std::map> + Data) { + for (auto &Entry : Data) { + revng_assert(Entry.first->kind() == Kind); + std::string EntryData(Entry.second.data(), Entry.second.size()); + Storage[*Entry.first] = std::move(EntryData); + } +} + +std::map +StringContainer::serialize(const std::vector Objects) const { + std::map Result; + for (auto &Element : Objects) { + const std::string &StoredElement = Storage.at(*Element); + llvm::ArrayRef Ref = { StoredElement.data(), StoredElement.size() }; + Result[*Element] = llvm::SmallVector{ Ref }; + } + return Result; +} + +bool StringContainer::verify() const { + return true; +} + +// +// AppendFooPipe +// + +revng::pypeline::ObjectDependencies +AppendFooPipe::run(const Model &TheModel, + const revng::pypeline::Request &Incoming, + const revng::pypeline::Request &Outgoing, + llvm::StringRef Configuration, + StringContainer &Container) { + revng_assert(Outgoing.size() == 1); + + auto &Storage = Container.getStorage(); + for (auto &Elem : Outgoing[0]) { + if (Storage.contains(*Elem)) { + Storage[*Elem] = Storage[*Elem] + "foo"; + } else { + Storage[*Elem] = "foo"; + } + } + + return {}; +} + +// +// AppendFooLibAnalysis +// + +llvm::Error AppendFooLibAnalysis::run(Model &TheModel, + const revng::pypeline::Request &Incoming, + llvm::StringRef Configuration, + StringContainer &Container) { + revng_assert(Incoming.size() == 1); + TheModel.get()->ImportedLibraries().insert("foo.so"); + + return llvm::Error::success(); +} diff --git a/tests/pypeline/MockPipeboxImpl.h b/tests/pypeline/MockPipeboxImpl.h new file mode 100644 index 000000000..e54fa1e8b --- /dev/null +++ b/tests/pypeline/MockPipeboxImpl.h @@ -0,0 +1,56 @@ +#pragma once + +// +// This file is distributed under the MIT License. See LICENSE.md for details. +// + +#include "revng/PipeboxCommon/Common.h" +#include "revng/PipeboxCommon/Model.h" +#include "revng/PipeboxCommon/ObjectID.h" + +class StringContainer { +private: + std::map Storage; + +public: + static constexpr llvm::StringRef Name = "StringContainer"; + static constexpr Kind Kind = Kinds::Function; + std::set objects() const; + + void deserialize(const std::map> + Data); + std::map + serialize(const std::vector Objects) const; + bool verify() const; + +public: + auto &getStorage() { return Storage; } +}; + +class AppendFooPipe { +public: + static constexpr llvm::StringRef Name = "AppendFooPipe"; + static constexpr revng::pypeline::PipeArgumentDocumentation + ArgumentsDocumentation[]{ { "Container", "" } }; + + const std::string StaticConfiguration; + + AppendFooPipe(llvm::StringRef Config) : StaticConfiguration(Config.str()) {} + + revng::pypeline::ObjectDependencies + run(const Model &TheModel, + const revng::pypeline::Request &Incoming, + const revng::pypeline::Request &Outgoing, + llvm::StringRef Configuration, + StringContainer &Container); +}; + +class AppendFooLibAnalysis { +public: + static constexpr llvm::StringRef Name = "AppendFooLibAnalysis"; + + llvm::Error run(Model &TheModel, + const revng::pypeline::Request &Incoming, + llvm::StringRef Configuration, + StringContainer &Container); +}; diff --git a/tests/pypeline/nanobind_test_annotations.py b/tests/pypeline/nanobind_test_annotations.py new file mode 100755 index 000000000..66fbb4605 --- /dev/null +++ b/tests/pypeline/nanobind_test_annotations.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 + +# +# This file is distributed under the MIT License. See LICENSE.md for details. +# + +# This test checks the stubs emitted by nanobind for missing casters. When a +# function returns a C++ type without a caster, the return type will be a +# string (a ast.Constant). For example, if one forgets to include +# `nanobind/stl/vector.h` then a function returning a +# `std::vector` will show up in stubs as: +# ``` +# def foo() -> "std::vector": ... # noqa: E800 +# ``` +# This program reads all the function definitions and +# checks that the annotations of each one do not contain a string. + +import ast +import sys + + +class NodeVisitor(ast.NodeVisitor): + def __init__(self): + super().__init__() + self.bad_annotations = set() + + def visit_AnnAssign(self, node: ast.AnnAssign): # noqa: N802 + self._check_constant(node.annotation) + + def visit_FunctionDef(self, node: ast.FunctionDef): # noqa: N802 + for arg in self._collect_arguments(node.args): + self._check_constant(arg.annotation) + self._check_constant(node.returns) + + @staticmethod + def _collect_arguments(arguments: ast.arguments) -> list[ast.arg]: + result: list[ast.arg] = [] + result.extend(arguments.posonlyargs) + result.extend(arguments.args) + if arguments.vararg is not None: + result.append(arguments.vararg) + result.extend(arguments.kwonlyargs) + if arguments.kwarg is not None: + result.append(arguments.kwarg) + return result + + def _check_constant(self, expr: ast.expr | None): + if not (expr is not None and isinstance(expr, ast.Constant)): + return + if isinstance(expr.value, str): + # TODO: could be a forward reference, but for now it's most likely + # a sign that a caster is missing. If needed in the future + # it'd be easy to have a `visit_ClassDef` method that removes + # entries if they are present + self.bad_annotations.add(expr.value) + + +def main(): + parsed_input = ast.parse(sys.stdin.read()) + node_visitor = NodeVisitor() + node_visitor.visit(parsed_input) + if len(node_visitor.bad_annotations) == 0: + return 0 + + for entry in node_visitor.bad_annotations: + print(f"Found string annotation for {entry}, probably a caster is missing!") + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/pypeline/nanobind_test_annotations.sh b/tests/pypeline/nanobind_test_annotations.sh new file mode 100755 index 000000000..6fafe22eb --- /dev/null +++ b/tests/pypeline/nanobind_test_annotations.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +# +# This file is distributed under the MIT License. See LICENSE.md for details. +# +set -euo pipefail + +SCRIPT_DIR=$(realpath "$(dirname "${BASH_SOURCE[0]}")") +ROOT_DIR="$1" +shift + +"$ROOT_DIR/scripts/nanobind_generate_stubs.py" "$@" | \ + "$SCRIPT_DIR/nanobind_test_annotations.py" diff --git a/tests/pypeline/python_cpp_test.py b/tests/pypeline/python_cpp_test.py new file mode 100755 index 000000000..25369e35d --- /dev/null +++ b/tests/pypeline/python_cpp_test.py @@ -0,0 +1,216 @@ +#!/usr/bin/env python3 + +# +# This file is distributed under the MIT License. See LICENSE.md for details. +# + +import sys +from collections.abc import Buffer + +import yaml + +from revng.internal.support import import_pipebox +from revng.pypeline import initialize_pypeline +from revng.pypeline.analysis import Analysis, AnalysisBinding +from revng.pypeline.container import Container, ContainerDeclaration +from revng.pypeline.model import Model, ReadOnlyModel +from revng.pypeline.object import Kind, ObjectID, ObjectSet +from revng.pypeline.pipeline import Pipeline +from revng.pypeline.pipeline_node import PipelineConfiguration, PipelineNode +from revng.pypeline.storage.memory import InMemoryStorageProvider +from revng.pypeline.storage.storage_provider import ContainerLocation +from revng.pypeline.task.pipe import Pipe +from revng.pypeline.task.requests import Requests +from revng.pypeline.task.savepoint import SavePoint +from revng.pypeline.task.task import TaskArgument, TaskArgumentAccess +from revng.pypeline.utils.registry import get_registry, get_singleton + + +def check_names(ext): + """Check that _pipebox has all the classes we expect it to have""" + + # Names that we know are always present in `_pipebox` + known_names = ("Buffer",) + + names = [ + x for x in dir(ext) if not ((x.startswith("__") and x.endswith("__")) or x in known_names) + ] + + # Check that the classes in the registry all came from the ext module + for type_ in (Analysis, Container, Kind, Model, ObjectID, Pipe): + registry = get_registry(type_) + for key, value in registry.items(): + assert issubclass(value, type_) + assert key in names + assert getattr(ext, key) is value + names.remove(key) + + # Check that all the classes have been registered, the names variable + # should be empty at this point + assert len(names) == 0 + + +def compare_dicts(dict1: dict[ObjectID, Buffer], dict2: dict[ObjectID, Buffer]): + assert all(isinstance(k, ObjectID) and isinstance(v, Buffer) for k, v in dict1.items()) + assert all(isinstance(k, ObjectID) and isinstance(v, Buffer) for k, v in dict2.items()) + assert dict1.keys() == dict2.keys() + for key in dict1.keys(): + assert memoryview(dict1[key]) == memoryview(dict2[key]) + + +def check_pipeline(): + """Check that all the components of fakePipebox behave as expected. This + function **should not** use the `_pipebox` module directly, rather + rely on the `get_registy` functionality to retrieve the classes""" + + # Create a new model + model = get_singleton(Model)() + assert isinstance(model, Model) + + # Check that the model is cloneable + model2 = model.clone() + assert isinstance(model2, Model) + + objectid_cls = get_singleton(ObjectID) + # Create a couple of ObjectIDs + foo = objectid_cls.deserialize("/function/0x400000:Code_x86_64") + bar = objectid_cls.deserialize("/function/0x400020:Code_x86_64") + assert isinstance(foo, ObjectID) + assert isinstance(bar, ObjectID) + # Check equality works + assert foo == objectid_cls.deserialize("/function/0x400000:Code_x86_64") + + # Create a StringContainer and deserialize some data to it + StringContainer = get_registry(Container)["StringContainer"] # noqa: N806 + container = StringContainer() + container.deserialize({foo: b"foo", bar: b"bar"}) + container_serialized = container.serialize(ObjectSet.from_list([foo])) + compare_dicts(container_serialized, {foo: b"foo"}) + # Re-feed the serialized data back to the container, this is to make sure + # that the `Buffer` C++ class is also handled by `deserialize` + container.deserialize(container_serialized) + + # Run the AppendFooPipe + AppendFooPipe = get_registry(Pipe)["AppendFooPipe"] # noqa: N806 + assert AppendFooPipe.signature() == ( + TaskArgument("Container", StringContainer, TaskArgumentAccess.READ_WRITE, ""), + ) + pipe = AppendFooPipe("{}") + pipe.run( + model=ReadOnlyModel(model), + containers=[container], + incoming=[ObjectSet(foo.kind())], + outgoing=[ObjectSet.from_list([foo])], + configuration="", + ) + container_serialized2 = container.serialize(ObjectSet.from_list([foo, bar])) + compare_dicts(container_serialized2, {foo: b"foofoo", bar: b"bar"}) + + # Run the AppendFooLibAnalysis + AppendFooLibAnalysis = get_registry(Analysis)["AppendFooLibAnalysis"] # noqa: N806 + assert AppendFooLibAnalysis.signature() == (StringContainer,) + analysis = AppendFooLibAnalysis() + analysis.run( + model=model, containers=[container], incoming=[ObjectSet.from_list([foo])], configuration="" + ) + + model_serialized = model.serialize() + assert isinstance(model_serialized, Buffer) + yaml_model = yaml.safe_load(bytes(model_serialized)) + assert "foo.so" in yaml_model["ImportedLibraries"] + + +def check_simple_pipeline(): + StringContainer = get_registry(Container)["StringContainer"] # noqa: N806 + AppendFooPipe = get_registry(Pipe)["AppendFooPipe"] # noqa: N806 + AppendFooLibAnalysis = get_registry(Analysis)["AppendFooLibAnalysis"] # noqa: N806 + + model = get_singleton(Model)() + storage_provider = InMemoryStorageProvider() + child_cont = ContainerDeclaration("Container", StringContainer) + declarations = [child_cont] + pipeline_configuration: PipelineConfiguration = {} + storage_provider.set_model(model.serialize()) + + # Create the pipeline + begin_node = PipelineNode(SavePoint("begin", to_save=declarations)) + inplace_node = PipelineNode(AppendFooPipe("{}"), bindings=[child_cont]) + end_node = PipelineNode(SavePoint("end", to_save=declarations)) + begin_node.add_successor(inplace_node) + inplace_node.add_successor(end_node) + pipeline = Pipeline( + set(declarations), + begin_node, + analyses={AnalysisBinding(AppendFooLibAnalysis(), (child_cont,), begin_node)}, + ) + + # Force the savepoint to store the objects + container = StringContainer() + + objectid_cls = get_singleton(ObjectID) + foo = objectid_cls.deserialize("/function/0x400000:Code_x86_64") + bar = objectid_cls.deserialize("/function/0x400020:Code_x86_64") + container.deserialize({foo: b"foo", bar: b"bar"}) + foo_bar_request = Requests({child_cont: ObjectSet.from_list([foo, bar])}) + + # Pre-populate the storage + begin_node.run( + model=ReadOnlyModel(model), + containers={child_cont: container}, + incoming=foo_bar_request, + outgoing=foo_bar_request, + pipeline_configuration=pipeline_configuration, + storage_provider=storage_provider, + ) + + # Check + begin_node_config = begin_node.configuration_id(pipeline_configuration) + container_location_begin = ContainerLocation(1, "Container", begin_node_config) + compare_dicts(storage_provider.storage[container_location_begin], {foo: b"foo", bar: b"bar"}) + + # Run a schedule + schedule = pipeline.schedule( + model=ReadOnlyModel(model), + target_node=end_node, + requests=Requests({child_cont: ObjectSet(foo.kind(), {foo})}), + pipeline_configuration=pipeline_configuration, + storage_provider=storage_provider, + ) + schedule.run( + model=ReadOnlyModel(model), + storage_provider=storage_provider, + ) + + # Check that schedule.serialize works + schedule.serialize() + + # Check + end_node_config = end_node.configuration_id(pipeline_configuration) + container_location_end = ContainerLocation(2, "Container", end_node_config) + compare_dicts(storage_provider.storage[container_location_end], {foo: b"foofoo"}) + + # Run analysis + new_model = pipeline.run_analysis( + model=ReadOnlyModel(model), + analysis_name="AppendFooLibAnalysis", + requests=Requests({child_cont: ObjectSet(foo.kind())}), + analysis_configuration="", + pipeline_configuration=pipeline_configuration, + storage_provider=storage_provider, + ) + + # Check + yaml_model = yaml.safe_load(bytes(new_model.serialize())) + assert "foo.so" in yaml_model["ImportedLibraries"] + + +def main(): + ext, _ = import_pipebox(sys.argv[1:]) + initialize_pypeline() + check_names(ext) + check_pipeline() + check_simple_pipeline() + + +if __name__ == "__main__": + main()