Introduce compatibility with revng-pypeline

Add the necessary machinery to allow `Pipe`s, `Analysis`es,
`Container`s, `Model` and `ObjectID` to be implemented in C++ and used
by Python.
This commit is contained in:
Giacomo Vercesi
2025-07-07 10:26:20 +02:00
committed by Alessandro Di Federico
parent c68b7f1ab6
commit 688b9fe111
32 changed files with 1891 additions and 14 deletions
+10 -1
View File
@@ -282,7 +282,16 @@ set(PYTHON_GENERATED_MODEL_PATH revng/model/_generated.py)
#
enable_testing()
find_package(Python REQUIRED)
#
# Find Python using the FindPython module. We request two components: *
# Interpreter: needed for some variables, such as $Python_SITELIB * Development:
# needed to compile nanobind module and friends, provides the
# $Python_INCLUDE_DIRS and $Python_LIBRARIES variables
#
find_package(
Python
COMPONENTS Interpreter Development
REQUIRED)
file(RELATIVE_PATH PYTHON_INSTALL_PATH "${CMAKE_INSTALL_PREFIX}"
"${Python_SITELIB}")
+35
View File
@@ -11,6 +11,8 @@
#include "llvm/ADT/StringRef.h"
#include "revng/ADT/Concepts.h"
namespace compile_time {
namespace detail {
@@ -94,6 +96,15 @@ constexpr std::optional<size_t> select(CallableType &&Callable) {
std::forward<CallableType>(Callable));
}
/// Calls \ref Callable on each element of \ref TupleType. Each time the element
/// type and index will be provided as template parameters.
template<SpecializationOf<std::tuple> TupleType, typename CallableType>
constexpr void forEach(CallableType &&Callable) {
repeat<std::tuple_size_v<TupleType>>([&Callable]<size_t I>() {
Callable.template operator()<std::tuple_element_t<I, TupleType>, I>();
});
}
namespace detail {
template<size_t N, size_t I = 0>
@@ -134,4 +145,28 @@ split(llvm::StringRef Separator, llvm::StringRef Input) {
return std::nullopt;
}
namespace detail {
template<typename>
struct ArrayTraits {};
template<typename T, size_t N>
struct ArrayTraits<T[N]> {
using value_type = T;
static constexpr size_t Size = N;
};
template<typename T, size_t N>
struct ArrayTraits<std::array<T, N>> {
using value_type = T;
static constexpr size_t Size = N;
};
} // namespace detail
/// Helper struct that reports the value_type and Size of an array at
/// compile-time
template<auto &T>
using ArrayTraits = detail::ArrayTraits<std::remove_reference_t<decltype(T)>>;
} // namespace compile_time
+50
View File
@@ -0,0 +1,50 @@
#pragma once
//
// This file is distributed under the MIT License. See LICENSE.md for details.
//
#include <vector>
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/StringRef.h"
#include "revng/PipeboxCommon/ObjectID.h"
namespace revng::pypeline {
/// Type used for both incoming and outgoing requests
/// Each index maps to the i-th container passed to the 'run' function
using Request = std::vector<std::vector<const ObjectID *>>;
/// Type representing a path in the model, returned in a set when an analysis
/// runs
using ModelPath = std::string;
/// Type used to return the dependencies of objects produced by a pipe
/// The first index maps to the i-th container of the pipe
using ObjectDependencies = std::vector<
std::vector<std::pair<ObjectID, ModelPath>>>;
/// Description of a single Pipe argument
struct PipeArgumentDocumentation {
/// Pretty-name for the argument, e.g. to show in the CLI
llvm::StringRef Name;
/// Long description of the container, to be used in help texts
llvm::StringRef HelpText;
};
class Buffer {
private:
llvm::SmallVector<char, 0> Vector;
public:
template<typename... T>
Buffer(T... Args) : Vector(std::forward<T>(Args)...) {}
llvm::SmallVector<char, 0> &data() { return Vector; }
// Read the contents
llvm::ArrayRef<char> data() const { return { Vector.data(), Vector.size() }; }
};
} // namespace revng::pypeline
+123
View File
@@ -0,0 +1,123 @@
#pragma once
//
// This file is distributed under the MIT License. See LICENSE.md for details.
//
#include <concepts>
#include "llvm/ADT/StringRef.h"
#include "revng/PipeboxCommon/Common.h"
#include "revng/PipeboxCommon/Model.h"
#include "revng/PipeboxCommon/ObjectID.h"
template<typename T>
concept HasName = requires {
{ T::Name } -> std::same_as<const llvm::StringRef &>;
requires not T::Name.empty();
};
//
// IsContainer
//
template<typename T>
concept IsContainer = requires(T &A, const T &AConst) {
requires HasName<T>;
{ T() } -> std::same_as<T>;
{ T::Kind } -> std::same_as<const Kind &>;
{ AConst.objects() } -> std::same_as<std::set<ObjectID>>;
{ AConst.verify() } -> std::same_as<bool>;
{
A.deserialize(std::declval<const std::map<const ObjectID *,
llvm::ArrayRef<const char>>>())
} -> std::same_as<void>;
{
AConst.serialize(std::declval<const std::vector<const ObjectID *>>())
} -> std::same_as<std::map<ObjectID, revng::pypeline::Buffer>>;
};
namespace detail {
template<typename T>
constexpr bool IsContainerReference = false;
template<typename T>
constexpr bool IsContainerReference<const T &> = IsContainer<T>;
template<typename T>
constexpr bool IsContainerReference<T &> = IsContainer<T>;
} // namespace detail
//
// IsAnalysis
//
namespace detail {
template<typename T>
struct AnalysisRunTraits {};
template<typename C, typename... Args>
requires(IsContainerReference<Args> and ...)
struct AnalysisRunTraits<llvm::Error (C::*)(Model &,
const revng::pypeline::Request &,
llvm::StringRef,
Args...)> {
using ContainerTypes = std::tuple<std::remove_reference_t<Args>...>;
static constexpr size_t Size = sizeof...(Args);
};
} // namespace detail
template<typename T>
using AnalysisRunTraits = detail::AnalysisRunTraits<decltype(&T::run)>;
template<typename T>
concept IsAnalysis = requires(T &A) {
requires HasName<T>;
{ T() } -> std::same_as<T>;
requires AnalysisRunTraits<T>::Size >= 0;
};
//
// IsPipe
//
namespace detail {
template<typename T>
struct PipeRunTraits {};
template<typename C, typename... Args>
requires(IsContainerReference<Args> and ...)
struct PipeRunTraits<
revng::pypeline::ObjectDependencies (C::*)(const Model &,
const revng::pypeline::Request &,
const revng::pypeline::Request &,
llvm::StringRef,
Args...)> {
using ContainerTypes = std::tuple<std::remove_reference_t<Args>...>;
static constexpr size_t Size = sizeof...(Args);
};
template<typename T>
using DocTraits = compile_time::ArrayTraits<T::ArgumentsDocumentation>;
} // namespace detail
template<typename T>
using PipeRunTraits = detail::PipeRunTraits<decltype(&T::run)>;
template<typename T>
concept IsPipe = requires(T &A, llvm::StringRef StaticConfig) {
requires HasName<T>;
{ T(StaticConfig) } -> std::same_as<T>;
{ A.StaticConfiguration } -> std::same_as<const std::string &>;
requires std::is_array_v<decltype(T::ArgumentsDocumentation)>;
requires std::is_same_v<typename detail::DocTraits<T>::value_type,
const revng::pypeline::PipeArgumentDocumentation>;
requires PipeRunTraits<T>::Size == detail::DocTraits<T>::Size;
};
@@ -0,0 +1,57 @@
#pragma once
//
// This file is distributed under the MIT License. See LICENSE.md for details.
//
#include <utility>
#include "llvm/Support/Error.h"
#include "revng/PipeboxCommon/Common.h"
#include "revng/PipeboxCommon/Concepts.h"
#include "revng/PipeboxCommon/Model.h"
namespace revng::pypeline::helpers {
/// Helper class that allows running an analysis
// The main hiccup in doing so is to unpack the arguments which are in some
// kind of sequence (e.g. std::vector) into arguments for the Analysis::run
// method. The type of the Analysis, the type of the sequence and how to unpack
// them are conveyed through the Info type, which cannot be a function pointer
// due to the unpacking function requiring template parameters.
template<typename ContainerListUnwrapper>
struct AnalysisRunner {
private:
template<size_t... I>
using integer_sequence = std::integer_sequence<size_t, I...>;
using ListType = ContainerListUnwrapper::ListType;
public:
template<IsAnalysis T, typename... ContainersT>
static llvm::Error run(T &Analysis,
llvm::Error (T::*RunMethod)(Model &,
const pypeline::Request &,
llvm::StringRef,
ContainersT...),
Model &TheModel,
const pypeline::Request &Incoming,
llvm::StringRef Configuration,
ListType Containers) {
revng_assert(Incoming.size() == sizeof...(ContainersT));
auto Runner = ([&]<size_t... ContainerIndexes>(const integer_sequence<
ContainerIndexes...> &) {
return (Analysis.*RunMethod)(TheModel,
Incoming,
Configuration,
ContainerListUnwrapper::template unwrap<
ContainersT,
ContainerIndexes>(Containers)...);
});
return Runner(std::make_integer_sequence<size_t, sizeof...(ContainersT)>());
}
};
} // namespace revng::pypeline::helpers
@@ -0,0 +1,60 @@
#pragma once
//
// This file is distributed under the MIT License. See LICENSE.md for details.
//
#include <utility>
#include "revng/PipeboxCommon/Common.h"
#include "revng/PipeboxCommon/Concepts.h"
#include "revng/PipeboxCommon/Model.h"
namespace revng::pypeline::helpers {
/// Helper class that allows running a pipe
// The main hiccup in doing so is to unpack the arguments which are in some
// kind of sequence (e.g. std::vector) into arguments for the Pipe::run
// method. The type of the Pipe, the type of the sequence and how to unpack
// them are conveyed through the Info type, which cannot be a function pointer
// due to the unpacking function requiring template parameters.
template<typename ContainerListUnwrapper>
struct PipeRunner {
private:
template<size_t... I>
using integer_sequence = std::integer_sequence<size_t, I...>;
using ObjectDeps = pypeline::ObjectDependencies;
using ListType = ContainerListUnwrapper::ListType;
public:
template<IsPipe T, typename... ContainersT>
static ObjectDeps run(T &Pipe,
ObjectDeps (T::*RunMethod)(const Model &,
const pypeline::Request &,
const pypeline::Request &,
llvm::StringRef,
ContainersT...),
const Model &TheModel,
const pypeline::Request &Incoming,
const pypeline::Request &Outgoing,
llvm::StringRef Configuration,
ListType Containers) {
revng_assert(Incoming.size() == sizeof...(ContainersT));
revng_assert(Outgoing.size() == sizeof...(ContainersT));
auto Runner = ([&]<size_t... ContainerIndexes>(const integer_sequence<
ContainerIndexes...> &) {
return (Pipe.*RunMethod)(TheModel,
Incoming,
Outgoing,
Configuration,
ContainerListUnwrapper::template unwrap<
ContainersT,
ContainerIndexes>(Containers)...);
});
return Runner(std::make_integer_sequence<size_t, sizeof...(ContainersT)>());
}
};
} // namespace revng::pypeline::helpers
@@ -0,0 +1,123 @@
#pragma once
//
// This file is distributed under the MIT License. See LICENSE.md for details.
//
#include "nanobind/nanobind.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/Support/Error.h"
#include "revng/PipeboxCommon/Common.h"
#include "revng/PipeboxCommon/Helpers/Python/Helpers.h"
NAMESPACE_BEGIN(NB_NAMESPACE)
NAMESPACE_BEGIN(detail)
template<>
struct type_caster<revng::pypeline::Request> {
NB_TYPE_CASTER(revng::pypeline::Request,
const_name("list[revng.pypeline.object.ObjectSet]"))
bool from_python(handle Source, uint8_t, cleanup_list *) {
using namespace revng::pypeline::helpers::python;
nanobind::object ObjectSet = importObject("revng.pypeline.object."
"ObjectSet");
revng_assert(nanobind::isinstance<nanobind::list>(Source));
for (const auto &OuterElement : Source) {
revng_assert(nanobind::isinstance(OuterElement, ObjectSet));
nanobind::object OuterSet = nanobind::getattr(OuterElement, "objects");
std::vector<const ObjectID *> Chunk;
for (const auto &Element : OuterSet)
Chunk.push_back(nanobind::cast<ObjectID *>(Element));
value.push_back(Chunk);
}
return true;
}
// from_cpp is not implemented since we don't support returning Request
// from C++
};
/// This is a caster class, it allows nanobind to automatically convert python
/// strings to `llvm::StringRef`s and back
///
/// Copied from nanobind's caster implementation for std::string_view
/// See the header "nanobind/stl/string_view.h"
template<>
struct type_caster<llvm::StringRef> {
NB_TYPE_CASTER(llvm::StringRef, const_name("str"))
bool from_python(handle Source, uint8_t, cleanup_list *) {
Py_ssize_t Size = 0;
const char *String = PyUnicode_AsUTF8AndSize(Source.ptr(), &Size);
if (String == NULL) {
PyErr_Clear();
return false;
}
value = llvm::StringRef(String, Size);
return true;
}
static handle from_cpp(llvm::StringRef Value, rv_policy, cleanup_list *) {
return PyUnicode_FromStringAndSize(Value.data(), Value.size());
}
};
namespace detail {
inline void llvmErrorToPythonException(llvm::Error &&Error) {
std::string Message = llvm::toString(std::move(Error));
// TODO: specialize the exception based on the returned llvm::Error
PyErr_SetString(PyExc_RuntimeError, Message.c_str());
}
} // namespace detail
/// This is a caster class for llvm::Error, it allows converting functions
/// returning it into a thrown python exception
template<>
struct type_caster<llvm::Error> {
NB_TYPE_CASTER(llvm::Error, const_name("None"))
// from_python is intentionally not defined because we do not support passing
// `llvm::Error`s as arguments
static handle from_cpp(llvm::Error &&Error, rv_policy, cleanup_list *) {
if (Error) {
detail::llvmErrorToPythonException(std::move(Error));
return {};
}
return nanobind::none().release();
}
};
/// This is a caster class for llvm::Expected<T>, it allows converting functions
/// returning it to either the wrapped T or a thrown python exception
/// Inspired by "nanobind/stl/detail/optional.h"
template<typename T>
struct type_caster<llvm::Expected<T>> {
using Caster = make_caster<T>;
NB_TYPE_CASTER(llvm::Expected<T>, Caster::Name)
// from_python is intentionally not defined because we do not support passing
// `llvm::Expected`s as arguments
static handle
from_cpp(llvm::Expected<T> &&Error, rv_policy Policy, cleanup_list *Cleanup) {
if (not Error) {
detail::llvmErrorToPythonException(Error.takeError());
return {};
}
return Caster::from_cpp(std::move(Error.get()), Policy, Cleanup);
}
};
NAMESPACE_END(detail)
NAMESPACE_END(NB_NAMESPACE)
@@ -0,0 +1,94 @@
#pragma once
//
// This file is distributed under the MIT License. See LICENSE.md for details.
//
#include "nanobind/nanobind.h"
#include "revng/PipeboxCommon/Common.h"
#include "revng/PipeboxCommon/Concepts.h"
#include "revng/PipeboxCommon/Helpers/Python/Helpers.h"
#include "revng/PipeboxCommon/ObjectID.h"
namespace revng::pypeline::helpers::python {
template<IsContainer T>
struct ContainerIO {
static nanobind::object objects(T &Handle) {
nanobind::object ObjectSet = importObject("revng.pypeline.object."
"ObjectSet");
std::set<ObjectID> Objects = Handle.objects();
nanobind::list Result;
for (const auto &Object : Objects)
Result.append(nanobind::cast(Object));
return ObjectSet(nanobind::cast(T::Kind), nanobind::set(Result));
}
static llvm::Error deserialize(T &Handle, nanobind::dict &Data) {
// Input map that will be fed to the Container's deserialize method
std::map<const ObjectID *, llvm::ArrayRef<const char>> Input;
// Temporary PyBuffer-s that are needed to read the contents of Data's
// values
std::vector<ManagedPyBuffer> Buffers;
for (const auto &[Key, Value] : Data) {
ObjectID *First = nanobind::cast<ObjectID *>(Key);
// Retrieve the value as a PyBuffer, this allows both bytes,
// revng::pypeline::Buffer-s and any other container implementing the
// buffer protocol to be used
revng_assert(PyObject_CheckBuffer(Value.ptr()) == 1);
auto MaybeBuffer = ManagedPyBuffer::make(Value.ptr(), PyBUF_SIMPLE);
if (std::holds_alternative<int>(MaybeBuffer)) {
// Return a success because the `PyObject_GetBuffer` already set the
// correct PyError
return llvm::Error::success();
}
Buffers.push_back(std::move(std::get<ManagedPyBuffer>(MaybeBuffer)));
ManagedPyBuffer &Buffer = Buffers.back();
// Buffer support 2D arrays, we requested PyBUF_SIMPLE which should be
// equivalent to void[], but check that is 1D anyways
if (Buffer->itemsize != 1 or Buffer->ndim != 1)
return revng::createError("Invalid buffer shape");
// Convert the buffer to ArrayRef
const char *DataPtr = static_cast<const char *>(Buffer->buf);
Input[First] = llvm::ArrayRef<const char>(DataPtr, Buffer->len);
}
Handle.deserialize(Input);
return llvm::Error::success();
}
static nanobind::dict serialize(T &Handle, nanobind::object ObjectSet) {
nanobind::object ObjectSetCls = importObject("revng.pypeline.object."
"ObjectSet");
revng_assert(nanobind::isinstance(ObjectSet, ObjectSetCls));
nanobind::object Objects = nanobind::getattr(ObjectSet, "objects");
// Convert the input Objects set into a vector of ObjectIDs
std::vector<const ObjectID *> CppObjects;
for (const auto &Object : Objects)
CppObjects.push_back(nanobind::cast<ObjectID *>(Object));
std::map<ObjectID, Buffer> Result = Handle.serialize(CppObjects);
// Manually convert the map above to a Python dict
nanobind::dict Return;
for (auto &Entry : Result) {
// Here the key is copied on purpose because it needs to be moved.
// We use the special syntax of nanobind::cast to move-construct the
// python object, so that the amount of memory copying is minimized
ObjectID KeyCopy = Entry.first;
nanobind::object Key = nanobind::cast<ObjectID>(std::move(KeyCopy));
nanobind::object Value = nanobind::cast<Buffer>(std::move(Entry.second));
Return[Key] = Value;
}
return Return;
};
};
} // namespace revng::pypeline::helpers::python
@@ -0,0 +1,75 @@
#pragma once
//
// This file is distributed under the MIT License. See LICENSE.md for details.
//
#include <vector>
#include "nanobind/nanobind.h"
#include "llvm/ADT/StringRef.h"
#include "revng/PipeboxCommon/Common.h"
#include "revng/PipeboxCommon/ObjectID.h"
namespace revng::pypeline::helpers::python {
inline nanobind::object importObject(llvm::StringRef String) {
auto [ModulePath, ObjectName] = String.rsplit('.');
nanobind::module_ Module = nanobind::module_::import_(ModulePath.str()
.c_str());
return Module.attr(ObjectName.str().c_str());
}
// Helper class to unpack containers from a nanobind::list.
// To be used in conjunction with PipeRunner or AnalysisRunner
class ContainerListUnwrapper {
public:
using ListType = nanobind::list &;
template<typename C, size_t I>
static C unwrap(ListType ContainerList) {
using C_ref_removed = std::remove_reference_t<C>;
return *nanobind::cast<C_ref_removed *>(ContainerList[I]);
}
};
class ManagedPyBuffer {
private:
Py_buffer Buffer;
ManagedPyBuffer() { Buffer.obj = NULL; }
public:
~ManagedPyBuffer() {
if (Buffer.obj != NULL)
PyBuffer_Release(&Buffer);
}
ManagedPyBuffer(const ManagedPyBuffer &) = delete;
ManagedPyBuffer &operator=(const ManagedPyBuffer &) = delete;
ManagedPyBuffer(ManagedPyBuffer &&Other) { *this = std::move(Other); }
ManagedPyBuffer &operator=(ManagedPyBuffer &&Other) {
if (this == &Other)
return *this;
Buffer = std::move(Other.Buffer);
Other.Buffer.obj = NULL;
return *this;
}
static std::variant<int, ManagedPyBuffer> make(PyObject *Obj, int Flags) {
ManagedPyBuffer Result;
int RC = PyObject_GetBuffer(Obj, &Result.Buffer, Flags);
if (RC != 0)
return RC;
return Result;
}
const Py_buffer &operator*() { return Buffer; }
const Py_buffer *operator->() { return &Buffer; }
};
} // namespace revng::pypeline::helpers::python
@@ -0,0 +1,47 @@
#pragma once
//
// This file is distributed under the MIT License. See LICENSE.md for details.
//
#include <vector>
#include "nanobind/nanobind.h"
namespace revng::pypeline::helpers::python {
struct BaseClasses {
nanobind::object BaseContainer;
nanobind::object BaseAnalysis;
nanobind::object BasePipe;
};
/// Registry class that allows registering functions that will modify the
/// provided Python module by adding Container, Analysis and Pipes subclasses.
class RegistryImpl {
private:
using PythonModuleInitializer = void (*)(nanobind::module_ &, BaseClasses &);
std::vector<PythonModuleInitializer> ModuleInitializers;
public:
RegistryImpl() = default;
~RegistryImpl() = default;
RegistryImpl(const RegistryImpl &) = delete;
RegistryImpl &operator=(const RegistryImpl &) = delete;
RegistryImpl(RegistryImpl &&) = delete;
RegistryImpl &operator=(RegistryImpl &&) = delete;
public:
void registerModuleInitializer(PythonModuleInitializer PMI) {
ModuleInitializers.push_back(PMI);
}
void callAll(nanobind::module_ &M, BaseClasses &BC) {
for (auto &Element : ModuleInitializers)
Element(M, BC);
}
};
inline RegistryImpl Registry;
} // namespace revng::pypeline::helpers::python
@@ -0,0 +1,32 @@
#pragma once
//
// This file is distributed under the MIT License. See LICENSE.md for details.
//
#include "nanobind/nanobind.h"
#include "revng/PipeboxCommon/Helpers/AnalysisRunner.h"
#include "revng/PipeboxCommon/Helpers/Python/Helpers.h"
#include "revng/PipeboxCommon/Model.h"
namespace revng::pypeline::helpers::python {
template<typename T>
inline llvm::Error runAnalysis(T &Handle,
nanobind::handle_t<Model> TheModel,
nanobind::list Containers,
Request Incoming,
llvm::StringRef Configuration) {
using namespace revng::pypeline::helpers::python;
Model *CppModel = nanobind::cast<Model *>(TheModel);
return AnalysisRunner<ContainerListUnwrapper>::run(Handle,
&T::run,
*CppModel,
Incoming,
Configuration,
Containers);
}
} // namespace revng::pypeline::helpers::python
@@ -0,0 +1,38 @@
#pragma once
//
// This file is distributed under the MIT License. See LICENSE.md for details.
//
#include "nanobind/nanobind.h"
#include "revng/PipeboxCommon/Common.h"
#include "revng/PipeboxCommon/Helpers/PipeRunner.h"
#include "revng/PipeboxCommon/Helpers/Python/Helpers.h"
#include "revng/PipeboxCommon/Model.h"
namespace revng::pypeline::helpers::python {
template<typename T>
inline ObjectDependencies runPipe(T &Handle,
nanobind::object TheModel,
nanobind::list Containers,
Request Incoming,
Request Outgoing,
llvm::StringRef Configuration) {
nanobind::object ReadOnlyModel = importObject("revng.pypeline.model."
"ReadOnlyModel");
revng_assert(nanobind::isinstance(TheModel, ReadOnlyModel));
nanobind::object ActualModel = nanobind::getattr(TheModel, "downcast")();
const Model *CppModel = nanobind::cast<Model *>(ActualModel);
return PipeRunner<ContainerListUnwrapper>::run(Handle,
&T::run,
*CppModel,
Incoming,
Outgoing,
Configuration,
Containers);
}
} // namespace revng::pypeline::helpers::python
@@ -0,0 +1,108 @@
#pragma once
//
// This file is distributed under the MIT License. See LICENSE.md for details.
//
#include <optional>
#include "nanobind/nanobind.h"
#include "revng/PipeboxCommon/Common.h"
#include "revng/PipeboxCommon/Concepts.h"
#include "revng/PipeboxCommon/Helpers/Python/Helpers.h"
namespace revng::pypeline::helpers::python {
template<typename T>
requires IsPipe<T> or IsAnalysis<T>
class SignatureHelper {
public:
/// Function that gets the signature of a given Pipe or Analysis type. Since
/// the signature might be requested many times, the result is cached in the
/// `_signature` attribute inside the class.
static nanobind::object getSignature() {
// Get the PyObject of the specified class T.
// We need to `borrow` it since we want to increase the refcount for the
// duration of this function.
nanobind::object Class = nanobind::borrow(nanobind::type<T>());
// Check that it's actually not NULL (e.g. not registered through nanobind)
revng_assert(Class.is_valid());
// Check if we previously computed the signature, if so return that
nanobind::object Signature = nanobind::getattr(Class,
"_signature",
nanobind::none());
if (not Signature.is_none())
return Signature;
// If here we've yet to compute the signature, do that and save it as an
// attribute inside the class
if constexpr (IsAnalysis<T>)
Signature = parseAnalysisSignature();
else
Signature = parsePipeSignature();
nanobind::setattr(Class, "_signature", Signature);
return Signature;
}
private:
/// Given a pipe type T, compute its signature
static nanobind::object parsePipeSignature() {
nanobind::object TaskArgument = importObject("revng.pypeline.task.task."
"TaskArgument");
nanobind::object TaskArgumentAccess = importObject("revng.pypeline.task."
"task."
"TaskArgumentAccess");
// Result list, this will be appended with a TaskArgument object for each
// argument of the pipe
nanobind::list Result;
using CT = PipeRunTraits<T>::ContainerTypes;
compile_time::forEach<CT>([&Result,
&TaskArgument,
&TaskArgumentAccess]<typename A, size_t I>() {
const PipeArgumentDocumentation &Argument = T::ArgumentsDocumentation[I];
// Create a Kwargs dictionary, this will be passed to the constructor of
// TaskArgument
nanobind::dict Kwargs;
Kwargs["name"] = nanobind::str(Argument.Name.data(),
Argument.Name.size());
Kwargs["help_text"] = nanobind::str(Argument.HelpText.data(),
Argument.HelpText.size());
// nanobind::type<T> returns a reference, so we need to borrow it and
// increase the reference count
Kwargs["container_type"] = nanobind::borrow(nanobind::type<A>());
// If the argument is const then the access is READ, otherwise READ_WRITE
if constexpr (std::is_const_v<A>)
Kwargs["access"] = nanobind::getattr(TaskArgumentAccess, "READ");
else
Kwargs["access"] = nanobind::getattr(TaskArgumentAccess, "READ_WRITE");
// kwargs_proxy is a special nanobind class that allows passing a
// nanobind::dict as kwargs, this is equivalent to doing
// `TaskArgument(**Kwargs)`.
Result.append(TaskArgument(nanobind::detail::kwargs_proxy(Kwargs)));
});
// This version of the nanobind::tuple constructor automatically converts
// the nanobind::list to a tuple.
return nanobind::tuple(Result);
}
/// Given an analysis type T, compute its signature
static nanobind::object parseAnalysisSignature() {
nanobind::list Result;
using CT = AnalysisRunTraits<T>::ContainerTypes;
compile_time::forEach<CT>([&Result]<typename A, size_t I>() {
// For each tuple element, add the python type to the Result list
Result.append(nanobind::borrow(nanobind::type<A>()));
});
return nanobind::tuple(Result);
}
};
} // namespace revng::pypeline::helpers::python
@@ -0,0 +1,96 @@
#pragma once
//
// This file is distributed under the MIT License. See LICENSE.md for details.
//
#include "nanobind/nanobind.h"
#include "nanobind/stl/optional.h"
#include "nanobind/stl/pair.h"
#include "nanobind/stl/set.h"
#include "nanobind/stl/string.h"
#include "nanobind/stl/vector.h"
#include "revng/PipeboxCommon/Concepts.h"
#include "revng/PipeboxCommon/Helpers/Python/Casters.h"
#include "revng/PipeboxCommon/Helpers/Python/ContainerIO.h"
#include "revng/PipeboxCommon/Helpers/Python/Registry.h"
#include "revng/PipeboxCommon/Helpers/Python/RunAnalysis.h"
#include "revng/PipeboxCommon/Helpers/Python/RunPipe.h"
#include "revng/PipeboxCommon/Helpers/Python/SignatureHelper.h"
template<IsAnalysis T>
struct RegisterAnalysis {
RegisterAnalysis() {
using namespace nanobind::literals;
using namespace revng::pypeline::helpers;
// Python
python::Registry.registerModuleInitializer([](nanobind::module_ &M,
python::BaseClasses &BC) {
nanobind::class_<T>(M, T::Name.data(), BC.BaseAnalysis)
.def_ro_static("name", &T::Name)
.def(nanobind::init<>())
.def_static("signature",
&python::SignatureHelper<T>::getSignature,
nanobind::sig("def signature() -> "
"tuple[type[revng.pypeline.container."
"Container], ...]"))
.def("run",
&python::runAnalysis<T>,
"model"_a,
"containers"_a,
"incoming"_a,
"configuration"_a);
});
}
};
template<IsContainer T>
struct RegisterContainer {
RegisterContainer() {
using namespace revng::pypeline::helpers;
// Python
python::Registry.registerModuleInitializer([](nanobind::module_ &M,
python::BaseClasses &BC) {
nanobind::class_<T>(M, T::Name.data(), BC.BaseContainer)
.def_ro_static("kind", &T::Kind)
.def(nanobind::init<>())
.def("objects", &python::ContainerIO<T>::objects)
.def("verify", &T::verify)
.def("deserialize", &python::ContainerIO<T>::deserialize)
.def("serialize", &python::ContainerIO<T>::serialize);
});
}
};
template<IsPipe T>
struct RegisterPipe {
RegisterPipe() {
using namespace nanobind::literals;
using namespace revng::pypeline::helpers;
// Python
python::Registry.registerModuleInitializer([](nanobind::module_ &M,
python::BaseClasses &BC) {
nanobind::class_<T>(M, T::Name.data(), BC.BasePipe)
.def_ro_static("name", &T::Name)
.def_static("signature",
&python::SignatureHelper<T>::getSignature,
nanobind::sig("def signature() -> "
"tuple[revng.pypeline.task.task.TaskArgument,"
" ...]"))
.def(nanobind::init<llvm::StringRef>())
.def_prop_ro("static_configuration",
[](T &Handle) { return Handle.StaticConfiguration; })
.def("run",
&python::runPipe<T>,
"model"_a,
"containers"_a,
"incoming"_a,
"outgoing"_a,
"configuration"_a);
});
}
};
+65
View File
@@ -0,0 +1,65 @@
#pragma once
//
// This file is distributed under the MIT License. See LICENSE.md for details.
//
#include "revng/Model/Binary.h"
#include "revng/PipeboxCommon/Common.h"
#include "revng/Support/MetaAddress.h"
class Model {
private:
TupleTree<model::Binary> TheModel;
public:
std::set<revng::pypeline::ModelPath> diff(const Model &Other) const {
auto Diff = ::diff(*TheModel.get(), *Other.TheModel.get());
std::set<revng::pypeline::ModelPath> Result;
for (auto &Entry : Diff.Changes) {
auto MaybePath = pathAsString<model::Binary>(Entry.Path);
Result.insert(MaybePath.value());
}
return Result;
}
Model clone() const { return *this; }
std::set<ObjectID> children(const ObjectID &Obj, Kind Kind) const {
if (Obj.kind() == Kinds::Binary and Kind == Kinds::Function) {
std::set<ObjectID> Result;
for (const model::Function &F : TheModel->Functions())
Result.insert(ObjectID(F.Entry()));
return Result;
}
if (Obj.kind() == Kinds::Binary and Kind == Kinds::TypeDefinition) {
std::set<ObjectID> Result;
for (const UpcastablePointer<model::TypeDefinition> &TD :
TheModel->TypeDefinitions())
Result.insert(ObjectID(TD->key()));
return Result;
}
revng_abort();
}
revng::pypeline::Buffer serialize() const {
revng::pypeline::Buffer Out;
llvm::raw_svector_ostream OS(Out.data());
TheModel.serialize(OS);
return Out;
}
llvm::Error deserialize(llvm::StringRef Input) {
auto MaybeModel = TupleTree<model::Binary>::fromString(Input);
if (not MaybeModel)
return MaybeModel.takeError();
TheModel = std::move(*MaybeModel);
return llvm::Error::success();
}
public:
TupleTree<model::Binary> &get() { return TheModel; }
const TupleTree<model::Binary> &get() const { return TheModel; }
};
+184
View File
@@ -0,0 +1,184 @@
#pragma once
//
// This file is distributed under the MIT License. See LICENSE.md for details.
//
#include <set>
#include "llvm/ADT/StringRef.h"
#include "revng/Model/TypeDefinition.h"
#include "revng/Pipeline/Location.h"
#include "revng/Pipes/Ranks.h"
#include "revng/Support/MetaAddress.h"
struct Kinds;
class Kind {
private:
enum class ValueType {
Binary = 0,
Function,
TypeDefinition,
};
public:
ValueType Value;
private:
constexpr Kind(ValueType Value) : Value(Value){};
friend std::hash<const Kind>;
friend Kinds;
public:
static std::vector<Kind> kinds() {
return { Kind{ ValueType::Binary },
Kind{ ValueType::Function },
Kind{ ValueType::TypeDefinition } };
};
std::optional<Kind> parent() {
if (Value == ValueType::Binary)
return std::nullopt;
else if (Value == ValueType::Function or Value == ValueType::TypeDefinition)
return Kind{ ValueType::Binary };
else
revng_abort();
}
static Kind deserialize(llvm::StringRef Value) {
if (Value == "binary") {
return Kind{ ValueType::Binary };
} else if (Value == "function") {
return Kind{ ValueType::Function };
} else if (Value == "type-definition") {
return Kind{ ValueType::TypeDefinition };
} else {
revng_abort();
}
}
std::string serlialize() {
if (Value == ValueType::Binary) {
return "binary";
} else if (Value == ValueType::Function) {
return "function";
} else if (Value == ValueType::TypeDefinition) {
return "type-definition";
} else {
revng_abort();
}
}
std::strong_ordering operator<=>(const Kind &) const = default;
};
template<>
struct std::hash<const Kind> {
uint64_t operator()(const Kind &Kind) const {
return std::hash<Kind::ValueType>{}(Kind.Value);
}
};
template<>
struct std::hash<Kind> : std::hash<const Kind> {};
struct Kinds {
static inline constexpr Kind Binary{ Kind::ValueType::Binary };
static inline constexpr Kind Function{ Kind::ValueType::Function };
static inline constexpr Kind TypeDefinition{
Kind::ValueType::TypeDefinition
};
};
class ObjectID {
private:
using TypeDefinitionKey = model::TypeDefinition::Key;
std::variant<std::monostate, MetaAddress, TypeDefinitionKey> Key;
public:
// Create a root ObjectID
ObjectID() = default;
// Create a Function ObjectID
ObjectID(const MetaAddress &Addr) : Key(Addr) {}
// Create a TypeDefinition ObjectID
ObjectID(const TypeDefinitionKey &Key) : Key(Key) {}
Kind kind() const {
auto Visitor = []<typename T>(const T &) {
if constexpr (std::is_same_v<T, std::monostate>)
return Kinds::Binary;
else if constexpr (std::is_same_v<T, MetaAddress>)
return Kinds::Function;
else if constexpr (std::is_same_v<T, TypeDefinitionKey>)
return Kinds::TypeDefinition;
else
revng_abort();
};
return std::visit(Visitor, Key);
}
std::optional<ObjectID> parent() const {
const Kind &TheKind = kind();
if (TheKind == Kinds::Binary)
return std::nullopt;
else if (TheKind == Kinds::Function or TheKind == Kinds::TypeDefinition)
return ObjectID();
else
revng_abort();
}
static ObjectID root() { return ObjectID(); }
std::string serialize() const {
using namespace revng::ranks;
using pipeline::locationString;
const Kind &TheKind = kind();
if (TheKind == Kinds::Binary)
return locationString(Binary);
else if (TheKind == Kinds::Function)
return locationString(Function, std::get<MetaAddress>(Key));
else if (TheKind == Kinds::TypeDefinition)
return locationString(TypeDefinition, std::get<TypeDefinitionKey>(Key));
else
revng_abort();
}
static llvm::Expected<ObjectID> deserialize(llvm::StringRef Input) {
using namespace revng::ranks;
using pipeline::locationFromString;
if (Input == "/binary") {
return ObjectID();
} else if (Input.starts_with("/function/")) {
auto MaybeKey = locationFromString(Function, Input);
if (not MaybeKey.has_value())
return revng::createError("Failed deserializing ObjectID");
MetaAddress Key = std::get<0>(std::get<0>(MaybeKey->tuple()));
return ObjectID(Key);
} else if (Input.starts_with("/type-definition/")) {
auto MaybeKey = locationFromString(TypeDefinition, Input);
if (not MaybeKey.has_value())
return revng::createError("Failed deserializing ObjectID");
TypeDefinitionKey Key = std::get<0>(MaybeKey->tuple());
return ObjectID(Key);
} else {
return revng::createError("Failed deserializing ObjectID");
}
}
std::strong_ordering operator<=>(const ObjectID &) const = default;
friend std::hash<const ObjectID>;
};
template<>
struct std::hash<const ObjectID> {
uint64_t operator()(const ObjectID &Obj) const {
return std::hash<decltype(Obj.Key)>{}(Obj.Key);
}
};
template<>
struct std::hash<ObjectID> : std::hash<const ObjectID> {};
+99
View File
@@ -0,0 +1,99 @@
//
// This file is distributed under the MIT License. See LICENSE.md for details.
//
#include "nanobind/nanobind.h"
#include "nanobind/stl/optional.h"
#include "nanobind/stl/set.h"
#include "nanobind/stl/string.h"
#include "nanobind/stl/vector.h"
#include "revng/PipeboxCommon/Helpers/Python/Casters.h"
#include "revng/PipeboxCommon/Helpers/Python/Helpers.h"
#include "revng/PipeboxCommon/Helpers/Python/Registry.h"
#include "revng/PipeboxCommon/Model.h"
#include "revng/Support/InitRevng.h"
NB_MODULE(_pipebox, m) {
{
// Do InitRevng, use a capsule to call the destructor when the Python module
// is unloaded
int Argc = 1;
const char *Argv[] = { "" };
const char **ArgvPtr = Argv;
nanobind::capsule Capsule(new revng::InitRevng(Argc, ArgvPtr, "", {}),
[](void *Ptr) noexcept {
delete static_cast<revng::InitRevng *>(Ptr);
});
nanobind::setattr(m, "__init_revng__", Capsule);
}
using namespace revng::pypeline::helpers::python;
// Register the Buffer class
nanobind::class_<revng::pypeline::Buffer>(m, "Buffer")
.def(nanobind::init<>())
.def("__buffer__", [](revng::pypeline::Buffer &Handle, int Flags) {
// PyMemoryView_FromMemory returns a new reference, hence the need to
// `steal` it with nanobind.
return nanobind::steal(PyMemoryView_FromMemory(Handle.data().data(),
Handle.data().size(),
Flags));
});
// Register ObjectID
nanobind::object ObjectIDBaseClass = importObject("revng.pypeline.object."
"ObjectID");
nanobind::class_<ObjectID>(m, "ObjectID", ObjectIDBaseClass)
.def(nanobind::init<>())
.def("kind", &ObjectID::kind)
.def("parent", &ObjectID::parent)
.def_static("root", &ObjectID::root)
.def("serialize", &ObjectID::serialize)
.def_static("deserialize", &ObjectID::deserialize)
.def("__eq__",
[](ObjectID &Handle, nanobind::object Other) {
ObjectID *OtherHandle;
if (not nanobind::try_cast<ObjectID *>(Other, OtherHandle))
return false;
return Handle == *OtherHandle;
})
.def("__hash__",
[](ObjectID &Handle) { return std::hash<ObjectID>{}(Handle); });
// Register Kind
nanobind::object KindBaseClass = importObject("revng.pypeline.object.Kind");
nanobind::class_<Kind>(m, "Kind", KindBaseClass)
.def_static("kinds", &Kind::kinds)
.def("parent", &Kind::parent)
.def_static("deserialize", &Kind::deserialize)
.def("serialize", &Kind::serlialize)
.def("__eq__",
[](Kind &Handle, nanobind::object Other) {
Kind *OtherHandle;
if (not nanobind::try_cast<Kind *>(Other, OtherHandle))
return false;
return Handle == *OtherHandle;
})
.def("__hash__", [](Kind &Handle) { return std::hash<Kind>{}(Handle); });
// Register Model
nanobind::object ModelBaseClass = importObject("revng.pypeline.model.Model");
nanobind::class_<Model>(m, "Model", ModelBaseClass)
.def(nanobind::init<>())
.def("diff",
[](Model &Handle, nanobind::handle_t<Model> Other) {
return Handle.diff(*nanobind::cast<Model *>(Other));
})
.def("clone", &Model::clone)
.def("serialize", &Model::serialize)
.def("deserialize", &Model::deserialize);
// Register all Pipes, Analyses and Containers
BaseClasses BC{
.BaseContainer = importObject("revng.pypeline.container.Container"),
.BaseAnalysis = importObject("revng.pypeline.analysis.Analysis"),
.BasePipe = importObject("revng.pypeline.task.pipe.Pipe"),
};
Registry.callAll(m, BC);
}
+31 -2
View File
@@ -154,8 +154,7 @@ python_module(
MODULE_INIT
revng/internal/__init__.py
MODULE_FILES
revng/internal/py.typed
revng/internal/pipebox.py)
revng/internal/py.typed)
#
# Install revng.model (including autogenerated classes)
@@ -416,6 +415,7 @@ set(REVNG_PYPELINE_MODULE_FILES
revng/pypeline/utils/registry.py
revng/pypeline/utils/cabc.py
revng/pypeline/utils/__init__.py)
python_module(
TARGET_NAME
revng-python-pypeline
@@ -426,6 +426,35 @@ python_module(
MODULE_FILES
${REVNG_PYPELINE_MODULE_FILES})
#
# Nanobind module generation and creation of revng.internal.{,_}cpp_pypeline
#
llvm_map_components_to_libnames(LLVM_LIBRARIES Support Core)
set(CPP_MODULE_NAME _pipebox)
add_library("${CPP_MODULE_NAME}" MODULE
"${CMAKE_SOURCE_DIR}/lib/PipeboxCommon/Python/_pipebox.cpp")
set_target_properties(
"${CPP_MODULE_NAME}"
PROPERTIES LINKER_LANGUAGE CXX
PREFIX ""
LIBRARY_OUTPUT_DIRECTORY
"${CMAKE_BINARY_DIR}/${PYTHON_INSTALL_PATH}/revng/internal")
target_link_libraries(
"${CPP_MODULE_NAME}" PUBLIC ${Python_LIBRARIES} ${LLVM_LIBRARIES} nanobind
revngSupport revngModel)
target_include_directories("${CPP_MODULE_NAME}" PUBLIC ${Python_INCLUDE_DIRS})
python_module(
TARGET_NAME
python-cpp-pipebox
WHEEL
revng_internal
MODULE_FILES
"revng/internal/pipebox.py"
MODULE_GENERATED_FILES
"revng/internal/${CPP_MODULE_NAME}.so")
#
# Generate actual wheel rules
#
+1
View File
@@ -42,6 +42,7 @@ build-backend = "setuptools.build_meta"
[tool.setuptools.package-data]
revng = [
"internal/py.typed",
"internal/_pipebox.so",
"internal/daemon/schema.graphql",
"internal/cli/_commands/generate_migrations/metaschema.yml",
"internal/cli/_commands/generate_migrations/migration.py.tpl",
+27 -1
View File
@@ -3,9 +3,11 @@
#
import os
import sys
from collections.abc import Iterable as CIterable
from ctypes import CDLL
from pathlib import Path
from typing import Iterable, TypeVar
from typing import Any, Iterable, TypeVar
from xdg import xdg_cache_home
@@ -25,3 +27,27 @@ def cache_directory() -> Path:
return Path(os.environ["REVNG_CACHE_DIR"])
else:
return xdg_cache_home() / "revng"
def import_pipebox(libraries: Iterable[str | Path]) -> tuple[Any, list[Any]]:
"""Load libraries and return the pipebox module"""
# Load the provided shared libraries, these contain static initializers
# like RegisterPipe<T> which allows registering Pipes, Analyses and
# Containers to be later created as python classes when the `_pipeline`
# module is imported.
#
# RTLD_GLOBAL is needed due to nanobind using weak symbols to derive the TypeID
handles = [CDLL(path, os.RTLD_NOW | os.RTLD_GLOBAL) for path in libraries]
# Check that the module hasn't been previously imported, otherwise result
# might be unpredictable
assert "revng.internal._pipebox" not in sys.modules
# Import the actual module, this will do the following:
# 1. Create actual classes for Model, ObjectID and Kind
# 2. Create, on the fly, classes that have been previously registered by
# the static initializers in the imported libraries above
import revng.internal._pipebox as ext
return (ext, handles)
-2
View File
@@ -21,8 +21,6 @@ class Analysis(ABC):
objects in save points.
"""
__slots__: tuple = ("name",)
def __init__(self, name: str):
"""
Initialize the analysis with a name.
+3 -5
View File
@@ -7,7 +7,7 @@ from __future__ import annotations
import json
from collections.abc import Mapping
from dataclasses import dataclass
from typing import Annotated, Dict, Generator, Optional, Tuple, Type
from typing import Annotated, Dict, Generator, Tuple, Type
from .object import Kind, ObjectID, ObjectSet
from .utils.cabc import ABC, abstractmethod
@@ -109,11 +109,9 @@ class Container(ABC):
"""
@abstractmethod
def serialize(self, objects: Optional[ObjectSet] = None) -> Mapping[ObjectID, bytes]:
def serialize(self, objects: ObjectSet) -> Mapping[ObjectID, bytes]:
"""
Dump objects from this container into a serialized format.
If objects is provided, only those objects will be dumped.
If not, all objects in the container will be dumped.
"""
@classmethod
@@ -182,7 +180,7 @@ def dump_container(
"""
with open(path, "w", encoding="utf-8") as f:
json.dump(
{k.serialize(): v.hex() for k, v in container.serialize().items()},
{k.serialize(): v.hex() for k, v in container.serialize(container.objects()).items()},
f,
indent=4,
sort_keys=True,
-2
View File
@@ -37,8 +37,6 @@ class Pipe(ABC):
argument should not be added.
"""
__slots__: tuple = ("name", "static_configuration")
def __init__(
self,
static_configuration: str = "",
+41
View File
@@ -0,0 +1,41 @@
#!/usr/bin/env python3
#
# This file is distributed under the MIT License. See LICENSE.md for details.
#
import argparse
import sys
from nanobind.stubgen import StubGen
from revng.internal.support import import_pipebox
def main():
parser = argparse.ArgumentParser()
parser.add_argument("-o", "--output", help="Output file (stdout if omitted)")
parser.add_argument(
"-l",
"--library",
default=[],
action="append",
help="Shared library to preload before generating stubs",
)
args = parser.parse_args()
module, handles = import_pipebox(args.library)
stub_gen = StubGen(module, quiet=False)
stub_gen.put(module)
stub_text = stub_gen.get()
if args.output is None:
sys.stdout.write(stub_text)
sys.stdout.flush()
else:
with open(args.output, "w") as f:
f.write(stub_text)
if __name__ == "__main__":
main()
+1 -1
View File
@@ -81,7 +81,7 @@
{ Regex: '^"clang/', Priority: -3 },
{ Regex: '^"mlir/', Priority: -4 },
{ Regex: '^"llvm/', Priority: -5 },
{ Regex: '^"aws/', Priority: -6 },
{ Regex: '^"nanobind/', Priority: -6 },
{ Regex: '^"boost/', Priority: -7 },
{ Regex: "^<", Priority: -8 },
],
+38
View File
@@ -19,3 +19,41 @@ set_tests_properties(
test_pypeline
PROPERTIES LABELS "pypeline" ENVIRONMENT
"PYTHONPATH=${CMAKE_BINARY_DIR}/${PYTHON_INSTALL_PATH}")
#
# Generate libmockPipeboxImpl and libmockPipebox
#
add_library(mockPipeboxImpl SHARED MockPipeboxImpl.cpp)
add_library(mockPipebox SHARED MockPipebox.cpp)
llvm_map_components_to_libnames(LLVM_LIBRARIES Support Core)
target_link_libraries(mockPipeboxImpl ${LLVM_LIBRARIES} revngSupport revngModel)
target_link_libraries(mockPipebox mockPipeboxImpl ${Python_LIBRARIES}
${LLVM_LIBRARIES} nanobind revngSupport)
target_include_directories(mockPipebox PUBLIC ${Python_INCLUDE_DIRS})
#
# Add tests that use libmockPipebox
#
revng_add_test(
NAME pypeline-cpp-test COMMAND
"${CMAKE_CURRENT_SOURCE_DIR}/python_cpp_test.py"
"${CMAKE_CURRENT_BINARY_DIR}/libmockPipebox.so")
set_tests_properties(
pypeline-cpp-test
PROPERTIES LABELS "pypeline;test" ENVIRONMENT
"PYTHONPATH=${CMAKE_BINARY_DIR}/${PYTHON_INSTALL_PATH}")
revng_add_test(
NAME
pypeline-annotations-test
COMMAND
"${CMAKE_CURRENT_SOURCE_DIR}/nanobind_test_annotations.sh"
"${CMAKE_SOURCE_DIR}"
-l
"${CMAKE_CURRENT_BINARY_DIR}/libmockPipebox.so")
set_tests_properties(
pypeline-annotations-test
PROPERTIES LABELS "pypeline;test" ENVIRONMENT
"PYTHONPATH=${CMAKE_BINARY_DIR}/${PYTHON_INSTALL_PATH}")
+11
View File
@@ -0,0 +1,11 @@
//
// This file is distributed under the MIT License. See LICENSE.md for details.
//
#include "revng/PipeboxCommon/Helpers/Registrars.h"
#include "MockPipeboxImpl.h"
static RegisterContainer<StringContainer> X;
static RegisterPipe<AppendFooPipe> Y;
static RegisterAnalysis<AppendFooLibAnalysis> Z;
+86
View File
@@ -0,0 +1,86 @@
//
// This file is distributed under the MIT License. See LICENSE.md for details.
//
#include <map>
#include <set>
#include <string>
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/StringRef.h"
#include "revng/PipeboxCommon/Model.h"
#include "revng/PipeboxCommon/ObjectID.h"
#include "MockPipeboxImpl.h"
//
// StringContainer
//
std::set<ObjectID> StringContainer::objects() const {
return std::views::keys(Storage) | revng::to<std::set<ObjectID>>();
}
void StringContainer::deserialize(const std::map<const ObjectID *,
llvm::ArrayRef<const char>>
Data) {
for (auto &Entry : Data) {
revng_assert(Entry.first->kind() == Kind);
std::string EntryData(Entry.second.data(), Entry.second.size());
Storage[*Entry.first] = std::move(EntryData);
}
}
std::map<ObjectID, revng::pypeline::Buffer>
StringContainer::serialize(const std::vector<const ObjectID *> Objects) const {
std::map<ObjectID, revng::pypeline::Buffer> Result;
for (auto &Element : Objects) {
const std::string &StoredElement = Storage.at(*Element);
llvm::ArrayRef<char> Ref = { StoredElement.data(), StoredElement.size() };
Result[*Element] = llvm::SmallVector<char, 0>{ Ref };
}
return Result;
}
bool StringContainer::verify() const {
return true;
}
//
// AppendFooPipe
//
revng::pypeline::ObjectDependencies
AppendFooPipe::run(const Model &TheModel,
const revng::pypeline::Request &Incoming,
const revng::pypeline::Request &Outgoing,
llvm::StringRef Configuration,
StringContainer &Container) {
revng_assert(Outgoing.size() == 1);
auto &Storage = Container.getStorage();
for (auto &Elem : Outgoing[0]) {
if (Storage.contains(*Elem)) {
Storage[*Elem] = Storage[*Elem] + "foo";
} else {
Storage[*Elem] = "foo";
}
}
return {};
}
//
// AppendFooLibAnalysis
//
llvm::Error AppendFooLibAnalysis::run(Model &TheModel,
const revng::pypeline::Request &Incoming,
llvm::StringRef Configuration,
StringContainer &Container) {
revng_assert(Incoming.size() == 1);
TheModel.get()->ImportedLibraries().insert("foo.so");
return llvm::Error::success();
}
+56
View File
@@ -0,0 +1,56 @@
#pragma once
//
// This file is distributed under the MIT License. See LICENSE.md for details.
//
#include "revng/PipeboxCommon/Common.h"
#include "revng/PipeboxCommon/Model.h"
#include "revng/PipeboxCommon/ObjectID.h"
class StringContainer {
private:
std::map<ObjectID, std::string> Storage;
public:
static constexpr llvm::StringRef Name = "StringContainer";
static constexpr Kind Kind = Kinds::Function;
std::set<ObjectID> objects() const;
void deserialize(const std::map<const ObjectID *, llvm::ArrayRef<const char>>
Data);
std::map<ObjectID, revng::pypeline::Buffer>
serialize(const std::vector<const ObjectID *> Objects) const;
bool verify() const;
public:
auto &getStorage() { return Storage; }
};
class AppendFooPipe {
public:
static constexpr llvm::StringRef Name = "AppendFooPipe";
static constexpr revng::pypeline::PipeArgumentDocumentation
ArgumentsDocumentation[]{ { "Container", "" } };
const std::string StaticConfiguration;
AppendFooPipe(llvm::StringRef Config) : StaticConfiguration(Config.str()) {}
revng::pypeline::ObjectDependencies
run(const Model &TheModel,
const revng::pypeline::Request &Incoming,
const revng::pypeline::Request &Outgoing,
llvm::StringRef Configuration,
StringContainer &Container);
};
class AppendFooLibAnalysis {
public:
static constexpr llvm::StringRef Name = "AppendFooLibAnalysis";
llvm::Error run(Model &TheModel,
const revng::pypeline::Request &Incoming,
llvm::StringRef Configuration,
StringContainer &Container);
};
+71
View File
@@ -0,0 +1,71 @@
#!/usr/bin/env python3
#
# This file is distributed under the MIT License. See LICENSE.md for details.
#
# This test checks the stubs emitted by nanobind for missing casters. When a
# function returns a C++ type without a caster, the return type will be a
# string (a ast.Constant). For example, if one forgets to include
# `nanobind/stl/vector.h` then a function returning a
# `std::vector<std::string>` will show up in stubs as:
# ```
# def foo() -> "std::vector<std::string>": ... # noqa: E800
# ```
# This program reads all the function definitions and
# checks that the annotations of each one do not contain a string.
import ast
import sys
class NodeVisitor(ast.NodeVisitor):
def __init__(self):
super().__init__()
self.bad_annotations = set()
def visit_AnnAssign(self, node: ast.AnnAssign): # noqa: N802
self._check_constant(node.annotation)
def visit_FunctionDef(self, node: ast.FunctionDef): # noqa: N802
for arg in self._collect_arguments(node.args):
self._check_constant(arg.annotation)
self._check_constant(node.returns)
@staticmethod
def _collect_arguments(arguments: ast.arguments) -> list[ast.arg]:
result: list[ast.arg] = []
result.extend(arguments.posonlyargs)
result.extend(arguments.args)
if arguments.vararg is not None:
result.append(arguments.vararg)
result.extend(arguments.kwonlyargs)
if arguments.kwarg is not None:
result.append(arguments.kwarg)
return result
def _check_constant(self, expr: ast.expr | None):
if not (expr is not None and isinstance(expr, ast.Constant)):
return
if isinstance(expr.value, str):
# TODO: could be a forward reference, but for now it's most likely
# a sign that a caster is missing. If needed in the future
# it'd be easy to have a `visit_ClassDef` method that removes
# entries if they are present
self.bad_annotations.add(expr.value)
def main():
parsed_input = ast.parse(sys.stdin.read())
node_visitor = NodeVisitor()
node_visitor.visit(parsed_input)
if len(node_visitor.bad_annotations) == 0:
return 0
for entry in node_visitor.bad_annotations:
print(f"Found string annotation for {entry}, probably a caster is missing!")
return 1
if __name__ == "__main__":
sys.exit(main())
+13
View File
@@ -0,0 +1,13 @@
#!/usr/bin/env bash
#
# This file is distributed under the MIT License. See LICENSE.md for details.
#
set -euo pipefail
SCRIPT_DIR=$(realpath "$(dirname "${BASH_SOURCE[0]}")")
ROOT_DIR="$1"
shift
"$ROOT_DIR/scripts/nanobind_generate_stubs.py" "$@" | \
"$SCRIPT_DIR/nanobind_test_annotations.py"
+216
View File
@@ -0,0 +1,216 @@
#!/usr/bin/env python3
#
# This file is distributed under the MIT License. See LICENSE.md for details.
#
import sys
from collections.abc import Buffer
import yaml
from revng.internal.support import import_pipebox
from revng.pypeline import initialize_pypeline
from revng.pypeline.analysis import Analysis, AnalysisBinding
from revng.pypeline.container import Container, ContainerDeclaration
from revng.pypeline.model import Model, ReadOnlyModel
from revng.pypeline.object import Kind, ObjectID, ObjectSet
from revng.pypeline.pipeline import Pipeline
from revng.pypeline.pipeline_node import PipelineConfiguration, PipelineNode
from revng.pypeline.storage.memory import InMemoryStorageProvider
from revng.pypeline.storage.storage_provider import ContainerLocation
from revng.pypeline.task.pipe import Pipe
from revng.pypeline.task.requests import Requests
from revng.pypeline.task.savepoint import SavePoint
from revng.pypeline.task.task import TaskArgument, TaskArgumentAccess
from revng.pypeline.utils.registry import get_registry, get_singleton
def check_names(ext):
"""Check that _pipebox has all the classes we expect it to have"""
# Names that we know are always present in `_pipebox`
known_names = ("Buffer",)
names = [
x for x in dir(ext) if not ((x.startswith("__") and x.endswith("__")) or x in known_names)
]
# Check that the classes in the registry all came from the ext module
for type_ in (Analysis, Container, Kind, Model, ObjectID, Pipe):
registry = get_registry(type_)
for key, value in registry.items():
assert issubclass(value, type_)
assert key in names
assert getattr(ext, key) is value
names.remove(key)
# Check that all the classes have been registered, the names variable
# should be empty at this point
assert len(names) == 0
def compare_dicts(dict1: dict[ObjectID, Buffer], dict2: dict[ObjectID, Buffer]):
assert all(isinstance(k, ObjectID) and isinstance(v, Buffer) for k, v in dict1.items())
assert all(isinstance(k, ObjectID) and isinstance(v, Buffer) for k, v in dict2.items())
assert dict1.keys() == dict2.keys()
for key in dict1.keys():
assert memoryview(dict1[key]) == memoryview(dict2[key])
def check_pipeline():
"""Check that all the components of fakePipebox behave as expected. This
function **should not** use the `_pipebox` module directly, rather
rely on the `get_registy` functionality to retrieve the classes"""
# Create a new model
model = get_singleton(Model)()
assert isinstance(model, Model)
# Check that the model is cloneable
model2 = model.clone()
assert isinstance(model2, Model)
objectid_cls = get_singleton(ObjectID)
# Create a couple of ObjectIDs
foo = objectid_cls.deserialize("/function/0x400000:Code_x86_64")
bar = objectid_cls.deserialize("/function/0x400020:Code_x86_64")
assert isinstance(foo, ObjectID)
assert isinstance(bar, ObjectID)
# Check equality works
assert foo == objectid_cls.deserialize("/function/0x400000:Code_x86_64")
# Create a StringContainer and deserialize some data to it
StringContainer = get_registry(Container)["StringContainer"] # noqa: N806
container = StringContainer()
container.deserialize({foo: b"foo", bar: b"bar"})
container_serialized = container.serialize(ObjectSet.from_list([foo]))
compare_dicts(container_serialized, {foo: b"foo"})
# Re-feed the serialized data back to the container, this is to make sure
# that the `Buffer` C++ class is also handled by `deserialize`
container.deserialize(container_serialized)
# Run the AppendFooPipe
AppendFooPipe = get_registry(Pipe)["AppendFooPipe"] # noqa: N806
assert AppendFooPipe.signature() == (
TaskArgument("Container", StringContainer, TaskArgumentAccess.READ_WRITE, ""),
)
pipe = AppendFooPipe("{}")
pipe.run(
model=ReadOnlyModel(model),
containers=[container],
incoming=[ObjectSet(foo.kind())],
outgoing=[ObjectSet.from_list([foo])],
configuration="",
)
container_serialized2 = container.serialize(ObjectSet.from_list([foo, bar]))
compare_dicts(container_serialized2, {foo: b"foofoo", bar: b"bar"})
# Run the AppendFooLibAnalysis
AppendFooLibAnalysis = get_registry(Analysis)["AppendFooLibAnalysis"] # noqa: N806
assert AppendFooLibAnalysis.signature() == (StringContainer,)
analysis = AppendFooLibAnalysis()
analysis.run(
model=model, containers=[container], incoming=[ObjectSet.from_list([foo])], configuration=""
)
model_serialized = model.serialize()
assert isinstance(model_serialized, Buffer)
yaml_model = yaml.safe_load(bytes(model_serialized))
assert "foo.so" in yaml_model["ImportedLibraries"]
def check_simple_pipeline():
StringContainer = get_registry(Container)["StringContainer"] # noqa: N806
AppendFooPipe = get_registry(Pipe)["AppendFooPipe"] # noqa: N806
AppendFooLibAnalysis = get_registry(Analysis)["AppendFooLibAnalysis"] # noqa: N806
model = get_singleton(Model)()
storage_provider = InMemoryStorageProvider()
child_cont = ContainerDeclaration("Container", StringContainer)
declarations = [child_cont]
pipeline_configuration: PipelineConfiguration = {}
storage_provider.set_model(model.serialize())
# Create the pipeline
begin_node = PipelineNode(SavePoint("begin", to_save=declarations))
inplace_node = PipelineNode(AppendFooPipe("{}"), bindings=[child_cont])
end_node = PipelineNode(SavePoint("end", to_save=declarations))
begin_node.add_successor(inplace_node)
inplace_node.add_successor(end_node)
pipeline = Pipeline(
set(declarations),
begin_node,
analyses={AnalysisBinding(AppendFooLibAnalysis(), (child_cont,), begin_node)},
)
# Force the savepoint to store the objects
container = StringContainer()
objectid_cls = get_singleton(ObjectID)
foo = objectid_cls.deserialize("/function/0x400000:Code_x86_64")
bar = objectid_cls.deserialize("/function/0x400020:Code_x86_64")
container.deserialize({foo: b"foo", bar: b"bar"})
foo_bar_request = Requests({child_cont: ObjectSet.from_list([foo, bar])})
# Pre-populate the storage
begin_node.run(
model=ReadOnlyModel(model),
containers={child_cont: container},
incoming=foo_bar_request,
outgoing=foo_bar_request,
pipeline_configuration=pipeline_configuration,
storage_provider=storage_provider,
)
# Check
begin_node_config = begin_node.configuration_id(pipeline_configuration)
container_location_begin = ContainerLocation(1, "Container", begin_node_config)
compare_dicts(storage_provider.storage[container_location_begin], {foo: b"foo", bar: b"bar"})
# Run a schedule
schedule = pipeline.schedule(
model=ReadOnlyModel(model),
target_node=end_node,
requests=Requests({child_cont: ObjectSet(foo.kind(), {foo})}),
pipeline_configuration=pipeline_configuration,
storage_provider=storage_provider,
)
schedule.run(
model=ReadOnlyModel(model),
storage_provider=storage_provider,
)
# Check that schedule.serialize works
schedule.serialize()
# Check
end_node_config = end_node.configuration_id(pipeline_configuration)
container_location_end = ContainerLocation(2, "Container", end_node_config)
compare_dicts(storage_provider.storage[container_location_end], {foo: b"foofoo"})
# Run analysis
new_model = pipeline.run_analysis(
model=ReadOnlyModel(model),
analysis_name="AppendFooLibAnalysis",
requests=Requests({child_cont: ObjectSet(foo.kind())}),
analysis_configuration="",
pipeline_configuration=pipeline_configuration,
storage_provider=storage_provider,
)
# Check
yaml_model = yaml.safe_load(bytes(new_model.serialize()))
assert "foo.so" in yaml_model["ImportedLibraries"]
def main():
ext, _ = import_pipebox(sys.argv[1:])
initialize_pypeline()
check_names(ext)
check_pipeline()
check_simple_pipeline()
if __name__ == "__main__":
main()