mirror of
https://github.com/revng/revng
synced 2026-06-21 14:07:57 +00:00
Introduce compatibility with revng-pypeline
Add the necessary machinery to allow `Pipe`s, `Analysis`es, `Container`s, `Model` and `ObjectID` to be implemented in C++ and used by Python.
This commit is contained in:
committed by
Alessandro Di Federico
parent
c68b7f1ab6
commit
688b9fe111
+10
-1
@@ -282,7 +282,16 @@ set(PYTHON_GENERATED_MODEL_PATH revng/model/_generated.py)
|
||||
#
|
||||
enable_testing()
|
||||
|
||||
find_package(Python REQUIRED)
|
||||
#
|
||||
# Find Python using the FindPython module. We request two components: *
|
||||
# Interpreter: needed for some variables, such as $Python_SITELIB * Development:
|
||||
# needed to compile nanobind module and friends, provides the
|
||||
# $Python_INCLUDE_DIRS and $Python_LIBRARIES variables
|
||||
#
|
||||
find_package(
|
||||
Python
|
||||
COMPONENTS Interpreter Development
|
||||
REQUIRED)
|
||||
file(RELATIVE_PATH PYTHON_INSTALL_PATH "${CMAKE_INSTALL_PREFIX}"
|
||||
"${Python_SITELIB}")
|
||||
|
||||
|
||||
@@ -11,6 +11,8 @@
|
||||
|
||||
#include "llvm/ADT/StringRef.h"
|
||||
|
||||
#include "revng/ADT/Concepts.h"
|
||||
|
||||
namespace compile_time {
|
||||
|
||||
namespace detail {
|
||||
@@ -94,6 +96,15 @@ constexpr std::optional<size_t> select(CallableType &&Callable) {
|
||||
std::forward<CallableType>(Callable));
|
||||
}
|
||||
|
||||
/// Calls \ref Callable on each element of \ref TupleType. Each time the element
|
||||
/// type and index will be provided as template parameters.
|
||||
template<SpecializationOf<std::tuple> TupleType, typename CallableType>
|
||||
constexpr void forEach(CallableType &&Callable) {
|
||||
repeat<std::tuple_size_v<TupleType>>([&Callable]<size_t I>() {
|
||||
Callable.template operator()<std::tuple_element_t<I, TupleType>, I>();
|
||||
});
|
||||
}
|
||||
|
||||
namespace detail {
|
||||
|
||||
template<size_t N, size_t I = 0>
|
||||
@@ -134,4 +145,28 @@ split(llvm::StringRef Separator, llvm::StringRef Input) {
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
namespace detail {
|
||||
|
||||
template<typename>
|
||||
struct ArrayTraits {};
|
||||
|
||||
template<typename T, size_t N>
|
||||
struct ArrayTraits<T[N]> {
|
||||
using value_type = T;
|
||||
static constexpr size_t Size = N;
|
||||
};
|
||||
|
||||
template<typename T, size_t N>
|
||||
struct ArrayTraits<std::array<T, N>> {
|
||||
using value_type = T;
|
||||
static constexpr size_t Size = N;
|
||||
};
|
||||
|
||||
} // namespace detail
|
||||
|
||||
/// Helper struct that reports the value_type and Size of an array at
|
||||
/// compile-time
|
||||
template<auto &T>
|
||||
using ArrayTraits = detail::ArrayTraits<std::remove_reference_t<decltype(T)>>;
|
||||
|
||||
} // namespace compile_time
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
#pragma once
|
||||
|
||||
//
|
||||
// This file is distributed under the MIT License. See LICENSE.md for details.
|
||||
//
|
||||
|
||||
#include <vector>
|
||||
|
||||
#include "llvm/ADT/ArrayRef.h"
|
||||
#include "llvm/ADT/SmallVector.h"
|
||||
#include "llvm/ADT/StringRef.h"
|
||||
|
||||
#include "revng/PipeboxCommon/ObjectID.h"
|
||||
|
||||
namespace revng::pypeline {
|
||||
|
||||
/// Type used for both incoming and outgoing requests
|
||||
/// Each index maps to the i-th container passed to the 'run' function
|
||||
using Request = std::vector<std::vector<const ObjectID *>>;
|
||||
/// Type representing a path in the model, returned in a set when an analysis
|
||||
/// runs
|
||||
using ModelPath = std::string;
|
||||
/// Type used to return the dependencies of objects produced by a pipe
|
||||
/// The first index maps to the i-th container of the pipe
|
||||
using ObjectDependencies = std::vector<
|
||||
std::vector<std::pair<ObjectID, ModelPath>>>;
|
||||
|
||||
/// Description of a single Pipe argument
|
||||
struct PipeArgumentDocumentation {
|
||||
/// Pretty-name for the argument, e.g. to show in the CLI
|
||||
llvm::StringRef Name;
|
||||
/// Long description of the container, to be used in help texts
|
||||
llvm::StringRef HelpText;
|
||||
};
|
||||
|
||||
class Buffer {
|
||||
private:
|
||||
llvm::SmallVector<char, 0> Vector;
|
||||
|
||||
public:
|
||||
template<typename... T>
|
||||
Buffer(T... Args) : Vector(std::forward<T>(Args)...) {}
|
||||
|
||||
llvm::SmallVector<char, 0> &data() { return Vector; }
|
||||
|
||||
// Read the contents
|
||||
llvm::ArrayRef<char> data() const { return { Vector.data(), Vector.size() }; }
|
||||
};
|
||||
|
||||
} // namespace revng::pypeline
|
||||
@@ -0,0 +1,123 @@
|
||||
#pragma once
|
||||
|
||||
//
|
||||
// This file is distributed under the MIT License. See LICENSE.md for details.
|
||||
//
|
||||
|
||||
#include <concepts>
|
||||
|
||||
#include "llvm/ADT/StringRef.h"
|
||||
|
||||
#include "revng/PipeboxCommon/Common.h"
|
||||
#include "revng/PipeboxCommon/Model.h"
|
||||
#include "revng/PipeboxCommon/ObjectID.h"
|
||||
|
||||
template<typename T>
|
||||
concept HasName = requires {
|
||||
{ T::Name } -> std::same_as<const llvm::StringRef &>;
|
||||
requires not T::Name.empty();
|
||||
};
|
||||
|
||||
//
|
||||
// IsContainer
|
||||
//
|
||||
|
||||
template<typename T>
|
||||
concept IsContainer = requires(T &A, const T &AConst) {
|
||||
requires HasName<T>;
|
||||
{ T() } -> std::same_as<T>;
|
||||
{ T::Kind } -> std::same_as<const Kind &>;
|
||||
{ AConst.objects() } -> std::same_as<std::set<ObjectID>>;
|
||||
{ AConst.verify() } -> std::same_as<bool>;
|
||||
{
|
||||
A.deserialize(std::declval<const std::map<const ObjectID *,
|
||||
llvm::ArrayRef<const char>>>())
|
||||
} -> std::same_as<void>;
|
||||
{
|
||||
AConst.serialize(std::declval<const std::vector<const ObjectID *>>())
|
||||
} -> std::same_as<std::map<ObjectID, revng::pypeline::Buffer>>;
|
||||
};
|
||||
|
||||
namespace detail {
|
||||
|
||||
template<typename T>
|
||||
constexpr bool IsContainerReference = false;
|
||||
|
||||
template<typename T>
|
||||
constexpr bool IsContainerReference<const T &> = IsContainer<T>;
|
||||
|
||||
template<typename T>
|
||||
constexpr bool IsContainerReference<T &> = IsContainer<T>;
|
||||
|
||||
} // namespace detail
|
||||
|
||||
//
|
||||
// IsAnalysis
|
||||
//
|
||||
|
||||
namespace detail {
|
||||
|
||||
template<typename T>
|
||||
struct AnalysisRunTraits {};
|
||||
|
||||
template<typename C, typename... Args>
|
||||
requires(IsContainerReference<Args> and ...)
|
||||
struct AnalysisRunTraits<llvm::Error (C::*)(Model &,
|
||||
const revng::pypeline::Request &,
|
||||
llvm::StringRef,
|
||||
Args...)> {
|
||||
using ContainerTypes = std::tuple<std::remove_reference_t<Args>...>;
|
||||
static constexpr size_t Size = sizeof...(Args);
|
||||
};
|
||||
|
||||
} // namespace detail
|
||||
|
||||
template<typename T>
|
||||
using AnalysisRunTraits = detail::AnalysisRunTraits<decltype(&T::run)>;
|
||||
|
||||
template<typename T>
|
||||
concept IsAnalysis = requires(T &A) {
|
||||
requires HasName<T>;
|
||||
{ T() } -> std::same_as<T>;
|
||||
requires AnalysisRunTraits<T>::Size >= 0;
|
||||
};
|
||||
|
||||
//
|
||||
// IsPipe
|
||||
//
|
||||
|
||||
namespace detail {
|
||||
|
||||
template<typename T>
|
||||
struct PipeRunTraits {};
|
||||
|
||||
template<typename C, typename... Args>
|
||||
requires(IsContainerReference<Args> and ...)
|
||||
struct PipeRunTraits<
|
||||
revng::pypeline::ObjectDependencies (C::*)(const Model &,
|
||||
const revng::pypeline::Request &,
|
||||
const revng::pypeline::Request &,
|
||||
llvm::StringRef,
|
||||
Args...)> {
|
||||
using ContainerTypes = std::tuple<std::remove_reference_t<Args>...>;
|
||||
static constexpr size_t Size = sizeof...(Args);
|
||||
};
|
||||
|
||||
template<typename T>
|
||||
using DocTraits = compile_time::ArrayTraits<T::ArgumentsDocumentation>;
|
||||
|
||||
} // namespace detail
|
||||
|
||||
template<typename T>
|
||||
using PipeRunTraits = detail::PipeRunTraits<decltype(&T::run)>;
|
||||
|
||||
template<typename T>
|
||||
concept IsPipe = requires(T &A, llvm::StringRef StaticConfig) {
|
||||
requires HasName<T>;
|
||||
{ T(StaticConfig) } -> std::same_as<T>;
|
||||
{ A.StaticConfiguration } -> std::same_as<const std::string &>;
|
||||
requires std::is_array_v<decltype(T::ArgumentsDocumentation)>;
|
||||
requires std::is_same_v<typename detail::DocTraits<T>::value_type,
|
||||
const revng::pypeline::PipeArgumentDocumentation>;
|
||||
requires PipeRunTraits<T>::Size == detail::DocTraits<T>::Size;
|
||||
};
|
||||
@@ -0,0 +1,57 @@
|
||||
#pragma once
|
||||
|
||||
//
|
||||
// This file is distributed under the MIT License. See LICENSE.md for details.
|
||||
//
|
||||
|
||||
#include <utility>
|
||||
|
||||
#include "llvm/Support/Error.h"
|
||||
|
||||
#include "revng/PipeboxCommon/Common.h"
|
||||
#include "revng/PipeboxCommon/Concepts.h"
|
||||
#include "revng/PipeboxCommon/Model.h"
|
||||
|
||||
namespace revng::pypeline::helpers {
|
||||
|
||||
/// Helper class that allows running an analysis
|
||||
// The main hiccup in doing so is to unpack the arguments which are in some
|
||||
// kind of sequence (e.g. std::vector) into arguments for the Analysis::run
|
||||
// method. The type of the Analysis, the type of the sequence and how to unpack
|
||||
// them are conveyed through the Info type, which cannot be a function pointer
|
||||
// due to the unpacking function requiring template parameters.
|
||||
template<typename ContainerListUnwrapper>
|
||||
struct AnalysisRunner {
|
||||
private:
|
||||
template<size_t... I>
|
||||
using integer_sequence = std::integer_sequence<size_t, I...>;
|
||||
|
||||
using ListType = ContainerListUnwrapper::ListType;
|
||||
|
||||
public:
|
||||
template<IsAnalysis T, typename... ContainersT>
|
||||
static llvm::Error run(T &Analysis,
|
||||
llvm::Error (T::*RunMethod)(Model &,
|
||||
const pypeline::Request &,
|
||||
llvm::StringRef,
|
||||
ContainersT...),
|
||||
Model &TheModel,
|
||||
const pypeline::Request &Incoming,
|
||||
llvm::StringRef Configuration,
|
||||
ListType Containers) {
|
||||
revng_assert(Incoming.size() == sizeof...(ContainersT));
|
||||
|
||||
auto Runner = ([&]<size_t... ContainerIndexes>(const integer_sequence<
|
||||
ContainerIndexes...> &) {
|
||||
return (Analysis.*RunMethod)(TheModel,
|
||||
Incoming,
|
||||
Configuration,
|
||||
ContainerListUnwrapper::template unwrap<
|
||||
ContainersT,
|
||||
ContainerIndexes>(Containers)...);
|
||||
});
|
||||
return Runner(std::make_integer_sequence<size_t, sizeof...(ContainersT)>());
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace revng::pypeline::helpers
|
||||
@@ -0,0 +1,60 @@
|
||||
#pragma once
|
||||
|
||||
//
|
||||
// This file is distributed under the MIT License. See LICENSE.md for details.
|
||||
//
|
||||
|
||||
#include <utility>
|
||||
|
||||
#include "revng/PipeboxCommon/Common.h"
|
||||
#include "revng/PipeboxCommon/Concepts.h"
|
||||
#include "revng/PipeboxCommon/Model.h"
|
||||
|
||||
namespace revng::pypeline::helpers {
|
||||
|
||||
/// Helper class that allows running a pipe
|
||||
// The main hiccup in doing so is to unpack the arguments which are in some
|
||||
// kind of sequence (e.g. std::vector) into arguments for the Pipe::run
|
||||
// method. The type of the Pipe, the type of the sequence and how to unpack
|
||||
// them are conveyed through the Info type, which cannot be a function pointer
|
||||
// due to the unpacking function requiring template parameters.
|
||||
template<typename ContainerListUnwrapper>
|
||||
struct PipeRunner {
|
||||
private:
|
||||
template<size_t... I>
|
||||
using integer_sequence = std::integer_sequence<size_t, I...>;
|
||||
using ObjectDeps = pypeline::ObjectDependencies;
|
||||
|
||||
using ListType = ContainerListUnwrapper::ListType;
|
||||
|
||||
public:
|
||||
template<IsPipe T, typename... ContainersT>
|
||||
static ObjectDeps run(T &Pipe,
|
||||
ObjectDeps (T::*RunMethod)(const Model &,
|
||||
const pypeline::Request &,
|
||||
const pypeline::Request &,
|
||||
llvm::StringRef,
|
||||
ContainersT...),
|
||||
const Model &TheModel,
|
||||
const pypeline::Request &Incoming,
|
||||
const pypeline::Request &Outgoing,
|
||||
llvm::StringRef Configuration,
|
||||
ListType Containers) {
|
||||
revng_assert(Incoming.size() == sizeof...(ContainersT));
|
||||
revng_assert(Outgoing.size() == sizeof...(ContainersT));
|
||||
|
||||
auto Runner = ([&]<size_t... ContainerIndexes>(const integer_sequence<
|
||||
ContainerIndexes...> &) {
|
||||
return (Pipe.*RunMethod)(TheModel,
|
||||
Incoming,
|
||||
Outgoing,
|
||||
Configuration,
|
||||
ContainerListUnwrapper::template unwrap<
|
||||
ContainersT,
|
||||
ContainerIndexes>(Containers)...);
|
||||
});
|
||||
return Runner(std::make_integer_sequence<size_t, sizeof...(ContainersT)>());
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace revng::pypeline::helpers
|
||||
@@ -0,0 +1,123 @@
|
||||
#pragma once
|
||||
|
||||
//
|
||||
// This file is distributed under the MIT License. See LICENSE.md for details.
|
||||
//
|
||||
|
||||
#include "nanobind/nanobind.h"
|
||||
|
||||
#include "llvm/ADT/StringRef.h"
|
||||
#include "llvm/Support/Error.h"
|
||||
|
||||
#include "revng/PipeboxCommon/Common.h"
|
||||
#include "revng/PipeboxCommon/Helpers/Python/Helpers.h"
|
||||
|
||||
NAMESPACE_BEGIN(NB_NAMESPACE)
|
||||
NAMESPACE_BEGIN(detail)
|
||||
|
||||
template<>
|
||||
struct type_caster<revng::pypeline::Request> {
|
||||
NB_TYPE_CASTER(revng::pypeline::Request,
|
||||
const_name("list[revng.pypeline.object.ObjectSet]"))
|
||||
|
||||
bool from_python(handle Source, uint8_t, cleanup_list *) {
|
||||
using namespace revng::pypeline::helpers::python;
|
||||
nanobind::object ObjectSet = importObject("revng.pypeline.object."
|
||||
"ObjectSet");
|
||||
|
||||
revng_assert(nanobind::isinstance<nanobind::list>(Source));
|
||||
for (const auto &OuterElement : Source) {
|
||||
revng_assert(nanobind::isinstance(OuterElement, ObjectSet));
|
||||
nanobind::object OuterSet = nanobind::getattr(OuterElement, "objects");
|
||||
|
||||
std::vector<const ObjectID *> Chunk;
|
||||
for (const auto &Element : OuterSet)
|
||||
Chunk.push_back(nanobind::cast<ObjectID *>(Element));
|
||||
|
||||
value.push_back(Chunk);
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
// from_cpp is not implemented since we don't support returning Request
|
||||
// from C++
|
||||
};
|
||||
|
||||
/// This is a caster class, it allows nanobind to automatically convert python
|
||||
/// strings to `llvm::StringRef`s and back
|
||||
///
|
||||
/// Copied from nanobind's caster implementation for std::string_view
|
||||
/// See the header "nanobind/stl/string_view.h"
|
||||
template<>
|
||||
struct type_caster<llvm::StringRef> {
|
||||
NB_TYPE_CASTER(llvm::StringRef, const_name("str"))
|
||||
|
||||
bool from_python(handle Source, uint8_t, cleanup_list *) {
|
||||
Py_ssize_t Size = 0;
|
||||
const char *String = PyUnicode_AsUTF8AndSize(Source.ptr(), &Size);
|
||||
if (String == NULL) {
|
||||
PyErr_Clear();
|
||||
return false;
|
||||
}
|
||||
value = llvm::StringRef(String, Size);
|
||||
return true;
|
||||
}
|
||||
|
||||
static handle from_cpp(llvm::StringRef Value, rv_policy, cleanup_list *) {
|
||||
return PyUnicode_FromStringAndSize(Value.data(), Value.size());
|
||||
}
|
||||
};
|
||||
|
||||
namespace detail {
|
||||
|
||||
inline void llvmErrorToPythonException(llvm::Error &&Error) {
|
||||
std::string Message = llvm::toString(std::move(Error));
|
||||
// TODO: specialize the exception based on the returned llvm::Error
|
||||
PyErr_SetString(PyExc_RuntimeError, Message.c_str());
|
||||
}
|
||||
|
||||
} // namespace detail
|
||||
|
||||
/// This is a caster class for llvm::Error, it allows converting functions
|
||||
/// returning it into a thrown python exception
|
||||
template<>
|
||||
struct type_caster<llvm::Error> {
|
||||
NB_TYPE_CASTER(llvm::Error, const_name("None"))
|
||||
|
||||
// from_python is intentionally not defined because we do not support passing
|
||||
// `llvm::Error`s as arguments
|
||||
|
||||
static handle from_cpp(llvm::Error &&Error, rv_policy, cleanup_list *) {
|
||||
if (Error) {
|
||||
detail::llvmErrorToPythonException(std::move(Error));
|
||||
return {};
|
||||
}
|
||||
return nanobind::none().release();
|
||||
}
|
||||
};
|
||||
|
||||
/// This is a caster class for llvm::Expected<T>, it allows converting functions
|
||||
/// returning it to either the wrapped T or a thrown python exception
|
||||
/// Inspired by "nanobind/stl/detail/optional.h"
|
||||
template<typename T>
|
||||
struct type_caster<llvm::Expected<T>> {
|
||||
using Caster = make_caster<T>;
|
||||
NB_TYPE_CASTER(llvm::Expected<T>, Caster::Name)
|
||||
|
||||
// from_python is intentionally not defined because we do not support passing
|
||||
// `llvm::Expected`s as arguments
|
||||
|
||||
static handle
|
||||
from_cpp(llvm::Expected<T> &&Error, rv_policy Policy, cleanup_list *Cleanup) {
|
||||
if (not Error) {
|
||||
detail::llvmErrorToPythonException(Error.takeError());
|
||||
return {};
|
||||
}
|
||||
|
||||
return Caster::from_cpp(std::move(Error.get()), Policy, Cleanup);
|
||||
}
|
||||
};
|
||||
|
||||
NAMESPACE_END(detail)
|
||||
NAMESPACE_END(NB_NAMESPACE)
|
||||
@@ -0,0 +1,94 @@
|
||||
#pragma once
|
||||
|
||||
//
|
||||
// This file is distributed under the MIT License. See LICENSE.md for details.
|
||||
//
|
||||
|
||||
#include "nanobind/nanobind.h"
|
||||
|
||||
#include "revng/PipeboxCommon/Common.h"
|
||||
#include "revng/PipeboxCommon/Concepts.h"
|
||||
#include "revng/PipeboxCommon/Helpers/Python/Helpers.h"
|
||||
#include "revng/PipeboxCommon/ObjectID.h"
|
||||
|
||||
namespace revng::pypeline::helpers::python {
|
||||
|
||||
template<IsContainer T>
|
||||
struct ContainerIO {
|
||||
static nanobind::object objects(T &Handle) {
|
||||
nanobind::object ObjectSet = importObject("revng.pypeline.object."
|
||||
"ObjectSet");
|
||||
std::set<ObjectID> Objects = Handle.objects();
|
||||
|
||||
nanobind::list Result;
|
||||
for (const auto &Object : Objects)
|
||||
Result.append(nanobind::cast(Object));
|
||||
|
||||
return ObjectSet(nanobind::cast(T::Kind), nanobind::set(Result));
|
||||
}
|
||||
|
||||
static llvm::Error deserialize(T &Handle, nanobind::dict &Data) {
|
||||
// Input map that will be fed to the Container's deserialize method
|
||||
std::map<const ObjectID *, llvm::ArrayRef<const char>> Input;
|
||||
// Temporary PyBuffer-s that are needed to read the contents of Data's
|
||||
// values
|
||||
std::vector<ManagedPyBuffer> Buffers;
|
||||
|
||||
for (const auto &[Key, Value] : Data) {
|
||||
ObjectID *First = nanobind::cast<ObjectID *>(Key);
|
||||
|
||||
// Retrieve the value as a PyBuffer, this allows both bytes,
|
||||
// revng::pypeline::Buffer-s and any other container implementing the
|
||||
// buffer protocol to be used
|
||||
revng_assert(PyObject_CheckBuffer(Value.ptr()) == 1);
|
||||
auto MaybeBuffer = ManagedPyBuffer::make(Value.ptr(), PyBUF_SIMPLE);
|
||||
if (std::holds_alternative<int>(MaybeBuffer)) {
|
||||
// Return a success because the `PyObject_GetBuffer` already set the
|
||||
// correct PyError
|
||||
return llvm::Error::success();
|
||||
}
|
||||
Buffers.push_back(std::move(std::get<ManagedPyBuffer>(MaybeBuffer)));
|
||||
ManagedPyBuffer &Buffer = Buffers.back();
|
||||
|
||||
// Buffer support 2D arrays, we requested PyBUF_SIMPLE which should be
|
||||
// equivalent to void[], but check that is 1D anyways
|
||||
if (Buffer->itemsize != 1 or Buffer->ndim != 1)
|
||||
return revng::createError("Invalid buffer shape");
|
||||
|
||||
// Convert the buffer to ArrayRef
|
||||
const char *DataPtr = static_cast<const char *>(Buffer->buf);
|
||||
Input[First] = llvm::ArrayRef<const char>(DataPtr, Buffer->len);
|
||||
}
|
||||
|
||||
Handle.deserialize(Input);
|
||||
return llvm::Error::success();
|
||||
}
|
||||
|
||||
static nanobind::dict serialize(T &Handle, nanobind::object ObjectSet) {
|
||||
nanobind::object ObjectSetCls = importObject("revng.pypeline.object."
|
||||
"ObjectSet");
|
||||
revng_assert(nanobind::isinstance(ObjectSet, ObjectSetCls));
|
||||
nanobind::object Objects = nanobind::getattr(ObjectSet, "objects");
|
||||
|
||||
// Convert the input Objects set into a vector of ObjectIDs
|
||||
std::vector<const ObjectID *> CppObjects;
|
||||
for (const auto &Object : Objects)
|
||||
CppObjects.push_back(nanobind::cast<ObjectID *>(Object));
|
||||
|
||||
std::map<ObjectID, Buffer> Result = Handle.serialize(CppObjects);
|
||||
// Manually convert the map above to a Python dict
|
||||
nanobind::dict Return;
|
||||
for (auto &Entry : Result) {
|
||||
// Here the key is copied on purpose because it needs to be moved.
|
||||
// We use the special syntax of nanobind::cast to move-construct the
|
||||
// python object, so that the amount of memory copying is minimized
|
||||
ObjectID KeyCopy = Entry.first;
|
||||
nanobind::object Key = nanobind::cast<ObjectID>(std::move(KeyCopy));
|
||||
nanobind::object Value = nanobind::cast<Buffer>(std::move(Entry.second));
|
||||
Return[Key] = Value;
|
||||
}
|
||||
return Return;
|
||||
};
|
||||
};
|
||||
|
||||
} // namespace revng::pypeline::helpers::python
|
||||
@@ -0,0 +1,75 @@
|
||||
#pragma once
|
||||
|
||||
//
|
||||
// This file is distributed under the MIT License. See LICENSE.md for details.
|
||||
//
|
||||
|
||||
#include <vector>
|
||||
|
||||
#include "nanobind/nanobind.h"
|
||||
|
||||
#include "llvm/ADT/StringRef.h"
|
||||
|
||||
#include "revng/PipeboxCommon/Common.h"
|
||||
#include "revng/PipeboxCommon/ObjectID.h"
|
||||
|
||||
namespace revng::pypeline::helpers::python {
|
||||
|
||||
inline nanobind::object importObject(llvm::StringRef String) {
|
||||
auto [ModulePath, ObjectName] = String.rsplit('.');
|
||||
nanobind::module_ Module = nanobind::module_::import_(ModulePath.str()
|
||||
.c_str());
|
||||
return Module.attr(ObjectName.str().c_str());
|
||||
}
|
||||
|
||||
// Helper class to unpack containers from a nanobind::list.
|
||||
// To be used in conjunction with PipeRunner or AnalysisRunner
|
||||
class ContainerListUnwrapper {
|
||||
public:
|
||||
using ListType = nanobind::list &;
|
||||
|
||||
template<typename C, size_t I>
|
||||
static C unwrap(ListType ContainerList) {
|
||||
using C_ref_removed = std::remove_reference_t<C>;
|
||||
return *nanobind::cast<C_ref_removed *>(ContainerList[I]);
|
||||
}
|
||||
};
|
||||
|
||||
class ManagedPyBuffer {
|
||||
private:
|
||||
Py_buffer Buffer;
|
||||
|
||||
ManagedPyBuffer() { Buffer.obj = NULL; }
|
||||
|
||||
public:
|
||||
~ManagedPyBuffer() {
|
||||
if (Buffer.obj != NULL)
|
||||
PyBuffer_Release(&Buffer);
|
||||
}
|
||||
|
||||
ManagedPyBuffer(const ManagedPyBuffer &) = delete;
|
||||
ManagedPyBuffer &operator=(const ManagedPyBuffer &) = delete;
|
||||
|
||||
ManagedPyBuffer(ManagedPyBuffer &&Other) { *this = std::move(Other); }
|
||||
ManagedPyBuffer &operator=(ManagedPyBuffer &&Other) {
|
||||
if (this == &Other)
|
||||
return *this;
|
||||
|
||||
Buffer = std::move(Other.Buffer);
|
||||
Other.Buffer.obj = NULL;
|
||||
return *this;
|
||||
}
|
||||
|
||||
static std::variant<int, ManagedPyBuffer> make(PyObject *Obj, int Flags) {
|
||||
ManagedPyBuffer Result;
|
||||
int RC = PyObject_GetBuffer(Obj, &Result.Buffer, Flags);
|
||||
if (RC != 0)
|
||||
return RC;
|
||||
return Result;
|
||||
}
|
||||
|
||||
const Py_buffer &operator*() { return Buffer; }
|
||||
const Py_buffer *operator->() { return &Buffer; }
|
||||
};
|
||||
|
||||
} // namespace revng::pypeline::helpers::python
|
||||
@@ -0,0 +1,47 @@
|
||||
#pragma once
|
||||
|
||||
//
|
||||
// This file is distributed under the MIT License. See LICENSE.md for details.
|
||||
//
|
||||
|
||||
#include <vector>
|
||||
|
||||
#include "nanobind/nanobind.h"
|
||||
|
||||
namespace revng::pypeline::helpers::python {
|
||||
|
||||
struct BaseClasses {
|
||||
nanobind::object BaseContainer;
|
||||
nanobind::object BaseAnalysis;
|
||||
nanobind::object BasePipe;
|
||||
};
|
||||
|
||||
/// Registry class that allows registering functions that will modify the
|
||||
/// provided Python module by adding Container, Analysis and Pipes subclasses.
|
||||
class RegistryImpl {
|
||||
private:
|
||||
using PythonModuleInitializer = void (*)(nanobind::module_ &, BaseClasses &);
|
||||
std::vector<PythonModuleInitializer> ModuleInitializers;
|
||||
|
||||
public:
|
||||
RegistryImpl() = default;
|
||||
~RegistryImpl() = default;
|
||||
RegistryImpl(const RegistryImpl &) = delete;
|
||||
RegistryImpl &operator=(const RegistryImpl &) = delete;
|
||||
RegistryImpl(RegistryImpl &&) = delete;
|
||||
RegistryImpl &operator=(RegistryImpl &&) = delete;
|
||||
|
||||
public:
|
||||
void registerModuleInitializer(PythonModuleInitializer PMI) {
|
||||
ModuleInitializers.push_back(PMI);
|
||||
}
|
||||
|
||||
void callAll(nanobind::module_ &M, BaseClasses &BC) {
|
||||
for (auto &Element : ModuleInitializers)
|
||||
Element(M, BC);
|
||||
}
|
||||
};
|
||||
|
||||
inline RegistryImpl Registry;
|
||||
|
||||
} // namespace revng::pypeline::helpers::python
|
||||
@@ -0,0 +1,32 @@
|
||||
#pragma once
|
||||
|
||||
//
|
||||
// This file is distributed under the MIT License. See LICENSE.md for details.
|
||||
//
|
||||
|
||||
#include "nanobind/nanobind.h"
|
||||
|
||||
#include "revng/PipeboxCommon/Helpers/AnalysisRunner.h"
|
||||
#include "revng/PipeboxCommon/Helpers/Python/Helpers.h"
|
||||
#include "revng/PipeboxCommon/Model.h"
|
||||
|
||||
namespace revng::pypeline::helpers::python {
|
||||
|
||||
template<typename T>
|
||||
inline llvm::Error runAnalysis(T &Handle,
|
||||
nanobind::handle_t<Model> TheModel,
|
||||
nanobind::list Containers,
|
||||
Request Incoming,
|
||||
llvm::StringRef Configuration) {
|
||||
using namespace revng::pypeline::helpers::python;
|
||||
Model *CppModel = nanobind::cast<Model *>(TheModel);
|
||||
|
||||
return AnalysisRunner<ContainerListUnwrapper>::run(Handle,
|
||||
&T::run,
|
||||
*CppModel,
|
||||
Incoming,
|
||||
Configuration,
|
||||
Containers);
|
||||
}
|
||||
|
||||
} // namespace revng::pypeline::helpers::python
|
||||
@@ -0,0 +1,38 @@
|
||||
#pragma once
|
||||
|
||||
//
|
||||
// This file is distributed under the MIT License. See LICENSE.md for details.
|
||||
//
|
||||
|
||||
#include "nanobind/nanobind.h"
|
||||
|
||||
#include "revng/PipeboxCommon/Common.h"
|
||||
#include "revng/PipeboxCommon/Helpers/PipeRunner.h"
|
||||
#include "revng/PipeboxCommon/Helpers/Python/Helpers.h"
|
||||
#include "revng/PipeboxCommon/Model.h"
|
||||
|
||||
namespace revng::pypeline::helpers::python {
|
||||
|
||||
template<typename T>
|
||||
inline ObjectDependencies runPipe(T &Handle,
|
||||
nanobind::object TheModel,
|
||||
nanobind::list Containers,
|
||||
Request Incoming,
|
||||
Request Outgoing,
|
||||
llvm::StringRef Configuration) {
|
||||
nanobind::object ReadOnlyModel = importObject("revng.pypeline.model."
|
||||
"ReadOnlyModel");
|
||||
revng_assert(nanobind::isinstance(TheModel, ReadOnlyModel));
|
||||
nanobind::object ActualModel = nanobind::getattr(TheModel, "downcast")();
|
||||
const Model *CppModel = nanobind::cast<Model *>(ActualModel);
|
||||
|
||||
return PipeRunner<ContainerListUnwrapper>::run(Handle,
|
||||
&T::run,
|
||||
*CppModel,
|
||||
Incoming,
|
||||
Outgoing,
|
||||
Configuration,
|
||||
Containers);
|
||||
}
|
||||
|
||||
} // namespace revng::pypeline::helpers::python
|
||||
@@ -0,0 +1,108 @@
|
||||
#pragma once
|
||||
|
||||
//
|
||||
// This file is distributed under the MIT License. See LICENSE.md for details.
|
||||
//
|
||||
|
||||
#include <optional>
|
||||
|
||||
#include "nanobind/nanobind.h"
|
||||
|
||||
#include "revng/PipeboxCommon/Common.h"
|
||||
#include "revng/PipeboxCommon/Concepts.h"
|
||||
#include "revng/PipeboxCommon/Helpers/Python/Helpers.h"
|
||||
|
||||
namespace revng::pypeline::helpers::python {
|
||||
|
||||
template<typename T>
|
||||
requires IsPipe<T> or IsAnalysis<T>
|
||||
class SignatureHelper {
|
||||
public:
|
||||
/// Function that gets the signature of a given Pipe or Analysis type. Since
|
||||
/// the signature might be requested many times, the result is cached in the
|
||||
/// `_signature` attribute inside the class.
|
||||
static nanobind::object getSignature() {
|
||||
// Get the PyObject of the specified class T.
|
||||
// We need to `borrow` it since we want to increase the refcount for the
|
||||
// duration of this function.
|
||||
nanobind::object Class = nanobind::borrow(nanobind::type<T>());
|
||||
// Check that it's actually not NULL (e.g. not registered through nanobind)
|
||||
revng_assert(Class.is_valid());
|
||||
// Check if we previously computed the signature, if so return that
|
||||
nanobind::object Signature = nanobind::getattr(Class,
|
||||
"_signature",
|
||||
nanobind::none());
|
||||
|
||||
if (not Signature.is_none())
|
||||
return Signature;
|
||||
|
||||
// If here we've yet to compute the signature, do that and save it as an
|
||||
// attribute inside the class
|
||||
if constexpr (IsAnalysis<T>)
|
||||
Signature = parseAnalysisSignature();
|
||||
else
|
||||
Signature = parsePipeSignature();
|
||||
nanobind::setattr(Class, "_signature", Signature);
|
||||
return Signature;
|
||||
}
|
||||
|
||||
private:
|
||||
/// Given a pipe type T, compute its signature
|
||||
static nanobind::object parsePipeSignature() {
|
||||
nanobind::object TaskArgument = importObject("revng.pypeline.task.task."
|
||||
"TaskArgument");
|
||||
nanobind::object TaskArgumentAccess = importObject("revng.pypeline.task."
|
||||
"task."
|
||||
"TaskArgumentAccess");
|
||||
// Result list, this will be appended with a TaskArgument object for each
|
||||
// argument of the pipe
|
||||
nanobind::list Result;
|
||||
|
||||
using CT = PipeRunTraits<T>::ContainerTypes;
|
||||
compile_time::forEach<CT>([&Result,
|
||||
&TaskArgument,
|
||||
&TaskArgumentAccess]<typename A, size_t I>() {
|
||||
const PipeArgumentDocumentation &Argument = T::ArgumentsDocumentation[I];
|
||||
// Create a Kwargs dictionary, this will be passed to the constructor of
|
||||
// TaskArgument
|
||||
nanobind::dict Kwargs;
|
||||
|
||||
Kwargs["name"] = nanobind::str(Argument.Name.data(),
|
||||
Argument.Name.size());
|
||||
Kwargs["help_text"] = nanobind::str(Argument.HelpText.data(),
|
||||
Argument.HelpText.size());
|
||||
// nanobind::type<T> returns a reference, so we need to borrow it and
|
||||
// increase the reference count
|
||||
Kwargs["container_type"] = nanobind::borrow(nanobind::type<A>());
|
||||
// If the argument is const then the access is READ, otherwise READ_WRITE
|
||||
if constexpr (std::is_const_v<A>)
|
||||
Kwargs["access"] = nanobind::getattr(TaskArgumentAccess, "READ");
|
||||
else
|
||||
Kwargs["access"] = nanobind::getattr(TaskArgumentAccess, "READ_WRITE");
|
||||
|
||||
// kwargs_proxy is a special nanobind class that allows passing a
|
||||
// nanobind::dict as kwargs, this is equivalent to doing
|
||||
// `TaskArgument(**Kwargs)`.
|
||||
Result.append(TaskArgument(nanobind::detail::kwargs_proxy(Kwargs)));
|
||||
});
|
||||
|
||||
// This version of the nanobind::tuple constructor automatically converts
|
||||
// the nanobind::list to a tuple.
|
||||
return nanobind::tuple(Result);
|
||||
}
|
||||
|
||||
/// Given an analysis type T, compute its signature
|
||||
static nanobind::object parseAnalysisSignature() {
|
||||
nanobind::list Result;
|
||||
|
||||
using CT = AnalysisRunTraits<T>::ContainerTypes;
|
||||
compile_time::forEach<CT>([&Result]<typename A, size_t I>() {
|
||||
// For each tuple element, add the python type to the Result list
|
||||
Result.append(nanobind::borrow(nanobind::type<A>()));
|
||||
});
|
||||
|
||||
return nanobind::tuple(Result);
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace revng::pypeline::helpers::python
|
||||
@@ -0,0 +1,96 @@
|
||||
#pragma once
|
||||
|
||||
//
|
||||
// This file is distributed under the MIT License. See LICENSE.md for details.
|
||||
//
|
||||
|
||||
#include "nanobind/nanobind.h"
|
||||
#include "nanobind/stl/optional.h"
|
||||
#include "nanobind/stl/pair.h"
|
||||
#include "nanobind/stl/set.h"
|
||||
#include "nanobind/stl/string.h"
|
||||
#include "nanobind/stl/vector.h"
|
||||
|
||||
#include "revng/PipeboxCommon/Concepts.h"
|
||||
#include "revng/PipeboxCommon/Helpers/Python/Casters.h"
|
||||
#include "revng/PipeboxCommon/Helpers/Python/ContainerIO.h"
|
||||
#include "revng/PipeboxCommon/Helpers/Python/Registry.h"
|
||||
#include "revng/PipeboxCommon/Helpers/Python/RunAnalysis.h"
|
||||
#include "revng/PipeboxCommon/Helpers/Python/RunPipe.h"
|
||||
#include "revng/PipeboxCommon/Helpers/Python/SignatureHelper.h"
|
||||
|
||||
template<IsAnalysis T>
|
||||
struct RegisterAnalysis {
|
||||
RegisterAnalysis() {
|
||||
using namespace nanobind::literals;
|
||||
using namespace revng::pypeline::helpers;
|
||||
|
||||
// Python
|
||||
python::Registry.registerModuleInitializer([](nanobind::module_ &M,
|
||||
python::BaseClasses &BC) {
|
||||
nanobind::class_<T>(M, T::Name.data(), BC.BaseAnalysis)
|
||||
.def_ro_static("name", &T::Name)
|
||||
.def(nanobind::init<>())
|
||||
.def_static("signature",
|
||||
&python::SignatureHelper<T>::getSignature,
|
||||
nanobind::sig("def signature() -> "
|
||||
"tuple[type[revng.pypeline.container."
|
||||
"Container], ...]"))
|
||||
.def("run",
|
||||
&python::runAnalysis<T>,
|
||||
"model"_a,
|
||||
"containers"_a,
|
||||
"incoming"_a,
|
||||
"configuration"_a);
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
template<IsContainer T>
|
||||
struct RegisterContainer {
|
||||
RegisterContainer() {
|
||||
using namespace revng::pypeline::helpers;
|
||||
|
||||
// Python
|
||||
python::Registry.registerModuleInitializer([](nanobind::module_ &M,
|
||||
python::BaseClasses &BC) {
|
||||
nanobind::class_<T>(M, T::Name.data(), BC.BaseContainer)
|
||||
.def_ro_static("kind", &T::Kind)
|
||||
.def(nanobind::init<>())
|
||||
.def("objects", &python::ContainerIO<T>::objects)
|
||||
.def("verify", &T::verify)
|
||||
.def("deserialize", &python::ContainerIO<T>::deserialize)
|
||||
.def("serialize", &python::ContainerIO<T>::serialize);
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
template<IsPipe T>
|
||||
struct RegisterPipe {
|
||||
RegisterPipe() {
|
||||
using namespace nanobind::literals;
|
||||
using namespace revng::pypeline::helpers;
|
||||
|
||||
// Python
|
||||
python::Registry.registerModuleInitializer([](nanobind::module_ &M,
|
||||
python::BaseClasses &BC) {
|
||||
nanobind::class_<T>(M, T::Name.data(), BC.BasePipe)
|
||||
.def_ro_static("name", &T::Name)
|
||||
.def_static("signature",
|
||||
&python::SignatureHelper<T>::getSignature,
|
||||
nanobind::sig("def signature() -> "
|
||||
"tuple[revng.pypeline.task.task.TaskArgument,"
|
||||
" ...]"))
|
||||
.def(nanobind::init<llvm::StringRef>())
|
||||
.def_prop_ro("static_configuration",
|
||||
[](T &Handle) { return Handle.StaticConfiguration; })
|
||||
.def("run",
|
||||
&python::runPipe<T>,
|
||||
"model"_a,
|
||||
"containers"_a,
|
||||
"incoming"_a,
|
||||
"outgoing"_a,
|
||||
"configuration"_a);
|
||||
});
|
||||
}
|
||||
};
|
||||
@@ -0,0 +1,65 @@
|
||||
#pragma once
|
||||
|
||||
//
|
||||
// This file is distributed under the MIT License. See LICENSE.md for details.
|
||||
//
|
||||
|
||||
#include "revng/Model/Binary.h"
|
||||
#include "revng/PipeboxCommon/Common.h"
|
||||
#include "revng/Support/MetaAddress.h"
|
||||
|
||||
class Model {
|
||||
private:
|
||||
TupleTree<model::Binary> TheModel;
|
||||
|
||||
public:
|
||||
std::set<revng::pypeline::ModelPath> diff(const Model &Other) const {
|
||||
auto Diff = ::diff(*TheModel.get(), *Other.TheModel.get());
|
||||
std::set<revng::pypeline::ModelPath> Result;
|
||||
for (auto &Entry : Diff.Changes) {
|
||||
auto MaybePath = pathAsString<model::Binary>(Entry.Path);
|
||||
Result.insert(MaybePath.value());
|
||||
}
|
||||
return Result;
|
||||
}
|
||||
|
||||
Model clone() const { return *this; }
|
||||
|
||||
std::set<ObjectID> children(const ObjectID &Obj, Kind Kind) const {
|
||||
if (Obj.kind() == Kinds::Binary and Kind == Kinds::Function) {
|
||||
std::set<ObjectID> Result;
|
||||
for (const model::Function &F : TheModel->Functions())
|
||||
Result.insert(ObjectID(F.Entry()));
|
||||
return Result;
|
||||
}
|
||||
|
||||
if (Obj.kind() == Kinds::Binary and Kind == Kinds::TypeDefinition) {
|
||||
std::set<ObjectID> Result;
|
||||
for (const UpcastablePointer<model::TypeDefinition> &TD :
|
||||
TheModel->TypeDefinitions())
|
||||
Result.insert(ObjectID(TD->key()));
|
||||
return Result;
|
||||
}
|
||||
revng_abort();
|
||||
}
|
||||
|
||||
revng::pypeline::Buffer serialize() const {
|
||||
revng::pypeline::Buffer Out;
|
||||
llvm::raw_svector_ostream OS(Out.data());
|
||||
TheModel.serialize(OS);
|
||||
return Out;
|
||||
}
|
||||
|
||||
llvm::Error deserialize(llvm::StringRef Input) {
|
||||
auto MaybeModel = TupleTree<model::Binary>::fromString(Input);
|
||||
if (not MaybeModel)
|
||||
return MaybeModel.takeError();
|
||||
|
||||
TheModel = std::move(*MaybeModel);
|
||||
return llvm::Error::success();
|
||||
}
|
||||
|
||||
public:
|
||||
TupleTree<model::Binary> &get() { return TheModel; }
|
||||
const TupleTree<model::Binary> &get() const { return TheModel; }
|
||||
};
|
||||
@@ -0,0 +1,184 @@
|
||||
#pragma once
|
||||
|
||||
//
|
||||
// This file is distributed under the MIT License. See LICENSE.md for details.
|
||||
//
|
||||
|
||||
#include <set>
|
||||
|
||||
#include "llvm/ADT/StringRef.h"
|
||||
|
||||
#include "revng/Model/TypeDefinition.h"
|
||||
#include "revng/Pipeline/Location.h"
|
||||
#include "revng/Pipes/Ranks.h"
|
||||
#include "revng/Support/MetaAddress.h"
|
||||
|
||||
struct Kinds;
|
||||
|
||||
class Kind {
|
||||
private:
|
||||
enum class ValueType {
|
||||
Binary = 0,
|
||||
Function,
|
||||
TypeDefinition,
|
||||
};
|
||||
|
||||
public:
|
||||
ValueType Value;
|
||||
|
||||
private:
|
||||
constexpr Kind(ValueType Value) : Value(Value){};
|
||||
friend std::hash<const Kind>;
|
||||
friend Kinds;
|
||||
|
||||
public:
|
||||
static std::vector<Kind> kinds() {
|
||||
return { Kind{ ValueType::Binary },
|
||||
Kind{ ValueType::Function },
|
||||
Kind{ ValueType::TypeDefinition } };
|
||||
};
|
||||
|
||||
std::optional<Kind> parent() {
|
||||
if (Value == ValueType::Binary)
|
||||
return std::nullopt;
|
||||
else if (Value == ValueType::Function or Value == ValueType::TypeDefinition)
|
||||
return Kind{ ValueType::Binary };
|
||||
else
|
||||
revng_abort();
|
||||
}
|
||||
|
||||
static Kind deserialize(llvm::StringRef Value) {
|
||||
if (Value == "binary") {
|
||||
return Kind{ ValueType::Binary };
|
||||
} else if (Value == "function") {
|
||||
return Kind{ ValueType::Function };
|
||||
} else if (Value == "type-definition") {
|
||||
return Kind{ ValueType::TypeDefinition };
|
||||
} else {
|
||||
revng_abort();
|
||||
}
|
||||
}
|
||||
|
||||
std::string serlialize() {
|
||||
if (Value == ValueType::Binary) {
|
||||
return "binary";
|
||||
} else if (Value == ValueType::Function) {
|
||||
return "function";
|
||||
} else if (Value == ValueType::TypeDefinition) {
|
||||
return "type-definition";
|
||||
} else {
|
||||
revng_abort();
|
||||
}
|
||||
}
|
||||
|
||||
std::strong_ordering operator<=>(const Kind &) const = default;
|
||||
};
|
||||
|
||||
template<>
|
||||
struct std::hash<const Kind> {
|
||||
uint64_t operator()(const Kind &Kind) const {
|
||||
return std::hash<Kind::ValueType>{}(Kind.Value);
|
||||
}
|
||||
};
|
||||
|
||||
template<>
|
||||
struct std::hash<Kind> : std::hash<const Kind> {};
|
||||
|
||||
struct Kinds {
|
||||
static inline constexpr Kind Binary{ Kind::ValueType::Binary };
|
||||
static inline constexpr Kind Function{ Kind::ValueType::Function };
|
||||
static inline constexpr Kind TypeDefinition{
|
||||
Kind::ValueType::TypeDefinition
|
||||
};
|
||||
};
|
||||
|
||||
class ObjectID {
|
||||
private:
|
||||
using TypeDefinitionKey = model::TypeDefinition::Key;
|
||||
std::variant<std::monostate, MetaAddress, TypeDefinitionKey> Key;
|
||||
|
||||
public:
|
||||
// Create a root ObjectID
|
||||
ObjectID() = default;
|
||||
// Create a Function ObjectID
|
||||
ObjectID(const MetaAddress &Addr) : Key(Addr) {}
|
||||
// Create a TypeDefinition ObjectID
|
||||
ObjectID(const TypeDefinitionKey &Key) : Key(Key) {}
|
||||
|
||||
Kind kind() const {
|
||||
auto Visitor = []<typename T>(const T &) {
|
||||
if constexpr (std::is_same_v<T, std::monostate>)
|
||||
return Kinds::Binary;
|
||||
else if constexpr (std::is_same_v<T, MetaAddress>)
|
||||
return Kinds::Function;
|
||||
else if constexpr (std::is_same_v<T, TypeDefinitionKey>)
|
||||
return Kinds::TypeDefinition;
|
||||
else
|
||||
revng_abort();
|
||||
};
|
||||
return std::visit(Visitor, Key);
|
||||
}
|
||||
|
||||
std::optional<ObjectID> parent() const {
|
||||
const Kind &TheKind = kind();
|
||||
if (TheKind == Kinds::Binary)
|
||||
return std::nullopt;
|
||||
else if (TheKind == Kinds::Function or TheKind == Kinds::TypeDefinition)
|
||||
return ObjectID();
|
||||
else
|
||||
revng_abort();
|
||||
}
|
||||
|
||||
static ObjectID root() { return ObjectID(); }
|
||||
|
||||
std::string serialize() const {
|
||||
using namespace revng::ranks;
|
||||
using pipeline::locationString;
|
||||
const Kind &TheKind = kind();
|
||||
|
||||
if (TheKind == Kinds::Binary)
|
||||
return locationString(Binary);
|
||||
else if (TheKind == Kinds::Function)
|
||||
return locationString(Function, std::get<MetaAddress>(Key));
|
||||
else if (TheKind == Kinds::TypeDefinition)
|
||||
return locationString(TypeDefinition, std::get<TypeDefinitionKey>(Key));
|
||||
else
|
||||
revng_abort();
|
||||
}
|
||||
|
||||
static llvm::Expected<ObjectID> deserialize(llvm::StringRef Input) {
|
||||
using namespace revng::ranks;
|
||||
using pipeline::locationFromString;
|
||||
|
||||
if (Input == "/binary") {
|
||||
return ObjectID();
|
||||
} else if (Input.starts_with("/function/")) {
|
||||
auto MaybeKey = locationFromString(Function, Input);
|
||||
if (not MaybeKey.has_value())
|
||||
return revng::createError("Failed deserializing ObjectID");
|
||||
MetaAddress Key = std::get<0>(std::get<0>(MaybeKey->tuple()));
|
||||
return ObjectID(Key);
|
||||
} else if (Input.starts_with("/type-definition/")) {
|
||||
auto MaybeKey = locationFromString(TypeDefinition, Input);
|
||||
if (not MaybeKey.has_value())
|
||||
return revng::createError("Failed deserializing ObjectID");
|
||||
TypeDefinitionKey Key = std::get<0>(MaybeKey->tuple());
|
||||
return ObjectID(Key);
|
||||
} else {
|
||||
return revng::createError("Failed deserializing ObjectID");
|
||||
}
|
||||
}
|
||||
|
||||
std::strong_ordering operator<=>(const ObjectID &) const = default;
|
||||
friend std::hash<const ObjectID>;
|
||||
};
|
||||
|
||||
template<>
|
||||
struct std::hash<const ObjectID> {
|
||||
uint64_t operator()(const ObjectID &Obj) const {
|
||||
return std::hash<decltype(Obj.Key)>{}(Obj.Key);
|
||||
}
|
||||
};
|
||||
|
||||
template<>
|
||||
struct std::hash<ObjectID> : std::hash<const ObjectID> {};
|
||||
@@ -0,0 +1,99 @@
|
||||
//
|
||||
// This file is distributed under the MIT License. See LICENSE.md for details.
|
||||
//
|
||||
|
||||
#include "nanobind/nanobind.h"
|
||||
#include "nanobind/stl/optional.h"
|
||||
#include "nanobind/stl/set.h"
|
||||
#include "nanobind/stl/string.h"
|
||||
#include "nanobind/stl/vector.h"
|
||||
|
||||
#include "revng/PipeboxCommon/Helpers/Python/Casters.h"
|
||||
#include "revng/PipeboxCommon/Helpers/Python/Helpers.h"
|
||||
#include "revng/PipeboxCommon/Helpers/Python/Registry.h"
|
||||
#include "revng/PipeboxCommon/Model.h"
|
||||
#include "revng/Support/InitRevng.h"
|
||||
|
||||
NB_MODULE(_pipebox, m) {
|
||||
{
|
||||
// Do InitRevng, use a capsule to call the destructor when the Python module
|
||||
// is unloaded
|
||||
int Argc = 1;
|
||||
const char *Argv[] = { "" };
|
||||
const char **ArgvPtr = Argv;
|
||||
nanobind::capsule Capsule(new revng::InitRevng(Argc, ArgvPtr, "", {}),
|
||||
[](void *Ptr) noexcept {
|
||||
delete static_cast<revng::InitRevng *>(Ptr);
|
||||
});
|
||||
nanobind::setattr(m, "__init_revng__", Capsule);
|
||||
}
|
||||
|
||||
using namespace revng::pypeline::helpers::python;
|
||||
|
||||
// Register the Buffer class
|
||||
nanobind::class_<revng::pypeline::Buffer>(m, "Buffer")
|
||||
.def(nanobind::init<>())
|
||||
.def("__buffer__", [](revng::pypeline::Buffer &Handle, int Flags) {
|
||||
// PyMemoryView_FromMemory returns a new reference, hence the need to
|
||||
// `steal` it with nanobind.
|
||||
return nanobind::steal(PyMemoryView_FromMemory(Handle.data().data(),
|
||||
Handle.data().size(),
|
||||
Flags));
|
||||
});
|
||||
|
||||
// Register ObjectID
|
||||
nanobind::object ObjectIDBaseClass = importObject("revng.pypeline.object."
|
||||
"ObjectID");
|
||||
nanobind::class_<ObjectID>(m, "ObjectID", ObjectIDBaseClass)
|
||||
.def(nanobind::init<>())
|
||||
.def("kind", &ObjectID::kind)
|
||||
.def("parent", &ObjectID::parent)
|
||||
.def_static("root", &ObjectID::root)
|
||||
.def("serialize", &ObjectID::serialize)
|
||||
.def_static("deserialize", &ObjectID::deserialize)
|
||||
.def("__eq__",
|
||||
[](ObjectID &Handle, nanobind::object Other) {
|
||||
ObjectID *OtherHandle;
|
||||
if (not nanobind::try_cast<ObjectID *>(Other, OtherHandle))
|
||||
return false;
|
||||
return Handle == *OtherHandle;
|
||||
})
|
||||
.def("__hash__",
|
||||
[](ObjectID &Handle) { return std::hash<ObjectID>{}(Handle); });
|
||||
|
||||
// Register Kind
|
||||
nanobind::object KindBaseClass = importObject("revng.pypeline.object.Kind");
|
||||
nanobind::class_<Kind>(m, "Kind", KindBaseClass)
|
||||
.def_static("kinds", &Kind::kinds)
|
||||
.def("parent", &Kind::parent)
|
||||
.def_static("deserialize", &Kind::deserialize)
|
||||
.def("serialize", &Kind::serlialize)
|
||||
.def("__eq__",
|
||||
[](Kind &Handle, nanobind::object Other) {
|
||||
Kind *OtherHandle;
|
||||
if (not nanobind::try_cast<Kind *>(Other, OtherHandle))
|
||||
return false;
|
||||
return Handle == *OtherHandle;
|
||||
})
|
||||
.def("__hash__", [](Kind &Handle) { return std::hash<Kind>{}(Handle); });
|
||||
|
||||
// Register Model
|
||||
nanobind::object ModelBaseClass = importObject("revng.pypeline.model.Model");
|
||||
nanobind::class_<Model>(m, "Model", ModelBaseClass)
|
||||
.def(nanobind::init<>())
|
||||
.def("diff",
|
||||
[](Model &Handle, nanobind::handle_t<Model> Other) {
|
||||
return Handle.diff(*nanobind::cast<Model *>(Other));
|
||||
})
|
||||
.def("clone", &Model::clone)
|
||||
.def("serialize", &Model::serialize)
|
||||
.def("deserialize", &Model::deserialize);
|
||||
|
||||
// Register all Pipes, Analyses and Containers
|
||||
BaseClasses BC{
|
||||
.BaseContainer = importObject("revng.pypeline.container.Container"),
|
||||
.BaseAnalysis = importObject("revng.pypeline.analysis.Analysis"),
|
||||
.BasePipe = importObject("revng.pypeline.task.pipe.Pipe"),
|
||||
};
|
||||
Registry.callAll(m, BC);
|
||||
}
|
||||
+31
-2
@@ -154,8 +154,7 @@ python_module(
|
||||
MODULE_INIT
|
||||
revng/internal/__init__.py
|
||||
MODULE_FILES
|
||||
revng/internal/py.typed
|
||||
revng/internal/pipebox.py)
|
||||
revng/internal/py.typed)
|
||||
|
||||
#
|
||||
# Install revng.model (including autogenerated classes)
|
||||
@@ -416,6 +415,7 @@ set(REVNG_PYPELINE_MODULE_FILES
|
||||
revng/pypeline/utils/registry.py
|
||||
revng/pypeline/utils/cabc.py
|
||||
revng/pypeline/utils/__init__.py)
|
||||
|
||||
python_module(
|
||||
TARGET_NAME
|
||||
revng-python-pypeline
|
||||
@@ -426,6 +426,35 @@ python_module(
|
||||
MODULE_FILES
|
||||
${REVNG_PYPELINE_MODULE_FILES})
|
||||
|
||||
#
|
||||
# Nanobind module generation and creation of revng.internal.{,_}cpp_pypeline
|
||||
#
|
||||
llvm_map_components_to_libnames(LLVM_LIBRARIES Support Core)
|
||||
|
||||
set(CPP_MODULE_NAME _pipebox)
|
||||
add_library("${CPP_MODULE_NAME}" MODULE
|
||||
"${CMAKE_SOURCE_DIR}/lib/PipeboxCommon/Python/_pipebox.cpp")
|
||||
set_target_properties(
|
||||
"${CPP_MODULE_NAME}"
|
||||
PROPERTIES LINKER_LANGUAGE CXX
|
||||
PREFIX ""
|
||||
LIBRARY_OUTPUT_DIRECTORY
|
||||
"${CMAKE_BINARY_DIR}/${PYTHON_INSTALL_PATH}/revng/internal")
|
||||
target_link_libraries(
|
||||
"${CPP_MODULE_NAME}" PUBLIC ${Python_LIBRARIES} ${LLVM_LIBRARIES} nanobind
|
||||
revngSupport revngModel)
|
||||
target_include_directories("${CPP_MODULE_NAME}" PUBLIC ${Python_INCLUDE_DIRS})
|
||||
|
||||
python_module(
|
||||
TARGET_NAME
|
||||
python-cpp-pipebox
|
||||
WHEEL
|
||||
revng_internal
|
||||
MODULE_FILES
|
||||
"revng/internal/pipebox.py"
|
||||
MODULE_GENERATED_FILES
|
||||
"revng/internal/${CPP_MODULE_NAME}.so")
|
||||
|
||||
#
|
||||
# Generate actual wheel rules
|
||||
#
|
||||
|
||||
@@ -42,6 +42,7 @@ build-backend = "setuptools.build_meta"
|
||||
[tool.setuptools.package-data]
|
||||
revng = [
|
||||
"internal/py.typed",
|
||||
"internal/_pipebox.so",
|
||||
"internal/daemon/schema.graphql",
|
||||
"internal/cli/_commands/generate_migrations/metaschema.yml",
|
||||
"internal/cli/_commands/generate_migrations/migration.py.tpl",
|
||||
|
||||
@@ -3,9 +3,11 @@
|
||||
#
|
||||
|
||||
import os
|
||||
import sys
|
||||
from collections.abc import Iterable as CIterable
|
||||
from ctypes import CDLL
|
||||
from pathlib import Path
|
||||
from typing import Iterable, TypeVar
|
||||
from typing import Any, Iterable, TypeVar
|
||||
|
||||
from xdg import xdg_cache_home
|
||||
|
||||
@@ -25,3 +27,27 @@ def cache_directory() -> Path:
|
||||
return Path(os.environ["REVNG_CACHE_DIR"])
|
||||
else:
|
||||
return xdg_cache_home() / "revng"
|
||||
|
||||
|
||||
def import_pipebox(libraries: Iterable[str | Path]) -> tuple[Any, list[Any]]:
|
||||
"""Load libraries and return the pipebox module"""
|
||||
|
||||
# Load the provided shared libraries, these contain static initializers
|
||||
# like RegisterPipe<T> which allows registering Pipes, Analyses and
|
||||
# Containers to be later created as python classes when the `_pipeline`
|
||||
# module is imported.
|
||||
#
|
||||
# RTLD_GLOBAL is needed due to nanobind using weak symbols to derive the TypeID
|
||||
handles = [CDLL(path, os.RTLD_NOW | os.RTLD_GLOBAL) for path in libraries]
|
||||
|
||||
# Check that the module hasn't been previously imported, otherwise result
|
||||
# might be unpredictable
|
||||
assert "revng.internal._pipebox" not in sys.modules
|
||||
|
||||
# Import the actual module, this will do the following:
|
||||
# 1. Create actual classes for Model, ObjectID and Kind
|
||||
# 2. Create, on the fly, classes that have been previously registered by
|
||||
# the static initializers in the imported libraries above
|
||||
import revng.internal._pipebox as ext
|
||||
|
||||
return (ext, handles)
|
||||
|
||||
@@ -21,8 +21,6 @@ class Analysis(ABC):
|
||||
objects in save points.
|
||||
"""
|
||||
|
||||
__slots__: tuple = ("name",)
|
||||
|
||||
def __init__(self, name: str):
|
||||
"""
|
||||
Initialize the analysis with a name.
|
||||
|
||||
@@ -7,7 +7,7 @@ from __future__ import annotations
|
||||
import json
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass
|
||||
from typing import Annotated, Dict, Generator, Optional, Tuple, Type
|
||||
from typing import Annotated, Dict, Generator, Tuple, Type
|
||||
|
||||
from .object import Kind, ObjectID, ObjectSet
|
||||
from .utils.cabc import ABC, abstractmethod
|
||||
@@ -109,11 +109,9 @@ class Container(ABC):
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def serialize(self, objects: Optional[ObjectSet] = None) -> Mapping[ObjectID, bytes]:
|
||||
def serialize(self, objects: ObjectSet) -> Mapping[ObjectID, bytes]:
|
||||
"""
|
||||
Dump objects from this container into a serialized format.
|
||||
If objects is provided, only those objects will be dumped.
|
||||
If not, all objects in the container will be dumped.
|
||||
"""
|
||||
|
||||
@classmethod
|
||||
@@ -182,7 +180,7 @@ def dump_container(
|
||||
"""
|
||||
with open(path, "w", encoding="utf-8") as f:
|
||||
json.dump(
|
||||
{k.serialize(): v.hex() for k, v in container.serialize().items()},
|
||||
{k.serialize(): v.hex() for k, v in container.serialize(container.objects()).items()},
|
||||
f,
|
||||
indent=4,
|
||||
sort_keys=True,
|
||||
|
||||
@@ -37,8 +37,6 @@ class Pipe(ABC):
|
||||
argument should not be added.
|
||||
"""
|
||||
|
||||
__slots__: tuple = ("name", "static_configuration")
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
static_configuration: str = "",
|
||||
|
||||
Executable
+41
@@ -0,0 +1,41 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
#
|
||||
# This file is distributed under the MIT License. See LICENSE.md for details.
|
||||
#
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
|
||||
from nanobind.stubgen import StubGen
|
||||
|
||||
from revng.internal.support import import_pipebox
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("-o", "--output", help="Output file (stdout if omitted)")
|
||||
parser.add_argument(
|
||||
"-l",
|
||||
"--library",
|
||||
default=[],
|
||||
action="append",
|
||||
help="Shared library to preload before generating stubs",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
module, handles = import_pipebox(args.library)
|
||||
stub_gen = StubGen(module, quiet=False)
|
||||
stub_gen.put(module)
|
||||
stub_text = stub_gen.get()
|
||||
|
||||
if args.output is None:
|
||||
sys.stdout.write(stub_text)
|
||||
sys.stdout.flush()
|
||||
else:
|
||||
with open(args.output, "w") as f:
|
||||
f.write(stub_text)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -81,7 +81,7 @@
|
||||
{ Regex: '^"clang/', Priority: -3 },
|
||||
{ Regex: '^"mlir/', Priority: -4 },
|
||||
{ Regex: '^"llvm/', Priority: -5 },
|
||||
{ Regex: '^"aws/', Priority: -6 },
|
||||
{ Regex: '^"nanobind/', Priority: -6 },
|
||||
{ Regex: '^"boost/', Priority: -7 },
|
||||
{ Regex: "^<", Priority: -8 },
|
||||
],
|
||||
|
||||
@@ -19,3 +19,41 @@ set_tests_properties(
|
||||
test_pypeline
|
||||
PROPERTIES LABELS "pypeline" ENVIRONMENT
|
||||
"PYTHONPATH=${CMAKE_BINARY_DIR}/${PYTHON_INSTALL_PATH}")
|
||||
|
||||
#
|
||||
# Generate libmockPipeboxImpl and libmockPipebox
|
||||
#
|
||||
add_library(mockPipeboxImpl SHARED MockPipeboxImpl.cpp)
|
||||
add_library(mockPipebox SHARED MockPipebox.cpp)
|
||||
|
||||
llvm_map_components_to_libnames(LLVM_LIBRARIES Support Core)
|
||||
target_link_libraries(mockPipeboxImpl ${LLVM_LIBRARIES} revngSupport revngModel)
|
||||
target_link_libraries(mockPipebox mockPipeboxImpl ${Python_LIBRARIES}
|
||||
${LLVM_LIBRARIES} nanobind revngSupport)
|
||||
|
||||
target_include_directories(mockPipebox PUBLIC ${Python_INCLUDE_DIRS})
|
||||
|
||||
#
|
||||
# Add tests that use libmockPipebox
|
||||
#
|
||||
revng_add_test(
|
||||
NAME pypeline-cpp-test COMMAND
|
||||
"${CMAKE_CURRENT_SOURCE_DIR}/python_cpp_test.py"
|
||||
"${CMAKE_CURRENT_BINARY_DIR}/libmockPipebox.so")
|
||||
set_tests_properties(
|
||||
pypeline-cpp-test
|
||||
PROPERTIES LABELS "pypeline;test" ENVIRONMENT
|
||||
"PYTHONPATH=${CMAKE_BINARY_DIR}/${PYTHON_INSTALL_PATH}")
|
||||
|
||||
revng_add_test(
|
||||
NAME
|
||||
pypeline-annotations-test
|
||||
COMMAND
|
||||
"${CMAKE_CURRENT_SOURCE_DIR}/nanobind_test_annotations.sh"
|
||||
"${CMAKE_SOURCE_DIR}"
|
||||
-l
|
||||
"${CMAKE_CURRENT_BINARY_DIR}/libmockPipebox.so")
|
||||
set_tests_properties(
|
||||
pypeline-annotations-test
|
||||
PROPERTIES LABELS "pypeline;test" ENVIRONMENT
|
||||
"PYTHONPATH=${CMAKE_BINARY_DIR}/${PYTHON_INSTALL_PATH}")
|
||||
|
||||
@@ -0,0 +1,11 @@
|
||||
//
|
||||
// This file is distributed under the MIT License. See LICENSE.md for details.
|
||||
//
|
||||
|
||||
#include "revng/PipeboxCommon/Helpers/Registrars.h"
|
||||
|
||||
#include "MockPipeboxImpl.h"
|
||||
|
||||
static RegisterContainer<StringContainer> X;
|
||||
static RegisterPipe<AppendFooPipe> Y;
|
||||
static RegisterAnalysis<AppendFooLibAnalysis> Z;
|
||||
@@ -0,0 +1,86 @@
|
||||
//
|
||||
// This file is distributed under the MIT License. See LICENSE.md for details.
|
||||
//
|
||||
|
||||
#include <map>
|
||||
#include <set>
|
||||
#include <string>
|
||||
|
||||
#include "llvm/ADT/ArrayRef.h"
|
||||
#include "llvm/ADT/StringRef.h"
|
||||
|
||||
#include "revng/PipeboxCommon/Model.h"
|
||||
#include "revng/PipeboxCommon/ObjectID.h"
|
||||
|
||||
#include "MockPipeboxImpl.h"
|
||||
|
||||
//
|
||||
// StringContainer
|
||||
//
|
||||
|
||||
std::set<ObjectID> StringContainer::objects() const {
|
||||
return std::views::keys(Storage) | revng::to<std::set<ObjectID>>();
|
||||
}
|
||||
|
||||
void StringContainer::deserialize(const std::map<const ObjectID *,
|
||||
llvm::ArrayRef<const char>>
|
||||
Data) {
|
||||
for (auto &Entry : Data) {
|
||||
revng_assert(Entry.first->kind() == Kind);
|
||||
std::string EntryData(Entry.second.data(), Entry.second.size());
|
||||
Storage[*Entry.first] = std::move(EntryData);
|
||||
}
|
||||
}
|
||||
|
||||
std::map<ObjectID, revng::pypeline::Buffer>
|
||||
StringContainer::serialize(const std::vector<const ObjectID *> Objects) const {
|
||||
std::map<ObjectID, revng::pypeline::Buffer> Result;
|
||||
for (auto &Element : Objects) {
|
||||
const std::string &StoredElement = Storage.at(*Element);
|
||||
llvm::ArrayRef<char> Ref = { StoredElement.data(), StoredElement.size() };
|
||||
Result[*Element] = llvm::SmallVector<char, 0>{ Ref };
|
||||
}
|
||||
return Result;
|
||||
}
|
||||
|
||||
bool StringContainer::verify() const {
|
||||
return true;
|
||||
}
|
||||
|
||||
//
|
||||
// AppendFooPipe
|
||||
//
|
||||
|
||||
revng::pypeline::ObjectDependencies
|
||||
AppendFooPipe::run(const Model &TheModel,
|
||||
const revng::pypeline::Request &Incoming,
|
||||
const revng::pypeline::Request &Outgoing,
|
||||
llvm::StringRef Configuration,
|
||||
StringContainer &Container) {
|
||||
revng_assert(Outgoing.size() == 1);
|
||||
|
||||
auto &Storage = Container.getStorage();
|
||||
for (auto &Elem : Outgoing[0]) {
|
||||
if (Storage.contains(*Elem)) {
|
||||
Storage[*Elem] = Storage[*Elem] + "foo";
|
||||
} else {
|
||||
Storage[*Elem] = "foo";
|
||||
}
|
||||
}
|
||||
|
||||
return {};
|
||||
}
|
||||
|
||||
//
|
||||
// AppendFooLibAnalysis
|
||||
//
|
||||
|
||||
llvm::Error AppendFooLibAnalysis::run(Model &TheModel,
|
||||
const revng::pypeline::Request &Incoming,
|
||||
llvm::StringRef Configuration,
|
||||
StringContainer &Container) {
|
||||
revng_assert(Incoming.size() == 1);
|
||||
TheModel.get()->ImportedLibraries().insert("foo.so");
|
||||
|
||||
return llvm::Error::success();
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
#pragma once
|
||||
|
||||
//
|
||||
// This file is distributed under the MIT License. See LICENSE.md for details.
|
||||
//
|
||||
|
||||
#include "revng/PipeboxCommon/Common.h"
|
||||
#include "revng/PipeboxCommon/Model.h"
|
||||
#include "revng/PipeboxCommon/ObjectID.h"
|
||||
|
||||
class StringContainer {
|
||||
private:
|
||||
std::map<ObjectID, std::string> Storage;
|
||||
|
||||
public:
|
||||
static constexpr llvm::StringRef Name = "StringContainer";
|
||||
static constexpr Kind Kind = Kinds::Function;
|
||||
std::set<ObjectID> objects() const;
|
||||
|
||||
void deserialize(const std::map<const ObjectID *, llvm::ArrayRef<const char>>
|
||||
Data);
|
||||
std::map<ObjectID, revng::pypeline::Buffer>
|
||||
serialize(const std::vector<const ObjectID *> Objects) const;
|
||||
bool verify() const;
|
||||
|
||||
public:
|
||||
auto &getStorage() { return Storage; }
|
||||
};
|
||||
|
||||
class AppendFooPipe {
|
||||
public:
|
||||
static constexpr llvm::StringRef Name = "AppendFooPipe";
|
||||
static constexpr revng::pypeline::PipeArgumentDocumentation
|
||||
ArgumentsDocumentation[]{ { "Container", "" } };
|
||||
|
||||
const std::string StaticConfiguration;
|
||||
|
||||
AppendFooPipe(llvm::StringRef Config) : StaticConfiguration(Config.str()) {}
|
||||
|
||||
revng::pypeline::ObjectDependencies
|
||||
run(const Model &TheModel,
|
||||
const revng::pypeline::Request &Incoming,
|
||||
const revng::pypeline::Request &Outgoing,
|
||||
llvm::StringRef Configuration,
|
||||
StringContainer &Container);
|
||||
};
|
||||
|
||||
class AppendFooLibAnalysis {
|
||||
public:
|
||||
static constexpr llvm::StringRef Name = "AppendFooLibAnalysis";
|
||||
|
||||
llvm::Error run(Model &TheModel,
|
||||
const revng::pypeline::Request &Incoming,
|
||||
llvm::StringRef Configuration,
|
||||
StringContainer &Container);
|
||||
};
|
||||
Executable
+71
@@ -0,0 +1,71 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
#
|
||||
# This file is distributed under the MIT License. See LICENSE.md for details.
|
||||
#
|
||||
|
||||
# This test checks the stubs emitted by nanobind for missing casters. When a
|
||||
# function returns a C++ type without a caster, the return type will be a
|
||||
# string (a ast.Constant). For example, if one forgets to include
|
||||
# `nanobind/stl/vector.h` then a function returning a
|
||||
# `std::vector<std::string>` will show up in stubs as:
|
||||
# ```
|
||||
# def foo() -> "std::vector<std::string>": ... # noqa: E800
|
||||
# ```
|
||||
# This program reads all the function definitions and
|
||||
# checks that the annotations of each one do not contain a string.
|
||||
|
||||
import ast
|
||||
import sys
|
||||
|
||||
|
||||
class NodeVisitor(ast.NodeVisitor):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.bad_annotations = set()
|
||||
|
||||
def visit_AnnAssign(self, node: ast.AnnAssign): # noqa: N802
|
||||
self._check_constant(node.annotation)
|
||||
|
||||
def visit_FunctionDef(self, node: ast.FunctionDef): # noqa: N802
|
||||
for arg in self._collect_arguments(node.args):
|
||||
self._check_constant(arg.annotation)
|
||||
self._check_constant(node.returns)
|
||||
|
||||
@staticmethod
|
||||
def _collect_arguments(arguments: ast.arguments) -> list[ast.arg]:
|
||||
result: list[ast.arg] = []
|
||||
result.extend(arguments.posonlyargs)
|
||||
result.extend(arguments.args)
|
||||
if arguments.vararg is not None:
|
||||
result.append(arguments.vararg)
|
||||
result.extend(arguments.kwonlyargs)
|
||||
if arguments.kwarg is not None:
|
||||
result.append(arguments.kwarg)
|
||||
return result
|
||||
|
||||
def _check_constant(self, expr: ast.expr | None):
|
||||
if not (expr is not None and isinstance(expr, ast.Constant)):
|
||||
return
|
||||
if isinstance(expr.value, str):
|
||||
# TODO: could be a forward reference, but for now it's most likely
|
||||
# a sign that a caster is missing. If needed in the future
|
||||
# it'd be easy to have a `visit_ClassDef` method that removes
|
||||
# entries if they are present
|
||||
self.bad_annotations.add(expr.value)
|
||||
|
||||
|
||||
def main():
|
||||
parsed_input = ast.parse(sys.stdin.read())
|
||||
node_visitor = NodeVisitor()
|
||||
node_visitor.visit(parsed_input)
|
||||
if len(node_visitor.bad_annotations) == 0:
|
||||
return 0
|
||||
|
||||
for entry in node_visitor.bad_annotations:
|
||||
print(f"Found string annotation for {entry}, probably a caster is missing!")
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Executable
+13
@@ -0,0 +1,13 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
#
|
||||
# This file is distributed under the MIT License. See LICENSE.md for details.
|
||||
#
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR=$(realpath "$(dirname "${BASH_SOURCE[0]}")")
|
||||
ROOT_DIR="$1"
|
||||
shift
|
||||
|
||||
"$ROOT_DIR/scripts/nanobind_generate_stubs.py" "$@" | \
|
||||
"$SCRIPT_DIR/nanobind_test_annotations.py"
|
||||
Executable
+216
@@ -0,0 +1,216 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
#
|
||||
# This file is distributed under the MIT License. See LICENSE.md for details.
|
||||
#
|
||||
|
||||
import sys
|
||||
from collections.abc import Buffer
|
||||
|
||||
import yaml
|
||||
|
||||
from revng.internal.support import import_pipebox
|
||||
from revng.pypeline import initialize_pypeline
|
||||
from revng.pypeline.analysis import Analysis, AnalysisBinding
|
||||
from revng.pypeline.container import Container, ContainerDeclaration
|
||||
from revng.pypeline.model import Model, ReadOnlyModel
|
||||
from revng.pypeline.object import Kind, ObjectID, ObjectSet
|
||||
from revng.pypeline.pipeline import Pipeline
|
||||
from revng.pypeline.pipeline_node import PipelineConfiguration, PipelineNode
|
||||
from revng.pypeline.storage.memory import InMemoryStorageProvider
|
||||
from revng.pypeline.storage.storage_provider import ContainerLocation
|
||||
from revng.pypeline.task.pipe import Pipe
|
||||
from revng.pypeline.task.requests import Requests
|
||||
from revng.pypeline.task.savepoint import SavePoint
|
||||
from revng.pypeline.task.task import TaskArgument, TaskArgumentAccess
|
||||
from revng.pypeline.utils.registry import get_registry, get_singleton
|
||||
|
||||
|
||||
def check_names(ext):
|
||||
"""Check that _pipebox has all the classes we expect it to have"""
|
||||
|
||||
# Names that we know are always present in `_pipebox`
|
||||
known_names = ("Buffer",)
|
||||
|
||||
names = [
|
||||
x for x in dir(ext) if not ((x.startswith("__") and x.endswith("__")) or x in known_names)
|
||||
]
|
||||
|
||||
# Check that the classes in the registry all came from the ext module
|
||||
for type_ in (Analysis, Container, Kind, Model, ObjectID, Pipe):
|
||||
registry = get_registry(type_)
|
||||
for key, value in registry.items():
|
||||
assert issubclass(value, type_)
|
||||
assert key in names
|
||||
assert getattr(ext, key) is value
|
||||
names.remove(key)
|
||||
|
||||
# Check that all the classes have been registered, the names variable
|
||||
# should be empty at this point
|
||||
assert len(names) == 0
|
||||
|
||||
|
||||
def compare_dicts(dict1: dict[ObjectID, Buffer], dict2: dict[ObjectID, Buffer]):
|
||||
assert all(isinstance(k, ObjectID) and isinstance(v, Buffer) for k, v in dict1.items())
|
||||
assert all(isinstance(k, ObjectID) and isinstance(v, Buffer) for k, v in dict2.items())
|
||||
assert dict1.keys() == dict2.keys()
|
||||
for key in dict1.keys():
|
||||
assert memoryview(dict1[key]) == memoryview(dict2[key])
|
||||
|
||||
|
||||
def check_pipeline():
|
||||
"""Check that all the components of fakePipebox behave as expected. This
|
||||
function **should not** use the `_pipebox` module directly, rather
|
||||
rely on the `get_registy` functionality to retrieve the classes"""
|
||||
|
||||
# Create a new model
|
||||
model = get_singleton(Model)()
|
||||
assert isinstance(model, Model)
|
||||
|
||||
# Check that the model is cloneable
|
||||
model2 = model.clone()
|
||||
assert isinstance(model2, Model)
|
||||
|
||||
objectid_cls = get_singleton(ObjectID)
|
||||
# Create a couple of ObjectIDs
|
||||
foo = objectid_cls.deserialize("/function/0x400000:Code_x86_64")
|
||||
bar = objectid_cls.deserialize("/function/0x400020:Code_x86_64")
|
||||
assert isinstance(foo, ObjectID)
|
||||
assert isinstance(bar, ObjectID)
|
||||
# Check equality works
|
||||
assert foo == objectid_cls.deserialize("/function/0x400000:Code_x86_64")
|
||||
|
||||
# Create a StringContainer and deserialize some data to it
|
||||
StringContainer = get_registry(Container)["StringContainer"] # noqa: N806
|
||||
container = StringContainer()
|
||||
container.deserialize({foo: b"foo", bar: b"bar"})
|
||||
container_serialized = container.serialize(ObjectSet.from_list([foo]))
|
||||
compare_dicts(container_serialized, {foo: b"foo"})
|
||||
# Re-feed the serialized data back to the container, this is to make sure
|
||||
# that the `Buffer` C++ class is also handled by `deserialize`
|
||||
container.deserialize(container_serialized)
|
||||
|
||||
# Run the AppendFooPipe
|
||||
AppendFooPipe = get_registry(Pipe)["AppendFooPipe"] # noqa: N806
|
||||
assert AppendFooPipe.signature() == (
|
||||
TaskArgument("Container", StringContainer, TaskArgumentAccess.READ_WRITE, ""),
|
||||
)
|
||||
pipe = AppendFooPipe("{}")
|
||||
pipe.run(
|
||||
model=ReadOnlyModel(model),
|
||||
containers=[container],
|
||||
incoming=[ObjectSet(foo.kind())],
|
||||
outgoing=[ObjectSet.from_list([foo])],
|
||||
configuration="",
|
||||
)
|
||||
container_serialized2 = container.serialize(ObjectSet.from_list([foo, bar]))
|
||||
compare_dicts(container_serialized2, {foo: b"foofoo", bar: b"bar"})
|
||||
|
||||
# Run the AppendFooLibAnalysis
|
||||
AppendFooLibAnalysis = get_registry(Analysis)["AppendFooLibAnalysis"] # noqa: N806
|
||||
assert AppendFooLibAnalysis.signature() == (StringContainer,)
|
||||
analysis = AppendFooLibAnalysis()
|
||||
analysis.run(
|
||||
model=model, containers=[container], incoming=[ObjectSet.from_list([foo])], configuration=""
|
||||
)
|
||||
|
||||
model_serialized = model.serialize()
|
||||
assert isinstance(model_serialized, Buffer)
|
||||
yaml_model = yaml.safe_load(bytes(model_serialized))
|
||||
assert "foo.so" in yaml_model["ImportedLibraries"]
|
||||
|
||||
|
||||
def check_simple_pipeline():
|
||||
StringContainer = get_registry(Container)["StringContainer"] # noqa: N806
|
||||
AppendFooPipe = get_registry(Pipe)["AppendFooPipe"] # noqa: N806
|
||||
AppendFooLibAnalysis = get_registry(Analysis)["AppendFooLibAnalysis"] # noqa: N806
|
||||
|
||||
model = get_singleton(Model)()
|
||||
storage_provider = InMemoryStorageProvider()
|
||||
child_cont = ContainerDeclaration("Container", StringContainer)
|
||||
declarations = [child_cont]
|
||||
pipeline_configuration: PipelineConfiguration = {}
|
||||
storage_provider.set_model(model.serialize())
|
||||
|
||||
# Create the pipeline
|
||||
begin_node = PipelineNode(SavePoint("begin", to_save=declarations))
|
||||
inplace_node = PipelineNode(AppendFooPipe("{}"), bindings=[child_cont])
|
||||
end_node = PipelineNode(SavePoint("end", to_save=declarations))
|
||||
begin_node.add_successor(inplace_node)
|
||||
inplace_node.add_successor(end_node)
|
||||
pipeline = Pipeline(
|
||||
set(declarations),
|
||||
begin_node,
|
||||
analyses={AnalysisBinding(AppendFooLibAnalysis(), (child_cont,), begin_node)},
|
||||
)
|
||||
|
||||
# Force the savepoint to store the objects
|
||||
container = StringContainer()
|
||||
|
||||
objectid_cls = get_singleton(ObjectID)
|
||||
foo = objectid_cls.deserialize("/function/0x400000:Code_x86_64")
|
||||
bar = objectid_cls.deserialize("/function/0x400020:Code_x86_64")
|
||||
container.deserialize({foo: b"foo", bar: b"bar"})
|
||||
foo_bar_request = Requests({child_cont: ObjectSet.from_list([foo, bar])})
|
||||
|
||||
# Pre-populate the storage
|
||||
begin_node.run(
|
||||
model=ReadOnlyModel(model),
|
||||
containers={child_cont: container},
|
||||
incoming=foo_bar_request,
|
||||
outgoing=foo_bar_request,
|
||||
pipeline_configuration=pipeline_configuration,
|
||||
storage_provider=storage_provider,
|
||||
)
|
||||
|
||||
# Check
|
||||
begin_node_config = begin_node.configuration_id(pipeline_configuration)
|
||||
container_location_begin = ContainerLocation(1, "Container", begin_node_config)
|
||||
compare_dicts(storage_provider.storage[container_location_begin], {foo: b"foo", bar: b"bar"})
|
||||
|
||||
# Run a schedule
|
||||
schedule = pipeline.schedule(
|
||||
model=ReadOnlyModel(model),
|
||||
target_node=end_node,
|
||||
requests=Requests({child_cont: ObjectSet(foo.kind(), {foo})}),
|
||||
pipeline_configuration=pipeline_configuration,
|
||||
storage_provider=storage_provider,
|
||||
)
|
||||
schedule.run(
|
||||
model=ReadOnlyModel(model),
|
||||
storage_provider=storage_provider,
|
||||
)
|
||||
|
||||
# Check that schedule.serialize works
|
||||
schedule.serialize()
|
||||
|
||||
# Check
|
||||
end_node_config = end_node.configuration_id(pipeline_configuration)
|
||||
container_location_end = ContainerLocation(2, "Container", end_node_config)
|
||||
compare_dicts(storage_provider.storage[container_location_end], {foo: b"foofoo"})
|
||||
|
||||
# Run analysis
|
||||
new_model = pipeline.run_analysis(
|
||||
model=ReadOnlyModel(model),
|
||||
analysis_name="AppendFooLibAnalysis",
|
||||
requests=Requests({child_cont: ObjectSet(foo.kind())}),
|
||||
analysis_configuration="",
|
||||
pipeline_configuration=pipeline_configuration,
|
||||
storage_provider=storage_provider,
|
||||
)
|
||||
|
||||
# Check
|
||||
yaml_model = yaml.safe_load(bytes(new_model.serialize()))
|
||||
assert "foo.so" in yaml_model["ImportedLibraries"]
|
||||
|
||||
|
||||
def main():
|
||||
ext, _ = import_pipebox(sys.argv[1:])
|
||||
initialize_pypeline()
|
||||
check_names(ext)
|
||||
check_pipeline()
|
||||
check_simple_pipeline()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user