From 07fe830202272bc182c8c516bf24d60de05fb63a Mon Sep 17 00:00:00 2001 From: Filippo Cremonese Date: Fri, 3 Dec 2021 16:37:39 +0100 Subject: [PATCH] python-model: Generation from model jsonschema --- lib/Model/CMakeLists.txt | 12 ++ python/CMakeLists.txt | 36 ++++ python/requirements.txt | 6 + python/revng/model/README.md | 23 +++ python/revng/model/__init__.py | 37 +++++ python/revng/model/_common/__init__.py | 3 + python/revng/model/_common/base.py | 76 +++++++++ python/revng/model/_common/monkeypatches.py | 68 ++++++++ python/revng/model/v1/__init__.py | 57 +++++++ python/revng/model/v1/base.py | 15 ++ python/revng/model/v1/metaaddress.py | 175 ++++++++++++++++++++ python/revng/model/v1/reference.py | 57 +++++++ 12 files changed, 565 insertions(+) create mode 100644 python/revng/model/README.md create mode 100644 python/revng/model/__init__.py create mode 100644 python/revng/model/_common/__init__.py create mode 100644 python/revng/model/_common/base.py create mode 100644 python/revng/model/_common/monkeypatches.py create mode 100644 python/revng/model/v1/__init__.py create mode 100644 python/revng/model/v1/base.py create mode 100644 python/revng/model/v1/metaaddress.py create mode 100644 python/revng/model/v1/reference.py diff --git a/lib/Model/CMakeLists.txt b/lib/Model/CMakeLists.txt index 5a9a17084..0a5a0bb3f 100644 --- a/lib/Model/CMakeLists.txt +++ b/lib/Model/CMakeLists.txt @@ -58,6 +58,18 @@ install( DESTINATION include/revng/Model ) +set(MODEL_JSONSCHEMA_PATH "${CMAKE_BINARY_DIR}/jsonschema.yml") +tuple_tree_generator_jsonschema_from_headers( + "${MODEL_HEADERS}" + TUPLE-TREE-YAML + model + Binary + "Identifier" + "MetaAddress" + "${MODEL_JSONSCHEMA_PATH}" +) +add_custom_target(generated-model-jsonschema DEPENDS "${MODEL_JSONSCHEMA_PATH}") + # Define revngModel library revng_add_analyses_library_internal( revngModel diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index 6a3facdf7..5c49fdfe2 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -82,6 +82,42 @@ install( TYPE BIN ) +# -- Generate Python classes from JSON schema +# TODO: find a clean way to get MODEL_JSONSCHEMA_PATH from the CMake file that actually generates it +set(MODEL_JSONSCHEMA_DIR "${CMAKE_BINARY_DIR}") +set(MODEL_JSONSCHEMA_PATH "${MODEL_JSONSCHEMA_DIR}/jsonschema.yml") +set(PYTHON_GENERATED_MODEL_PATH "revng/model/v1/_generated.py") + +add_custom_command( + OUTPUT "${CMAKE_BINARY_DIR}/lib/python/${PYTHON_GENERATED_MODEL_PATH}" + COMMAND "datamodel-codegen" + ARGS + --base-class .base.MonkeyPatchingBaseClass + --target-python-version 3.6 + --input "${MODEL_JSONSCHEMA_PATH}" + > "${CMAKE_BINARY_DIR}/lib/python/${PYTHON_GENERATED_MODEL_PATH}" + DEPENDS generated-model-jsonschema +) +add_custom_target(python-model-generated DEPENDS "${CMAKE_BINARY_DIR}/lib/python/${PYTHON_GENERATED_MODEL_PATH}") +add_dependencies(revng-lift python-model-generated) + +# -- Install revng.model (including autogenerated classes) +set(PYTHON_MODEL_FILES + revng/model/__init__.py + revng/model/_common/__init__.py + revng/model/_common/base.py + revng/model/_common/monkeypatches.py + revng/model/v1/__init__.py + revng/model/v1/base.py + revng/model/v1/metaaddress.py + revng/model/v1/reference.py +) +python_module( + TARGET_NAME python-model + MODULE_FILES ${PYTHON_MODEL_FILES} + MODULE_GENERATED_FILES "${PYTHON_GENERATED_MODEL_PATH}" +) + # -- Install revng.merge_dynamic set(MERGE_DYNAMIC_MODULE_FILES revng/cli/merge_dynamic/__init__.py diff --git a/python/requirements.txt b/python/requirements.txt index a6a3a8d0d..9ed6e2baa 100644 --- a/python/requirements.txt +++ b/python/requirements.txt @@ -3,3 +3,9 @@ pyelftools # Requirements for model_dump PyYAML + +# Common dependencies +pydantic + +# revng.model dependencies +datamodel-code-generator diff --git a/python/revng/model/README.md b/python/revng/model/README.md new file mode 100644 index 000000000..ffb1f566a --- /dev/null +++ b/python/revng/model/README.md @@ -0,0 +1,23 @@ +# rev.ng model classes + +By default, importing the top level package will expose the classes for the latest version of the model. + +Example: deserializing a model + +```python +import yaml +from revng import model as m + +with open("/path/to/model.yaml") as f: + serialized_model = yaml.load(f) + +model = m.Binary.parse_obj(serialized_model) +``` + +If you need to access a specific version of the model you can import it like so: + +```python +from revng.model import v1 + +v1.Binary.parse_obj(...) +``` diff --git a/python/revng/model/__init__.py b/python/revng/model/__init__.py new file mode 100644 index 000000000..9eb3a9173 --- /dev/null +++ b/python/revng/model/__init__.py @@ -0,0 +1,37 @@ +# +# This file is distributed under the MIT License. See LICENSE.md for details. +# + +# Automatically import the latest version +from glob import glob +from pathlib import Path + + +def _get_most_recent_version(): + most_recent_version = 0 + search_pattern = str(Path(__file__).parent / "v*") + for dirpath in glob(search_pattern): + dirname = Path(dirpath).name + version = int(dirname[1:]) + if version > most_recent_version: + most_recent_version = version + + return f"v{most_recent_version}" + + +_latest_version = _get_most_recent_version() + +# Equivalent to `from . import *` +_module = __import__( + _latest_version, + globals=globals(), + locals=locals(), + fromlist=("*",), + level=1, # Perform a relative import +) + +if hasattr(_module, '__all__'): + all_names = _module.__all__ +else: + all_names = [name for name in dir(_module) if not name.startswith('_')] +globals().update({name: getattr(_module, name) for name in all_names}) diff --git a/python/revng/model/_common/__init__.py b/python/revng/model/_common/__init__.py new file mode 100644 index 000000000..79d95cc20 --- /dev/null +++ b/python/revng/model/_common/__init__.py @@ -0,0 +1,3 @@ +# +# This file is distributed under the MIT License. See LICENSE.md for details. +# diff --git a/python/revng/model/_common/base.py b/python/revng/model/_common/base.py new file mode 100644 index 000000000..16f3c41d7 --- /dev/null +++ b/python/revng/model/_common/base.py @@ -0,0 +1,76 @@ +# +# This file is distributed under the MIT License. See LICENSE.md for details. +# +from typing import Dict + +import yaml +from pydantic import BaseModel +from pydantic.main import ModelMetaclass + + +def get_monkey_patching_metaclass( + substitutions: Dict[str, type], +): + """Returns a metaclass which transparently substitutes some types with others. This way pydantic uses the substitute + types when creating the validators for its types, without the need to modify the autogenerated classes. + """ + + class MonkeyPatchingMetaClass(ModelMetaclass): + @staticmethod + def __new__(mcs, clsname, bases, namespace): + substitution = substitutions.get(clsname) + if substitution is not None: + return substitution + + created_class = super(MonkeyPatchingMetaClass, mcs).__new__( + mcs, clsname, bases, namespace + ) + + if "__root__" in namespace: + + def yaml_representer(dumper: yaml.dumper.Dumper, instance): + classname = instance.__root__.__class__.__name__ + tag = f"!{classname}" + return dumper.represent_mapping( + tag, + {k: v for k, v in instance.__root__._iter(exclude_none=True)}, + ) + + else: + + def yaml_representer(dumper: yaml.dumper.Dumper, instance): + return dumper.represent_dict( + {k: v for k, v in instance._iter(exclude_none=True)}, + ) + + yaml.add_representer( + created_class, + yaml_representer, + ) + + def yaml_constructor(loader, node): + mapping = loader.construct_mapping(node, deep=True) + return created_class(**mapping) + + tag = f"!{clsname}" + yaml.add_constructor(tag, yaml_constructor) + + return created_class + + return MonkeyPatchingMetaClass + + +def get_monkey_patching_base_class( + substitutions: Dict[str, type], +): + """Returns a base class which transparently substitutes some model types with others""" + MonkeyPatchingMetaClass = get_monkey_patching_metaclass( + substitutions, + ) + + class MonkeyPatchingBaseClass(BaseModel, metaclass=MonkeyPatchingMetaClass): + class Config: + # Allows enums to be converted to strings when calling dict() on a model instance + use_enum_values = True + + return MonkeyPatchingBaseClass diff --git a/python/revng/model/_common/monkeypatches.py b/python/revng/model/_common/monkeypatches.py new file mode 100644 index 000000000..7f1088e80 --- /dev/null +++ b/python/revng/model/_common/monkeypatches.py @@ -0,0 +1,68 @@ +# +# This file is distributed under the MIT License. See LICENSE.md for details. +# +import random +from enum import Enum +from typing import Optional + + +def autoassign_constructor_argument(base_class: type, kwarg_name, kwarg_value): + """Monkeypatches __init__ so that the given keyword argument is set to the given value if was not set by the caller""" + assert isinstance(base_class, type) + + original_init = base_class.__init__ + + def init_with_value(self, *args, **kwargs): + if kwarg_name not in kwargs: + kwargs[kwarg_name] = kwarg_value + + original_init(self, *args, **kwargs) + + base_class.__init__ = init_with_value + + +def autoassign_random_id(base_class: type): + """Monkeypatches the __init__ method so that the ID is automatically assigned""" + assert isinstance(base_class, type) + + original_init = base_class.__init__ + + def init_with_random_id(self, *args, **kwargs): + if "ID" not in kwargs: + kwargs["ID"] = random.randint(2**10 + 1, 2**64 - 1) + original_init(self, *args, **kwargs) + + base_class.__init__ = init_with_random_id + + +def autoassign_primitive_id(base_class: type): + """Monkeypatches the __init__ method so that the ID is automatically assigned. + Meant for use with primitive types. + """ + assert isinstance(base_class, type) + + original_init = base_class.__init__ + + def init_with_computed_id(self, *args, PrimitiveKind: "PrimitiveTypeKind", Size: int, **kwargs): + if "ID" not in kwargs: + primitive_kind_value = enum_value_to_index(PrimitiveKind) + kwargs["ID"] = primitive_kind_value << 8 | Size + original_init(self, *args, PrimitiveKind=PrimitiveKind, Size=Size, **kwargs) + + base_class.__init__ = init_with_computed_id + + +def enum_value_to_index(enum_value: Enum): + """Converts an enum value to its index""" + return list(enum_value.__class__.__members__).index(enum_value.value) + + +def make_hashable_using_attribute(base_type, attribute_name: Optional[str]): + """Implements __hash__ by returning the given attribute""" + + def __hash__(self): + if attribute_name is None: + return id(self) + return self.__getattribute__(attribute_name) + + base_type.__hash__ = __hash__ diff --git a/python/revng/model/v1/__init__.py b/python/revng/model/v1/__init__.py new file mode 100644 index 000000000..006ddbcd1 --- /dev/null +++ b/python/revng/model/v1/__init__.py @@ -0,0 +1,57 @@ +# +# This file is distributed under the MIT License. See LICENSE.md for details. +# + +from ._generated import * +from .metaaddress import MetaAddress, MetaAddressType +from .reference import Reference +from .._common.monkeypatches import ( + autoassign_primitive_id, + autoassign_random_id, + autoassign_constructor_argument, + make_hashable_using_attribute, +) + +# Automatically compute primitive type ID if we weren't provided one +autoassign_primitive_id(Primitive) + +# Automatically assign a random ID if we weren't provided one +types_with_random_id = [ + CABIFunctionType, + RawFunctionType, + UnionType, + Struct, + EnumType, + Typedef, +] + +for t in types_with_random_id: + autoassign_random_id(t) + +# Autoassign the Kind constructor argument +types_to_kind = [ + (Primitive, TypeKind.Primitive), + (EnumType, TypeKind.Enum), + (Typedef, TypeKind.Typedef), + (Struct, TypeKind.Struct), + (UnionType, TypeKind.Union), + (CABIFunctionType, TypeKind.CABIFunctionType), + (RawFunctionType, TypeKind.RawFunctionType), +] +for t, kind_val in types_to_kind: + autoassign_constructor_argument(t, "Kind", kind_val) + +# Implement __hash__ based on the types ID +hashable_types = [ + (CABIFunctionType, "ID"), + (RawFunctionType, "ID"), + (UnionType, "ID"), + (Struct, "ID"), + (EnumType, "ID"), + (Typedef, "ID"), + (Primitive, "ID"), + (Function, None), +] + +for t, attr_name in hashable_types: + make_hashable_using_attribute(t, attr_name) diff --git a/python/revng/model/v1/base.py b/python/revng/model/v1/base.py new file mode 100644 index 000000000..ac7ef39ec --- /dev/null +++ b/python/revng/model/v1/base.py @@ -0,0 +1,15 @@ +# +# This file is distributed under the MIT License. See LICENSE.md for details. +# +from .._common.base import get_monkey_patching_base_class +from .metaaddress import MetaAddress +from .reference import Reference + +_substitutions = { + "Reference": Reference, + "MetaAddress": MetaAddress, +} + +MonkeyPatchingBaseClass = get_monkey_patching_base_class( + _substitutions, +) diff --git a/python/revng/model/v1/metaaddress.py b/python/revng/model/v1/metaaddress.py new file mode 100644 index 000000000..3b0571dee --- /dev/null +++ b/python/revng/model/v1/metaaddress.py @@ -0,0 +1,175 @@ +# +# This file is distributed under the MIT License. See LICENSE.md for details. +# +import re +from enum import Enum, auto +from typing import Optional + +import yaml +from pydantic import BaseModel, Extra, Field, PrivateAttr + + +class MetaAddressType(Enum): + Invalid = auto() + Generic32 = auto() + Generic64 = auto() + Code_x86 = auto() + Code_x86_64 = auto() + Code_mips = auto() + Code_mipsel = auto() + Code_arm = auto() + Code_arm_thumb = auto() + Code_aarch64 = auto() + Code_systemz = auto() + + +class MetaAddress(BaseModel): + class Config: + extra = Extra.forbid + + # Do not remove the leading underscore, otherwise pydantic will treat this as a property + # and crash while trying to find an appropriate validator for it + _yaml_regexp = re.compile( + # Address (can be empty for invalid MetaAddresses, ":Invalid") + "(?P
(0x[0-9a-fA-F]+)|)" + # Type + rf""":(?P{"|".join(v.name for v in MetaAddressType)})""" + # Optional epoch + rf"""(:(?P\d+))?""" + # Optional address space + rf"""(:(?P\d+))?""" + ) + + __root__: str = Field( + ..., + regex=_yaml_regexp.pattern, + ) + + _Address: int = PrivateAttr() + _Type: MetaAddressType = PrivateAttr() + _Epoch: Optional[int] = PrivateAttr(default=0) + _AddressSpace: Optional[int] = PrivateAttr(default=0) + + def __init__(self, **kwargs): + assert ("__root__" in kwargs) ^ ("Address" in kwargs and "Type" in kwargs), ( + "MetaAddress can be constructed by providing it in string form using the __root__ kwarg " + " or by explicitly providing Address, Type, and optional Epoch and AddressSpace" + ) + + if "__root__" in kwargs: + kwargs = self._parse_string(kwargs["__root__"]) + + self._Address = kwargs["Address"] + self._Type = kwargs["Type"] + self._Epoch = kwargs.get("Epoch", 0) + self._AddressSpace = kwargs.get("AddressSpace", 0) + super(MetaAddress, self).__init__(__root__=repr(self)) + + @property + def Address(self): + return self._Address + + @Address.setter + def Address(self, value): + self._Address = value + self._update_root() + + @property + def Type(self): + return self._Type + + @Type.setter + def Type(self, value): + self._Type = value + self._update_root() + + @property + def Epoch(self): + return self._Epoch + + @Epoch.setter + def Epoch(self, value): + self._Epoch = value + self._update_root() + + @property + def AddressSpace(self): + return self._AddressSpace + + @AddressSpace.setter + def AddressSpace(self, value): + self._AddressSpace = value + self._update_root() + + def _update_root(self): + self.__root__ = repr(self) + + @classmethod + def _parse_string(cls, s: str): + assert isinstance(s, str) + + match = cls._yaml_regexp.match(s) + if match is None: + raise ValueError(f"Could not parse {s} as a MetaAddress") + + address = match["Address"] or "0" + meta_address_type = match["Type"] + epoch = match["Epoch"] or "0" + address_space = match["AddressSpace"] or "0" + + return { + "Address": int(address, base=0), + "Type": MetaAddressType[meta_address_type], + "Epoch": int(epoch, base=0), + "AddressSpace": int(address_space, base=0), + } + + def is_default_epoch(self): + return self.Epoch == 0 + + def is_default_address_space(self): + return self.AddressSpace == 0 + + def is_invalid(self): + return self._Type == MetaAddressType.Invalid + + def __eq__(self, other): + if not isinstance(other, MetaAddress): + return False + + return ( + self._Address == other._Address + and self._Epoch == other._Epoch + and self._Type == other._Type + and self._AddressSpace == other._AddressSpace + ) + + def __hash__(self): + return self._Address + + def __repr__(self): + components = [ + hex(self._Address), + self._Type.name, + ] + if not self.is_default_epoch(): + components.append(str(self._Epoch)) + if not self.is_default_address_space(): + components.append(str(self._AddressSpace)) + + return ":".join(components) + + +def metaaddr_yaml_representer(dumper: yaml.dumper.Dumper, instance: MetaAddress): + return dumper.represent_str(repr(instance)) + + +yaml.add_representer( + MetaAddress, + metaaddr_yaml_representer, +) + +__all__ = [ + "MetaAddress", + "MetaAddressType", +] diff --git a/python/revng/model/v1/reference.py b/python/revng/model/v1/reference.py new file mode 100644 index 000000000..54f626ec5 --- /dev/null +++ b/python/revng/model/v1/reference.py @@ -0,0 +1,57 @@ +# +# This file is distributed under the MIT License. See LICENSE.md for details. +# +import yaml +from pydantic import BaseModel, Extra, Field, PrivateAttr + + +class Reference(BaseModel): + class Config: + extra = Extra.forbid + + __root__: str = Field( + ..., + ) + _original_ref = PrivateAttr() + + def __init__(self, *, __root__): + # Allow constructing references directly from revng types + if not isinstance(__root__, str): + self._original_ref = __root__ + __root__ = self.get_reference_str(__root__) + super().__init__(__root__=__root__) + + @staticmethod + def create(revng_type): + typedef_str = Reference.get_reference_str(revng_type) + return Reference(__root__=typedef_str) + + @staticmethod + def get_reference_str(revng_type): + # TODO: make this not-model specific + if hasattr(revng_type, "Kind"): + typename = str(revng_type.Kind) + else: + typename = type(revng_type).__name__ + id = revng_type.ID + return f"/Types/{typename}-{id}" + + @property + def id(self): + _, _, id = self.__root__.rpartition("-") + return int(id) + + def __repr__(self): + return self.__root__ + + +def reference_yaml_representer(dumper: yaml.dumper.Dumper, instance: Reference): + return dumper.represent_str(repr(instance)) + + +yaml.add_representer( + Reference, + reference_yaml_representer, +) + +__all__ = ["Reference"]