python-model: Generation from model jsonschema

python/CMakeLists.txt

@@ -82,6 +82,42 @@ install(
   TYPE BIN
 )
 
+# -- Generate Python classes from JSON schema
+# TODO: find a clean way to get MODEL_JSONSCHEMA_PATH from the CMake file that actually generates it
+set(MODEL_JSONSCHEMA_DIR "${CMAKE_BINARY_DIR}")
+set(MODEL_JSONSCHEMA_PATH "${MODEL_JSONSCHEMA_DIR}/jsonschema.yml")
+set(PYTHON_GENERATED_MODEL_PATH "revng/model/v1/_generated.py")
+
+add_custom_command(
+  OUTPUT "${CMAKE_BINARY_DIR}/lib/python/${PYTHON_GENERATED_MODEL_PATH}"
+  COMMAND "datamodel-codegen"
+  ARGS
+    --base-class .base.MonkeyPatchingBaseClass
+    --target-python-version 3.6
+    --input "${MODEL_JSONSCHEMA_PATH}"
+    > "${CMAKE_BINARY_DIR}/lib/python/${PYTHON_GENERATED_MODEL_PATH}"
+  DEPENDS generated-model-jsonschema
+)
+add_custom_target(python-model-generated DEPENDS "${CMAKE_BINARY_DIR}/lib/python/${PYTHON_GENERATED_MODEL_PATH}")
+add_dependencies(revng-lift python-model-generated)
+
+# -- Install revng.model (including autogenerated classes)
+set(PYTHON_MODEL_FILES
+  revng/model/__init__.py
+  revng/model/_common/__init__.py
+  revng/model/_common/base.py
+  revng/model/_common/monkeypatches.py
+  revng/model/v1/__init__.py
+  revng/model/v1/base.py
+  revng/model/v1/metaaddress.py
+  revng/model/v1/reference.py
+)
+python_module(
+  TARGET_NAME python-model
+  MODULE_FILES ${PYTHON_MODEL_FILES}
+  MODULE_GENERATED_FILES "${PYTHON_GENERATED_MODEL_PATH}"
+)
+
 # -- Install revng.merge_dynamic
 set(MERGE_DYNAMIC_MODULE_FILES
   revng/cli/merge_dynamic/__init__.py

python/requirements.txt

@@ -3,3 +3,9 @@ pyelftools
 
 # Requirements for model_dump
 PyYAML
+
+# Common dependencies
+pydantic
+
+# revng.model dependencies
+datamodel-code-generator

python/revng/model/README.md

@@ -0,0 +1,23 @@
+# rev.ng model classes
+
+By default, importing the top level package will expose the classes for the latest version of the model.
+
+Example: deserializing a model
+
+```python
+import yaml
+from revng import model as m
+
+with open("/path/to/model.yaml") as f:
+    serialized_model = yaml.load(f)
+
+model = m.Binary.parse_obj(serialized_model)
+```
+
+If you need to access a specific version of the model you can import it like so:
+
+```python
+from revng.model import v1
+
+v1.Binary.parse_obj(...)
+```

python/revng/model/__init__.py

@@ -0,0 +1,37 @@
+#
+# This file is distributed under the MIT License. See LICENSE.md for details.
+#
+
+# Automatically import the latest version
+from glob import glob
+from pathlib import Path
+
+
+def _get_most_recent_version():
+    most_recent_version = 0
+    search_pattern = str(Path(__file__).parent / "v*")
+    for dirpath in glob(search_pattern):
+        dirname = Path(dirpath).name
+        version = int(dirname[1:])
+        if version > most_recent_version:
+            most_recent_version = version
+
+    return f"v{most_recent_version}"
+
+
+_latest_version = _get_most_recent_version()
+
+# Equivalent to `from .<latest_version> import *`
+_module = __import__(
+    _latest_version,
+    globals=globals(),
+    locals=locals(),
+    fromlist=("*",),
+    level=1,  # Perform a relative import
+)
+
+if hasattr(_module, '__all__'):
+    all_names = _module.__all__
+else:
+    all_names = [name for name in dir(_module) if not name.startswith('_')]
+globals().update({name: getattr(_module, name) for name in all_names})

python/revng/model/_common/__init__.py

@@ -0,0 +1,3 @@
+#
+# This file is distributed under the MIT License. See LICENSE.md for details.
+#

python/revng/model/_common/base.py

@@ -0,0 +1,76 @@
+#
+# This file is distributed under the MIT License. See LICENSE.md for details.
+#
+from typing import Dict
+
+import yaml
+from pydantic import BaseModel
+from pydantic.main import ModelMetaclass
+
+
+def get_monkey_patching_metaclass(
+    substitutions: Dict[str, type],
+):
+    """Returns a metaclass which transparently substitutes some types with others. This way pydantic uses the substitute
+    types when creating the validators for its types, without the need to modify the autogenerated classes.
+    """
+
+    class MonkeyPatchingMetaClass(ModelMetaclass):
+        @staticmethod
+        def __new__(mcs, clsname, bases, namespace):
+            substitution = substitutions.get(clsname)
+            if substitution is not None:
+                return substitution
+
+            created_class = super(MonkeyPatchingMetaClass, mcs).__new__(
+                mcs, clsname, bases, namespace
+            )
+
+            if "__root__" in namespace:
+
+                def yaml_representer(dumper: yaml.dumper.Dumper, instance):
+                    classname = instance.__root__.__class__.__name__
+                    tag = f"!{classname}"
+                    return dumper.represent_mapping(
+                        tag,
+                        {k: v for k, v in instance.__root__._iter(exclude_none=True)},
+                    )
+
+            else:
+
+                def yaml_representer(dumper: yaml.dumper.Dumper, instance):
+                    return dumper.represent_dict(
+                        {k: v for k, v in instance._iter(exclude_none=True)},
+                    )
+
+            yaml.add_representer(
+                created_class,
+                yaml_representer,
+            )
+
+            def yaml_constructor(loader, node):
+                mapping = loader.construct_mapping(node, deep=True)
+                return created_class(**mapping)
+
+            tag = f"!{clsname}"
+            yaml.add_constructor(tag, yaml_constructor)
+
+            return created_class
+
+    return MonkeyPatchingMetaClass
+
+
+def get_monkey_patching_base_class(
+    substitutions: Dict[str, type],
+):
+    """Returns a base class which transparently substitutes some model types with others"""
+    MonkeyPatchingMetaClass = get_monkey_patching_metaclass(
+        substitutions,
+    )
+
+    class MonkeyPatchingBaseClass(BaseModel, metaclass=MonkeyPatchingMetaClass):
+        class Config:
+            # Allows enums to be converted to strings when calling dict() on a model instance
+            use_enum_values = True
+
+    return MonkeyPatchingBaseClass

python/revng/model/_common/monkeypatches.py

@@ -0,0 +1,68 @@
+#
+# This file is distributed under the MIT License. See LICENSE.md for details.
+#
+import random
+from enum import Enum
+from typing import Optional
+
+
+def autoassign_constructor_argument(base_class: type, kwarg_name, kwarg_value):
+    """Monkeypatches __init__ so that the given keyword argument is set to the given value if was not set by the caller"""
+    assert isinstance(base_class, type)
+
+    original_init = base_class.__init__
+
+    def init_with_value(self, *args, **kwargs):
+        if kwarg_name not in kwargs:
+            kwargs[kwarg_name] = kwarg_value
+
+        original_init(self, *args, **kwargs)
+
+    base_class.__init__ = init_with_value
+
+
+def autoassign_random_id(base_class: type):
+    """Monkeypatches the __init__ method so that the ID is automatically assigned"""
+    assert isinstance(base_class, type)
+
+    original_init = base_class.__init__
+
+    def init_with_random_id(self, *args, **kwargs):
+        if "ID" not in kwargs:
+            kwargs["ID"] = random.randint(2**10 + 1, 2**64 - 1)
+        original_init(self, *args, **kwargs)
+
+    base_class.__init__ = init_with_random_id
+
+
+def autoassign_primitive_id(base_class: type):
+    """Monkeypatches the __init__ method so that the ID is automatically assigned.
+    Meant for use with primitive types.
+    """
+    assert isinstance(base_class, type)
+
+    original_init = base_class.__init__
+
+    def init_with_computed_id(self, *args, PrimitiveKind: "PrimitiveTypeKind", Size: int, **kwargs):
+        if "ID" not in kwargs:
+            primitive_kind_value = enum_value_to_index(PrimitiveKind)
+            kwargs["ID"] = primitive_kind_value << 8 | Size
+        original_init(self, *args, PrimitiveKind=PrimitiveKind, Size=Size, **kwargs)
+
+    base_class.__init__ = init_with_computed_id
+
+
+def enum_value_to_index(enum_value: Enum):
+    """Converts an enum value to its index"""
+    return list(enum_value.__class__.__members__).index(enum_value.value)
+
+
+def make_hashable_using_attribute(base_type, attribute_name: Optional[str]):
+    """Implements __hash__ by returning the given attribute"""
+
+    def __hash__(self):
+        if attribute_name is None:
+            return id(self)
+        return self.__getattribute__(attribute_name)
+
+    base_type.__hash__ = __hash__

python/revng/model/v1/__init__.py

@@ -0,0 +1,57 @@
+#
+# This file is distributed under the MIT License. See LICENSE.md for details.
+#
+
+from ._generated import *
+from .metaaddress import MetaAddress, MetaAddressType
+from .reference import Reference
+from .._common.monkeypatches import (
+    autoassign_primitive_id,
+    autoassign_random_id,
+    autoassign_constructor_argument,
+    make_hashable_using_attribute,
+)
+
+# Automatically compute primitive type ID if we weren't provided one
+autoassign_primitive_id(Primitive)
+
+# Automatically assign a random ID if we weren't provided one
+types_with_random_id = [
+    CABIFunctionType,
+    RawFunctionType,
+    UnionType,
+    Struct,
+    EnumType,
+    Typedef,
+]
+
+for t in types_with_random_id:
+    autoassign_random_id(t)
+
+# Autoassign the Kind constructor argument
+types_to_kind = [
+    (Primitive, TypeKind.Primitive),
+    (EnumType, TypeKind.Enum),
+    (Typedef, TypeKind.Typedef),
+    (Struct, TypeKind.Struct),
+    (UnionType, TypeKind.Union),
+    (CABIFunctionType, TypeKind.CABIFunctionType),
+    (RawFunctionType, TypeKind.RawFunctionType),
+]
+for t, kind_val in types_to_kind:
+    autoassign_constructor_argument(t, "Kind", kind_val)
+
+# Implement __hash__ based on the types ID
+hashable_types = [
+    (CABIFunctionType, "ID"),
+    (RawFunctionType, "ID"),
+    (UnionType, "ID"),
+    (Struct, "ID"),
+    (EnumType, "ID"),
+    (Typedef, "ID"),
+    (Primitive, "ID"),
+    (Function, None),
+]
+
+for t, attr_name in hashable_types:
+    make_hashable_using_attribute(t, attr_name)

python/revng/model/v1/base.py

@@ -0,0 +1,15 @@
+#
+# This file is distributed under the MIT License. See LICENSE.md for details.
+#
+from .._common.base import get_monkey_patching_base_class
+from .metaaddress import MetaAddress
+from .reference import Reference
+
+_substitutions = {
+    "Reference": Reference,
+    "MetaAddress": MetaAddress,
+}
+
+MonkeyPatchingBaseClass = get_monkey_patching_base_class(
+    _substitutions,
+)

python/revng/model/v1/metaaddress.py

@@ -0,0 +1,175 @@
+#
+# This file is distributed under the MIT License. See LICENSE.md for details.
+#
+import re
+from enum import Enum, auto
+from typing import Optional
+
+import yaml
+from pydantic import BaseModel, Extra, Field, PrivateAttr
+
+
+class MetaAddressType(Enum):
+    Invalid = auto()
+    Generic32 = auto()
+    Generic64 = auto()
+    Code_x86 = auto()
+    Code_x86_64 = auto()
+    Code_mips = auto()
+    Code_mipsel = auto()
+    Code_arm = auto()
+    Code_arm_thumb = auto()
+    Code_aarch64 = auto()
+    Code_systemz = auto()
+
+
+class MetaAddress(BaseModel):
+    class Config:
+        extra = Extra.forbid
+
+    # Do not remove the leading underscore, otherwise pydantic will treat this as a property
+    # and crash while trying to find an appropriate validator for it
+    _yaml_regexp = re.compile(
+        # Address (can be empty for invalid MetaAddresses, ":Invalid")
+        "(?P<Address>(0x[0-9a-fA-F]+)|)"
+        # Type
+        rf""":(?P<Type>{"|".join(v.name for v in MetaAddressType)})"""
+        # Optional epoch
+        rf"""(:(?P<Epoch>\d+))?"""
+        # Optional address space
+        rf"""(:(?P<AddressSpace>\d+))?"""
+    )
+
+    __root__: str = Field(
+        ...,
+        regex=_yaml_regexp.pattern,
+    )
+
+    _Address: int = PrivateAttr()
+    _Type: MetaAddressType = PrivateAttr()
+    _Epoch: Optional[int] = PrivateAttr(default=0)
+    _AddressSpace: Optional[int] = PrivateAttr(default=0)
+
+    def __init__(self, **kwargs):
+        assert ("__root__" in kwargs) ^ ("Address" in kwargs and "Type" in kwargs), (
+            "MetaAddress can be constructed by providing it in string form using the __root__ kwarg "
+            " or by explicitly providing Address, Type, and optional Epoch and AddressSpace"
+        )
+
+        if "__root__" in kwargs:
+            kwargs = self._parse_string(kwargs["__root__"])
+
+        self._Address = kwargs["Address"]
+        self._Type = kwargs["Type"]
+        self._Epoch = kwargs.get("Epoch", 0)
+        self._AddressSpace = kwargs.get("AddressSpace", 0)
+        super(MetaAddress, self).__init__(__root__=repr(self))
+
+    @property
+    def Address(self):
+        return self._Address
+
+    @Address.setter
+    def Address(self, value):
+        self._Address = value
+        self._update_root()
+
+    @property
+    def Type(self):
+        return self._Type
+
+    @Type.setter
+    def Type(self, value):
+        self._Type = value
+        self._update_root()
+
+    @property
+    def Epoch(self):
+        return self._Epoch
+
+    @Epoch.setter
+    def Epoch(self, value):
+        self._Epoch = value
+        self._update_root()
+
+    @property
+    def AddressSpace(self):
+        return self._AddressSpace
+
+    @AddressSpace.setter
+    def AddressSpace(self, value):
+        self._AddressSpace = value
+        self._update_root()
+
+    def _update_root(self):
+        self.__root__ = repr(self)
+
+    @classmethod
+    def _parse_string(cls, s: str):
+        assert isinstance(s, str)
+
+        match = cls._yaml_regexp.match(s)
+        if match is None:
+            raise ValueError(f"Could not parse {s} as a MetaAddress")
+
+        address = match["Address"] or "0"
+        meta_address_type = match["Type"]
+        epoch = match["Epoch"] or "0"
+        address_space = match["AddressSpace"] or "0"
+
+        return {
+            "Address": int(address, base=0),
+            "Type": MetaAddressType[meta_address_type],
+            "Epoch": int(epoch, base=0),
+            "AddressSpace": int(address_space, base=0),
+        }
+
+    def is_default_epoch(self):
+        return self.Epoch == 0
+
+    def is_default_address_space(self):
+        return self.AddressSpace == 0
+
+    def is_invalid(self):
+        return self._Type == MetaAddressType.Invalid
+
+    def __eq__(self, other):
+        if not isinstance(other, MetaAddress):
+            return False
+
+        return (
+            self._Address == other._Address
+            and self._Epoch == other._Epoch
+            and self._Type == other._Type
+            and self._AddressSpace == other._AddressSpace
+        )
+
+    def __hash__(self):
+        return self._Address
+
+    def __repr__(self):
+        components = [
+            hex(self._Address),
+            self._Type.name,
+        ]
+        if not self.is_default_epoch():
+            components.append(str(self._Epoch))
+        if not self.is_default_address_space():
+            components.append(str(self._AddressSpace))
+
+        return ":".join(components)
+
+
+def metaaddr_yaml_representer(dumper: yaml.dumper.Dumper, instance: MetaAddress):
+    return dumper.represent_str(repr(instance))
+
+
+yaml.add_representer(
+    MetaAddress,
+    metaaddr_yaml_representer,
+)
+
+__all__ = [
+    "MetaAddress",
+    "MetaAddressType",
+]

python/revng/model/v1/reference.py

@@ -0,0 +1,57 @@
+#
+# This file is distributed under the MIT License. See LICENSE.md for details.
+#
+import yaml
+from pydantic import BaseModel, Extra, Field, PrivateAttr
+
+
+class Reference(BaseModel):
+    class Config:
+        extra = Extra.forbid
+
+    __root__: str = Field(
+        ...,
+    )
+    _original_ref = PrivateAttr()
+
+    def __init__(self, *, __root__):
+        # Allow constructing references directly from revng types
+        if not isinstance(__root__, str):
+            self._original_ref = __root__
+            __root__ = self.get_reference_str(__root__)
+        super().__init__(__root__=__root__)
+
+    @staticmethod
+    def create(revng_type):
+        typedef_str = Reference.get_reference_str(revng_type)
+        return Reference(__root__=typedef_str)
+
+    @staticmethod
+    def get_reference_str(revng_type):
+        # TODO: make this not-model specific
+        if hasattr(revng_type, "Kind"):
+            typename = str(revng_type.Kind)
+        else:
+            typename = type(revng_type).__name__
+        id = revng_type.ID
+        return f"/Types/{typename}-{id}"
+
+    @property
+    def id(self):
+        _, _, id = self.__root__.rpartition("-")
+        return int(id)
+
+    def __repr__(self):
+        return self.__root__
+
+
+def reference_yaml_representer(dumper: yaml.dumper.Dumper, instance: Reference):
+    return dumper.represent_str(repr(instance))
+
+
+yaml.add_representer(
+    Reference,
+    reference_yaml_representer,
+)
+
+__all__ = ["Reference"]