diff --git a/python/revng/model/_common/base.py b/python/revng/model/_common/base.py index 16f3c41d7..f04cc07c8 100644 --- a/python/revng/model/_common/base.py +++ b/python/revng/model/_common/base.py @@ -10,12 +10,29 @@ from pydantic.main import ModelMetaclass def get_monkey_patching_metaclass( substitutions: Dict[str, type], + register_global_yaml_helpers=False, ): """Returns a metaclass which transparently substitutes some types with others. This way pydantic uses the substitute types when creating the validators for its types, without the need to modify the autogenerated classes. """ class MonkeyPatchingMetaClass(ModelMetaclass): + class YamlLoader(yaml.Loader): + pass + + class YamlDumper(yaml.Dumper): + def increase_indent(self, flow=False, indentless=False): + """Improves indentation""" + return super(MonkeyPatchingMetaClass.YamlDumper, self).increase_indent(flow, False) + + def analyze_scalar(self, scalar): + """Ensures literals starting with ? are quoted to make LLVM YAML parser happy""" + analysis = super(MonkeyPatchingMetaClass.YamlDumper, self).analyze_scalar(scalar) + if isinstance(analysis.scalar, str) and analysis.scalar.startswith("?"): + analysis.allow_flow_plain = False + analysis.allow_block_plain = False + return analysis + @staticmethod def __new__(mcs, clsname, bases, namespace): substitution = substitutions.get(clsname) @@ -43,17 +60,25 @@ def get_monkey_patching_metaclass( {k: v for k, v in instance._iter(exclude_none=True)}, ) - yaml.add_representer( + mcs.YamlDumper.add_representer( created_class, yaml_representer, ) + if register_global_yaml_helpers: + yaml.add_representer( + created_class, + yaml_representer, + ) def yaml_constructor(loader, node): mapping = loader.construct_mapping(node, deep=True) return created_class(**mapping) tag = f"!{clsname}" - yaml.add_constructor(tag, yaml_constructor) + + mcs.YamlLoader.add_constructor(tag, yaml_constructor) + if register_global_yaml_helpers: + yaml.add_constructor(tag, yaml_constructor) return created_class @@ -62,10 +87,12 @@ def get_monkey_patching_metaclass( def get_monkey_patching_base_class( substitutions: Dict[str, type], + register_global_yaml_helpers=False, ): """Returns a base class which transparently substitutes some model types with others""" MonkeyPatchingMetaClass = get_monkey_patching_metaclass( substitutions, + register_global_yaml_helpers=register_global_yaml_helpers, ) class MonkeyPatchingBaseClass(BaseModel, metaclass=MonkeyPatchingMetaClass): diff --git a/python/revng/model/v1/__init__.py b/python/revng/model/v1/__init__.py index 006ddbcd1..898294b72 100644 --- a/python/revng/model/v1/__init__.py +++ b/python/revng/model/v1/__init__.py @@ -1,8 +1,12 @@ # # This file is distributed under the MIT License. See LICENSE.md for details. # +from typing import IO, Text, Union + +import yaml from ._generated import * +from .base import YamlLoader, YamlDumper from .metaaddress import MetaAddress, MetaAddressType from .reference import Reference from .._common.monkeypatches import ( @@ -55,3 +59,7 @@ hashable_types = [ for t, attr_name in hashable_types: make_hashable_using_attribute(t, attr_name) + + +def load_model(stream: Union[bytes, IO[bytes], Text, IO[Text]]): + return yaml.load(stream, Loader=YamlLoader) diff --git a/python/revng/model/v1/base.py b/python/revng/model/v1/base.py index ac7ef39ec..ce6fa9a65 100644 --- a/python/revng/model/v1/base.py +++ b/python/revng/model/v1/base.py @@ -12,4 +12,9 @@ _substitutions = { MonkeyPatchingBaseClass = get_monkey_patching_base_class( _substitutions, + register_global_yaml_helpers=False, ) +YamlDumper = MonkeyPatchingBaseClass.YamlDumper +YamlLoader = MonkeyPatchingBaseClass.YamlLoader +# Allows to deserialize a model YAML to a Binary object using yaml.load(stream, Loader=YamlLoader) +YamlLoader.add_path_resolver("!Binary", []) diff --git a/tests/Tests.cmake b/tests/Tests.cmake index 7e91fc05a..d1a347fde 100644 --- a/tests/Tests.cmake +++ b/tests/Tests.cmake @@ -13,3 +13,6 @@ add_subdirectory(tests/tools/pipeline) add_subdirectory(tests/abi) set(TEST_CFLAGS_${ARCH} "${TEST_CFLAGS_${ARCH}} -mthumb") + +# TODO: include() breaks CMAKE_CURRENT_SOURCE_DIR and other variables, we should stop using it +add_subdirectory(${CMAKE_SOURCE_DIR}/tests/unit/deserialize_multiple_versions) diff --git a/tests/unit/deserialize_multiple_versions/CMakeLists.txt b/tests/unit/deserialize_multiple_versions/CMakeLists.txt new file mode 100644 index 000000000..0b4bb5b46 --- /dev/null +++ b/tests/unit/deserialize_multiple_versions/CMakeLists.txt @@ -0,0 +1,56 @@ +set(TEST_DIR "${CMAKE_BINARY_DIR}/tests/python/multiversion") +set(MODULE_DIR "${CMAKE_BINARY_DIR}/tests/python/multiversion/testmodule") + +file(MAKE_DIRECTORY "${TEST_DIR}") +file(MAKE_DIRECTORY "${MODULE_DIR}/v1") +file(MAKE_DIRECTORY "${MODULE_DIR}/v2") + +set(JSONSCHEMA_V1 "${CMAKE_CURRENT_SOURCE_DIR}/v1_schema.yaml") +set(JSONSCHEMA_V2 "${CMAKE_CURRENT_SOURCE_DIR}/v2_schema.yaml") +set(GENERATED_V1 "${MODULE_DIR}/v1/_generated.py") +set(GENERATED_V2 "${MODULE_DIR}/v2/_generated.py") + +# Copy the model._common directory in the build directory +file(COPY + "${CMAKE_SOURCE_DIR}/python/revng/model/_common" + DESTINATION + "${MODULE_DIR}" +) + +# Copy the base class +file(COPY "base.py" DESTINATION "${MODULE_DIR}/v1") +file(COPY "base.py" DESTINATION "${MODULE_DIR}/v2") + +# Generate python model for v1 and v2 +add_custom_command( + OUTPUT "${GENERATED_V1}" + COMMAND "datamodel-codegen" + ARGS + --base-class .base.MonkeyPatchingBaseClass + --target-python-version 3.6 + --input "${JSONSCHEMA_V1}" + > "${GENERATED_V1}" +) + +add_custom_command( + OUTPUT "${GENERATED_V2}" + COMMAND "datamodel-codegen" + ARGS + --base-class .base.MonkeyPatchingBaseClass + --target-python-version 3.6 + --input "${JSONSCHEMA_V2}" + > "${GENERATED_V2}" +) + +add_custom_target( + test_deserialize_multiple_versions_generated_files + ALL + DEPENDS "${GENERATED_V1}" "${GENERATED_V2}" +) + +add_test( + NAME test_deserialize_multiple_versions + COMMAND env "PYTHONPATH=${TEST_DIR}" "${CMAKE_CURRENT_SOURCE_DIR}/deserialize_multiple_versions.py" + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" +) +set_tests_properties(test_deserialize_multiple_versions PROPERTIES LABELS "unit") diff --git a/tests/unit/deserialize_multiple_versions/README.md b/tests/unit/deserialize_multiple_versions/README.md new file mode 100644 index 000000000..e69de29bb diff --git a/tests/unit/deserialize_multiple_versions/base.py b/tests/unit/deserialize_multiple_versions/base.py new file mode 100644 index 000000000..5a45cd6ed --- /dev/null +++ b/tests/unit/deserialize_multiple_versions/base.py @@ -0,0 +1,10 @@ +from .._common.base import get_monkey_patching_base_class + +_substitutions = {} + +MonkeyPatchingBaseClass = get_monkey_patching_base_class( + _substitutions, + register_global_yaml_helpers=False, +) +YamlLoader = MonkeyPatchingBaseClass.YamlLoader +YamlDumper = MonkeyPatchingBaseClass.YamlDumper diff --git a/tests/unit/deserialize_multiple_versions/deserialize_multiple_versions.py b/tests/unit/deserialize_multiple_versions/deserialize_multiple_versions.py new file mode 100755 index 000000000..316d4557f --- /dev/null +++ b/tests/unit/deserialize_multiple_versions/deserialize_multiple_versions.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 + +import yaml +from testmodule.v1._generated import RootType as RootV1 +from testmodule.v2._generated import RootType as RootV2 +from testmodule.v1.base import YamlLoader as YamlLoaderV1 +from testmodule.v2.base import YamlLoader as YamlLoaderV2 + + +def assert_parsing_fails(serialized, root_type): + try: + root_type.parse_obj(serialized) + except: + return + + raise Exception("Parsing did not fail") + + +def test_deserialize_multiple_versions(): + """Tests that the custom YAML loaders can be used to deserialize multiple conflicting versions at the same time. + Also tests that deserializing using an invalid version fails. + """ + with open("v1.yaml") as f: + v1_serialized = yaml.load(f, Loader=YamlLoaderV1) + with open("v2.yaml") as f: + v2_serialized = yaml.load(f, Loader=YamlLoaderV2) + + RootV1.parse_obj(v1_serialized) + RootV2.parse_obj(v2_serialized) + assert_parsing_fails(v1_serialized, RootV2) + assert_parsing_fails(v2_serialized, RootV1) + + print("test_deserialize_multiple_versions: OK") + + +def test_tagged_deserialize_multiple_versions(): + """Tests that the custom YAML loaders can be used to deserialize multiple conflicting versions at the same time. + Differing from test_deserialize_multiple_versions, this test uses tagged documents, which means that they should be + deserialized directly as pydantic types by yaml.load, but also still be "reparsable" by using parse_obj. + Also tests that deserializing using an invalid version fails. + """ + with open("v1_tagged.yaml") as f: + v1_serialized = yaml.load(f, Loader=YamlLoaderV1) + with open("v2_tagged.yaml") as f: + v2_serialized = yaml.load(f, Loader=YamlLoaderV2) + + assert type(v1_serialized) is RootV1 + assert type(v2_serialized) is RootV2 + + RootV1.parse_obj(v1_serialized) + RootV2.parse_obj(v2_serialized) + assert_parsing_fails(v1_serialized, RootV2) + assert_parsing_fails(v2_serialized, RootV1) + + print("test_tagged_deserialize_multiple_versions: OK") + + +test_deserialize_multiple_versions() +test_tagged_deserialize_multiple_versions() diff --git a/tests/unit/deserialize_multiple_versions/v1.yaml b/tests/unit/deserialize_multiple_versions/v1.yaml new file mode 100644 index 000000000..a7ad198f8 --- /dev/null +++ b/tests/unit/deserialize_multiple_versions/v1.yaml @@ -0,0 +1 @@ +propertyA: somethingsomething diff --git a/tests/unit/deserialize_multiple_versions/v1_schema.yaml b/tests/unit/deserialize_multiple_versions/v1_schema.yaml new file mode 100644 index 000000000..c24b950ef --- /dev/null +++ b/tests/unit/deserialize_multiple_versions/v1_schema.yaml @@ -0,0 +1,9 @@ +$ref: '#/definitions/Binary' +$schema: http://json-schema.org/draft-07/schema# +definitions: + RootType: + type: object + additionalProperties: false + properties: + propertyA: + type: string diff --git a/tests/unit/deserialize_multiple_versions/v1_tagged.yaml b/tests/unit/deserialize_multiple_versions/v1_tagged.yaml new file mode 100644 index 000000000..744adc7db --- /dev/null +++ b/tests/unit/deserialize_multiple_versions/v1_tagged.yaml @@ -0,0 +1,2 @@ +!RootType +propertyA: somethingsomething diff --git a/tests/unit/deserialize_multiple_versions/v2.yaml b/tests/unit/deserialize_multiple_versions/v2.yaml new file mode 100644 index 000000000..574526f09 --- /dev/null +++ b/tests/unit/deserialize_multiple_versions/v2.yaml @@ -0,0 +1 @@ +propertyB: somethingsomething diff --git a/tests/unit/deserialize_multiple_versions/v2_schema.yaml b/tests/unit/deserialize_multiple_versions/v2_schema.yaml new file mode 100644 index 000000000..193475038 --- /dev/null +++ b/tests/unit/deserialize_multiple_versions/v2_schema.yaml @@ -0,0 +1,9 @@ +$ref: '#/definitions/Binary' +$schema: http://json-schema.org/draft-07/schema# +definitions: + RootType: + type: object + additionalProperties: false + properties: + propertyB: + type: string diff --git a/tests/unit/deserialize_multiple_versions/v2_tagged.yaml b/tests/unit/deserialize_multiple_versions/v2_tagged.yaml new file mode 100644 index 000000000..b7faabbf6 --- /dev/null +++ b/tests/unit/deserialize_multiple_versions/v2_tagged.yaml @@ -0,0 +1,2 @@ +!RootType +propertyB: somethingsomething