revng-revng/python/revng/support/artifacts.py

#
# This file is distributed under the MIT License. See LICENSE.md for details.
#

from __future__ import annotations

import re
import shutil
import subprocess
import sys
import tarfile
from collections.abc import Mapping
from contextlib import suppress
from functools import cached_property
from io import BytesIO
from pathlib import Path
from tempfile import NamedTemporaryFile, TemporaryDirectory
from typing import IO, Callable, Dict, Generator, Generic, Optional, Tuple, Type, TypeVar, Union
from typing import cast

import pycparser
import yaml
import zstandard as zstd

from revng.ptml.parser import PTMLDocument
from revng.ptml.parser import parse as ptml_parse
from revng.ptml.printer import ColorMode, ColorPrinter, PlainPrinter, PrinterBackend, ptml_print
from revng.ptml.printer import ptml_print_with_printer
from revng.support import get_llvmcpy

# Get yaml.CSafeLoader if present, fall back to yaml.SafeLoader otherwise
YAMLLoader = getattr(yaml, "CSafeLoader", yaml.SafeLoader)

T = TypeVar("T")


class Artifact:
    _CHILDREN: Dict[str, Type[Artifact]] = {}

    def __init_subclass__(cls, **kwargs):
        super().__init_subclass__(**kwargs)
        mimes = getattr(cls, "MIMES", None)
        if mimes is not None:
            Artifact._CHILDREN.update({mime: cls for mime in mimes})

    @classmethod
    def make(cls, data: bytes, mime: str) -> Artifact:
        class_ = cls._CHILDREN.get(mime, cls)
        return class_(data, mime)

    # This class and all its subclasses have the same constructor
    def __init__(self, data: bytes, mime: str):
        # If we're in a subclass check that the mime matches with the MIMES
        # filed in the class
        if type(self) is not Artifact:
            assert mime in self.__class__.MIMES  # type: ignore[attr-defined]
        self._data = data
        self._mime = mime

    # Common methods
    @property
    def raw_data(self) -> bytes:
        return self._data

    def write_to_disk(self, path: Union[str, Path]):
        with open(path, "wb") as f:
            f.write(self._data)


class PTMLArtifact(Artifact):
    MIMES = ("text/plain+ptml", "text/x.c+ptml", "text/x.asm+ptml", "text/x.hexdump+ptml")

    def parse(self) -> PTMLDocument:
        return ptml_parse(self._data)

    def print(  # noqa: A003
        self,
        output: IO[str] = sys.stdout,
        color: ColorMode = ColorMode.Autodetect,
    ):
        ptml_print(self._data, output, color)


class _TarMixin(Artifact, Mapping, Generic[T]):
    def __init__(self, data: bytes, mime: str):
        assert mime.endswith("+tar+gz")
        super().__init__(data, mime)
        self._extracted_mime = mime.removesuffix("+tar+gz")
        self._member_function = self.__class__.MEMBER_FUNCTION  # type: ignore[attr-defined]

    @cached_property
    def _keys(self) -> Dict[str, str]:
        with tarfile.open(fileobj=BytesIO(self._data)) as tar_file:
            names = tar_file.getnames()
        return {n.split(".", 1)[0]: n for n in names}

    def __getitem__(self, key: str) -> T:
        if key not in self._keys:
            raise KeyError
        archive_name = self._keys[key]
        with tarfile.open(fileobj=BytesIO(self._data)) as tar_file:
            contents = cast(IO[bytes], tar_file.extractfile(archive_name)).read()
            return self._member_function(contents, self._extracted_mime)

    def __contains__(self, key) -> bool:
        return key in self._keys

    def __len__(self) -> int:
        return len(self._keys)

    def __iter__(self):
        return iter(self._keys)

    def items(self) -> Generator[Tuple[str, T], None, None]:  # type: ignore
        with tarfile.open(fileobj=BytesIO(self._data)) as tar_file:
            for member in tar_file.getmembers():
                name = member.name.split(".", 1)[0]
                contents = cast(IO[bytes], tar_file.extractfile(member)).read()
                yield (name, self._member_function(contents, self._extracted_mime))

    def values(self) -> Generator[T, None, None]:  # type: ignore
        for _, value in self.items():
            yield value


# Filter function, if it returns 'False' the key will not be processed
Filter = Callable[[Optional[str]], bool]


class _MappedPTMLMixin:
    def parse(self) -> Dict[str, PTMLDocument]:
        assert isinstance(self, Mapping)
        return {k: v.parse() for k, v in self.items()}

    def print(  # noqa: A003
        self,
        output: IO[str] = sys.stdout,
        color: ColorMode = ColorMode.Autodetect,
        filter_: Optional[Filter] = None,
    ):
        assert isinstance(self, Mapping)
        if filter_ is None:
            filter_ = lambda x: True  # noqa: E731

        printer: PrinterBackend
        if color == ColorMode.Off:
            printer = PlainPrinter(output, indent="  ")
            key_writer = lambda key: output.write(f"{key}: |-\n  ")  # noqa: E731
        else:
            printer = ColorPrinter(output, indent="  ", color=color)
            key_color = printer.key_color()
            key_writer = lambda key: output.write(key_color(f"{key}:") + " |-\n  ")  # noqa: E731
        for key, value in self.items():
            if not filter_(key):
                continue

            key_writer(key)
            ptml_print_with_printer(value.raw_data, printer)
            output.write("\n")


class PTMLTarArtifact(_TarMixin[PTMLArtifact], _MappedPTMLMixin):
    MIMES = ("text/plain+ptml+tar+gz", "text/x.c+ptml+tar+gz", "text/x.asm+ptml+tar+gz")
    MEMBER_FUNCTION = PTMLArtifact


class TarArtifact(_TarMixin[str]):
    MIMES = ("text/x.c+tar+gz",)
    MEMBER_FUNCTION = lambda data, _: data.decode("utf-8")  # noqa; E731


class PTMLYAMLArtifact(Artifact, _MappedPTMLMixin, Mapping):
    MIMES = ("text/plain+ptml+yaml", "text/x.c+ptml+yaml", "text/x.asm+ptml+yaml")

    def __init__(self, data: bytes, mime: str):
        Artifact.__init__(self, data, mime)
        self._member_mime = mime.removesuffix("+yaml")
        self._dict: Dict[str, str] = yaml.load(data, Loader=YAMLLoader)

    def __getitem__(self, key: str) -> PTMLArtifact:
        return PTMLArtifact(self._dict[key].encode("utf-8"), self._member_mime)

    def __iter__(self):
        return iter(self._dict)

    def __len__(self):
        return len(self._dict)


class LLVMArtifact(Artifact):
    MIMES = ("application/x.llvm.bc+zstd",)

    def module(self, name: str = "module"):
        llvmcpy = get_llvmcpy()
        context = llvmcpy.get_global_context()
        buffer = llvmcpy.create_memory_buffer_with_memory_range_copy(
            self._data, len(self._data), name
        )
        try:
            return context.parse_ir(buffer)
        except llvmcpy.LLVMException as e:
            if self._data[:4] != zstd.FRAME_HEADER:
                raise e
            # If outside of revng parse_ir cannot parse bitcode, decompress it
            # manually via the zstandard library
            with zstd.ZstdDecompressor().stream_reader(BytesIO(self._data)) as stream:
                data = stream.read()
            buffer = llvmcpy.create_memory_buffer_with_memory_range_copy(data, len(data), name)
            return context.parse_ir(buffer)


class ImageArtifact(Artifact):
    MIMES = ("image/svg",)

    def show(self):
        with NamedTemporaryFile("wb", prefix="image-artifact-") as f:
            f.write(self._data)
            f.flush()
            subprocess.run(["xdg-open", f.name])


class RecompilableArchiveArtifact(Artifact):
    MIMES = ("application/x.recompilable-archive",)

    def parse(self) -> pycparser.c_ast.FileAST:
        with TemporaryDirectory() as temp_dir:
            with tarfile.open(fileobj=BytesIO(self._data), mode="r:*") as tar:
                tar.extractall(path=temp_dir)

            c_file = Path(temp_dir) / "decompiled/functions.c"
            args = ["-E", "-DDISABLE_ATTRIBUTES", "-DDISABLE_FLOAT16"]
            for executable in ("cpp", "clang-cpp", "gcc", "clang"):
                if shutil.which(executable) is not None:
                    processed_text = pycparser.preprocess_file(c_file, executable, args)
                    break
            else:
                raise ValueError("No suitable C preprocessor found")
            return pycparser.CParser().parse(processed_text, str(c_file))


def ptml_artifact_autodetect(
    input_: bytes,
) -> Union[PTMLArtifact, PTMLTarArtifact, PTMLYAMLArtifact]:
    if len(input_) == 0:
        raise ValueError("Input is empty!")

    with suppress(tarfile.ReadError):
        result = PTMLTarArtifact(input_, "text/plain+ptml+tar+gz")
        # Force listing keys to check if it's actually a tar
        list(result.keys())
        return result

    if re.match(rb"\s*<", input_) is None:
        with suppress(yaml.YAMLError):
            return PTMLYAMLArtifact(input_, "text/plain+ptml+yaml")

    return PTMLArtifact(input_, "text/plain+ptml")