# # This file is distributed under the MIT License. See LICENSE.md for details. # import os import re import shutil from abc import ABC, abstractmethod from collections import defaultdict from copy import deepcopy from itertools import chain from subprocess import CalledProcessError, run from tempfile import TemporaryDirectory from typing import Dict, Iterable, List, Mapping, Optional, Sequence, Set, TypeAlias, TypeVar from typing import Union from revng.model import Binary, iterate_fields # type: ignore[attr-defined] from revng.model.mixins import AllMixin from revng.pipeline_description import Container, PipelineDescription # type: ignore[attr-defined] from revng.support.artifacts import Artifact from revng.tupletree import StructBase StructBaseT = TypeVar("StructBaseT", bound=StructBase) ArtifactResult: TypeAlias = Union[Artifact, Mapping[str, Artifact]] class Project(ABC): """ This class is the basis for building a project. It defines abstract methods that the inherited classes have to implement and it also implements some common methods. """ def __init__(self): self.model: Binary = Binary() self._last_saved_model: Binary = Binary() self._pipeline_description: PipelineDescription = self._get_pipeline_description() self._rank_matchers: Dict[str, re.Pattern] = self._compute_rank_matchers() self._rank_targets: Dict[str, List[str]] = defaultdict(list) self._analysis_names = list( chain.from_iterable( [an.Name for an in st.Analyses] for st in self._pipeline_description.Steps ) ) self._analyses_list_names = [x.Name for x in self._pipeline_description.AnalysesLists] @abstractmethod def set_binary_path(self, binary_path: str): """ Use this method to import the binary into the project. This method is a prerequisite before running any analyses or getting artifacts. """ pass def list_targets(self, artifact_name: str) -> List[str]: """Retrieve the list of targets given the artifact""" kind_name = self._pipeline_description.Steps[artifact_name].Artifacts.Kind rank_name = self._pipeline_description.Kinds[kind_name].Rank return self._rank_targets[rank_name] def _get_artifact( self, artifact_name: str, targets: Optional[Sequence[Union[str, StructBaseT]]] = None ) -> ArtifactResult: """ Use the method to fetch one or more targets from an artifact. `targets` is a list of targets to produce, each element can be either a string which is interpreted as a target string or a model object, which will be converted to the corresponding target string. If the `targets` parameter is omitted then all targets of the specified artifacts are fetched. This method automatically performs MIME conversion and dispatches the work to the `_get_artifact`. The return value can either be a single Artifact object or a dictionary of them. """ all_targets = set(self.list_targets(artifact_name)) if targets is None or ( len(targets) == 1 and isinstance(targets[0], StructBase) and isinstance(targets[0], AllMixin) and targets[0]._keys == [] ): actual_targets = all_targets else: actual_targets = set() for target in targets: if isinstance(target, str): actual_targets.add(target) elif isinstance(target, StructBase) and isinstance(target, AllMixin): actual_targets.add("/".join(target._keys)) else: raise ValueError(f"Unknown parameter: {target}") assert len(actual_targets) > 0 assert actual_targets.issubset(all_targets) results = self._get_artifact_impl(artifact_name, actual_targets) container_mime = self._get_artifact_container(artifact_name).MIMEType kind_name = self._pipeline_description.Steps[artifact_name].Artifacts.Kind rank_name = self._pipeline_description.Kinds[kind_name].Rank rank = self._pipeline_description.Ranks[rank_name] if isinstance(results, dict): if rank.Depth == 0: keys = list(results.keys()) assert len(keys) == 1 return Artifact.make(results[keys[0]], container_mime) else: container_mime = self._mapped_artifact_mime(artifact_name) return {k: Artifact.make(v, container_mime) for k, v in results.items()} else: return Artifact.make(results, container_mime) @abstractmethod def _get_artifact_impl( self, artifact_name: str, targets: Set[str] ) -> Union[bytes, Dict[str, bytes]]: """ This method implements the actual fetching of the artifact(s). It is implemented by the children of this class. """ pass def _mapped_artifact_mime(self, artifact_name: str) -> str: """ Return the artifact's MIME if _get_artifact returns a mapping. By default throws an exception. Can be overridden by child classes. """ raise ValueError def _get_artifacts( self, params: Dict[str, Optional[Sequence[Union[str, StructBase]]]] ) -> Dict[str, ArtifactResult]: """ Allows fetching multiple artifacts at once. The `params` is a dict containing the name of the artifact and a list of targets (it can be empty to fetch all the targets). Example `params`: params = { "disassemble": [Function_1, Function_2], "decompile": [] } """ result: Dict[str, ArtifactResult] = {} for artifact_name, targets in params.items(): result[artifact_name] = self._get_artifact(artifact_name, targets) return result @abstractmethod def _analyze( self, analysis_name: str, targets: Dict[str, List[str]] = {}, options: Dict[str, str] = {} ): """ Run a single analysis. In addition to the `analysis_name` you need to specify a dict of targets, some analysis require you to also pass an `options` dict. """ pass @abstractmethod def _analyses_list(self, analysis_name: str): """ Run analysis list, these are predefined list of analysis that run sequentially. """ pass def get_analysis_inputs(self, analysis_name: str) -> Dict[str, List[str]]: """ Get the analysis container inputs and the associate acceptable kinds from the `pipeline description`. """ inputs: Dict[str, List[str]] = {} for step in self._pipeline_description.Steps: if analysis_name in step.Analyses: for i in step.Analyses[analysis_name].ContainerInputs: inputs[i.Name] = i.AcceptableKinds break return inputs @abstractmethod def _commit(self): """ Persist the changes from `self.model` to the backend. """ pass def _revert(self): """ Revert changes made to the `self.model` since the last call to `commit()`. """ self._set_model(deepcopy(self._last_saved_model)) @abstractmethod def _get_pipeline_description(self) -> PipelineDescription: """ Define how to implement getting the pipeline description data. This method should parse the data and return a PipelineDescription object. """ pass def import_and_analyze(self, binary_path: str): """ A helper method for setting up the project. This method imports the binary and runs the `revng` initial auto analysis. """ self.set_binary_path(binary_path) self._analyses_list("revng-initial-auto-analysis") def _set_model(self, model: Binary): self.model = model self._last_saved_model = deepcopy(self.model) # visit the model, and: # * Collect the keys based on _rank_matchers # * Fill _project on all classes # * Fill _keys where needed and add it to _rank_targets new_targets = defaultdict(list) for path, obj in iterate_fields(self.model): if not isinstance(obj, AllMixin): continue obj._project = self # type: ignore for rank, pattern in self._rank_matchers.items(): match = pattern.match(path) if match is not None: keys = [k.split("::", 1)[1] if "::" in k else k for k in match.groups()] obj._keys = keys # type: ignore new_targets[rank].append("/".join(keys)) self._rank_targets = new_targets def _get_step_name(self, analysis_name: str) -> str: for step in self._pipeline_description.Steps: if analysis_name in step.Analyses: return step.Name raise RuntimeError(f"Couldn't find step for analysis: {analysis_name}") def _get_artifact_container(self, artifact_name: str) -> Container: artifact_container = self._pipeline_description.Steps[artifact_name].Artifacts.Container container = self._pipeline_description.Containers[artifact_container] if container is None: raise RuntimeError(f"Couldn't find step name {artifact_name}") return container def _compute_rank_matchers(self) -> Dict[str, re.Pattern]: result = {} for rank in self._pipeline_description.Ranks: if rank.TupleTreePath == "": continue path = rank.TupleTreePath pattern = "^" + re.sub(r"\\\$\d+", "([^/]*)", re.escape(path)) + "$" result[rank.Name] = re.compile(pattern) return result class CLIError(Exception): pass class CLIProjectMixin: """Mixin for classes that use the `revng` command-line tool""" def __init__(self, revng_executable_path: Optional[str]): """ Check if the path of the user supplied or default revng executable is present on the system """ if not revng_executable_path: _revng_executable_path = shutil.which("revng") else: _revng_executable_path = revng_executable_path assert _revng_executable_path is not None assert os.access(_revng_executable_path, os.F_OK | os.X_OK) self._revng_executable_path = _revng_executable_path def _run_revng_cli(self, cmd_args: Iterable[str], *args, **kwargs): try: return run([self._revng_executable_path, *cmd_args], *args, **kwargs, check=True) except CalledProcessError as e: raise CLIError() from e class ResumeProjectMixin: """Mixin for classes that use a local resume directory""" def __init__(self, resume_path: Optional[str]): if resume_path is None: self._tmp_dir = TemporaryDirectory() resume_path = self._tmp_dir.name if not os.path.exists(resume_path): os.makedirs(resume_path) assert os.path.isdir(resume_path) self._resume_path = resume_path