diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index 7adadc828..556cf4357 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -158,3 +158,18 @@ foreach(HEADER ${REQUIRED_PIPELINE_C_HEADERS}) configure_file("${PIPELINE_C_HEADERS_SOURCE_DIR}/${HEADER}" "${PIPELINE_C_HEADERS_BUILD_DIR}/${HEADER}" COPYONLY) endforeach() + +# +# Install revng.daemon +# +set(REVNG_DAEMON_MODULE_FILES + revng/daemon/__init__.py + revng/daemon/api.py + revng/daemon/demo_webpage.py + revng/daemon/schema.py + revng/daemon/schema.graphql.tpl + revng/daemon/util.py + revng/daemon/templates/base.html + revng/daemon/templates/index.html) +python_module(TARGET_NAME revng-python-daemon MODULE_FILES + ${REVNG_DAEMON_MODULE_FILES}) diff --git a/python/requirements.txt b/python/requirements.txt index c79b6d63f..ea84678e4 100644 --- a/python/requirements.txt +++ b/python/requirements.txt @@ -9,3 +9,11 @@ grandiso # revng.api cffi + +# revng.daemon +Flask +Werkzeug +ariadne +aiodataloader +Jinja2 +xdg diff --git a/python/revng/daemon/__init__.py b/python/revng/daemon/__init__.py new file mode 100644 index 000000000..92073316e --- /dev/null +++ b/python/revng/daemon/__init__.py @@ -0,0 +1,58 @@ +# +# This file is distributed under the MIT License. See LICENSE.md for details. +# + +import atexit +import logging +import secrets +from pathlib import Path +from typing import Optional + +from flask import Flask, g + +from revng.api._capi import initialize as capi_initialize + +from .api import api_blueprint +from .demo_webpage import demo_blueprint +from .schema import SchemafulManager +from .util import project_workdir + +workdir: Path = project_workdir() +manager: Optional[SchemafulManager] = None + + +app = Flask(__name__) +app.register_blueprint(api_blueprint) +app.register_blueprint(demo_blueprint) + +app.secret_key = secrets.token_hex(16) + + +def cleanup(): + if manager is not None: + store_result = manager.store_containers() + if not store_result: + logging.warning("Failed to store manager's containers") + + +atexit.register(cleanup) + + +@app.before_first_request +def init(): + global manager + capi_initialize() + manager = SchemafulManager(workdir=str(workdir.resolve())) + + +@app.before_request +def init_global_object(): + assert manager, "Manager not initialized" + + g.workdir = workdir + g.manager = manager + + # Safety checks + assert g.workdir is not None + assert g.workdir.exists() and g.workdir.is_dir() + assert g.manager is not None diff --git a/python/revng/daemon/api.py b/python/revng/daemon/api.py new file mode 100644 index 000000000..34b7c2f2a --- /dev/null +++ b/python/revng/daemon/api.py @@ -0,0 +1,74 @@ +# +# This file is distributed under the MIT License. See LICENSE.md for details. +# + +import json +from typing import TYPE_CHECKING, cast + +from flask import Blueprint, current_app, jsonify, request + +from ariadne import graphql +from ariadne.constants import PLAYGROUND_HTML +from ariadne.contrib.tracing.apollotracing import ApolloTracingExtension +from ariadne.file_uploads import FilesDict, combine_multipart_data + +from .schema import SchemafulManager + +if TYPE_CHECKING: + from flask.ctx import _AppCtxGlobals + + class FlaskGlobals(_AppCtxGlobals): + manager: SchemafulManager + + g = FlaskGlobals() +else: + from flask import g + + +api_blueprint = Blueprint("api", __name__) + + +def json_response(data=None): + if data is None: + data = {} + return jsonify(data) + + +def json_error(message, http_code=404): + data = { + "error": message, + } + return jsonify(data), http_code + + +# When navigating to http:///graphql show a graphql sandbox +@api_blueprint.route("/graphql", methods=["GET"]) +def graphql_playground(): + return PLAYGROUND_HTML, 200 + + +@api_blueprint.route("/graphql", methods=["POST"]) +async def graphql_server(): + if request.content_type.startswith("multipart/form-data;"): + operations = request.form.get("operations") + req_map = request.form.get("map") + if operations is not None and req_map is not None: + request_files = cast(FilesDict, request.files) + data = combine_multipart_data( + json.loads(operations), json.loads(req_map), request_files + ) + else: + return json_error("Invalid form data") + else: + data = request.get_json() + + success, result = await graphql( + g.manager.schema, + data, + context_value={"g": g, "request": request}, + extensions=[ApolloTracingExtension], + debug=current_app.config["DEBUG"], + ) + + status_code = 200 if success else 400 + return jsonify(result), status_code diff --git a/python/revng/daemon/demo_webpage.py b/python/revng/daemon/demo_webpage.py new file mode 100644 index 000000000..b738178cc --- /dev/null +++ b/python/revng/daemon/demo_webpage.py @@ -0,0 +1,12 @@ +# +# This file is distributed under the MIT License. See LICENSE.md for details. +# + +from flask import Blueprint, render_template + +demo_blueprint = Blueprint("demo", __name__) + + +@demo_blueprint.route("/") +def index(): + return render_template("index.html") diff --git a/python/revng/daemon/schema.graphql.tpl b/python/revng/daemon/schema.graphql.tpl new file mode 100644 index 000000000..1d234d123 --- /dev/null +++ b/python/revng/daemon/schema.graphql.tpl @@ -0,0 +1,69 @@ +{# + This file is distributed under the MIT License. See LICENSE.md for details. +#} +type Query { + info: Info! + step(name: String!): Step + container(name: String!, step: String!): Container + targets(pathspec: String!): [Step!]! + produce(step: String!, container: String!, target_list: String!, only_if_ready: Boolean): String + produce_artifacts(step: String!, paths: String, only_if_ready: Boolean): String + + {%- for rank in structure.keys() %} + {{ rank.name }}{{ rank | rank_param }}: {{ rank.name | capitalize }}! + {%- endfor %} +} + +type Mutation { + upload_b64(input: String!, container: String!): Boolean! + upload_file(file: Upload, container: String!): Boolean! +} + +type Info { + kinds: [Kind!]! + ranks: [Rank!]! + steps: [Step!]! + model: String! +} + +type Kind { + name: ID + rank: String! + parent: String +} + +type Rank { + name: ID + depth: Int! + parent: String +} + +type Step { + name: ID + parent: String + containers: [Container!]! +} + +type Container { + name: ID + mime: String + targets: [Target!]! +} + +type Target { + serialized: String + exact: Boolean + path_components: [String!]! + kind: String + ready: Boolean +} + +{% for rank, steps in structure.items() %} +type {{ rank.name | capitalize }} { +{%- for step in steps %} + {{ step.name | snake_case }}(only_if_ready: Boolean): String! +{%- endfor %} +} +{% endfor %} + +scalar Upload diff --git a/python/revng/daemon/schema.py b/python/revng/daemon/schema.py new file mode 100644 index 000000000..4e7b9923b --- /dev/null +++ b/python/revng/daemon/schema.py @@ -0,0 +1,241 @@ +# +# This file is distributed under the MIT License. See LICENSE.md for details. +# + +import logging +from base64 import b64decode +from pathlib import Path +from typing import Dict, List, Optional + +from ariadne import MutationType, ObjectType, QueryType, make_executable_schema, upload_scalar +from graphql.type.schema import GraphQLSchema +from jinja2 import Environment, FileSystemLoader + +from revng.api.manager import Manager +from revng.api.rank import Rank +from revng.api.step import Step + +from .util import clean_step_list, str_to_snake_case + +query = QueryType() +mutation = MutationType() +info = ObjectType("Info") +step = ObjectType("Step") +container = ObjectType("Container") + + +@query.field("info") +async def resolve_root(_, info): + return {} + + +@query.field("produce") +async def resolve_produce(obj, info, *, step, container, target_list, only_if_ready=False): + manager: Manager = info.context["g"].manager + targets = target_list.split(",") + return manager.produce_target(step, targets, container, only_if_ready) + + +@query.field("produce_artifacts") +async def resolve_produce_artifacts(obj, info, *, step, paths=None, only_if_ready=False): + manager: Manager = info.context["g"].manager + target_paths = paths.split(",") if paths is not None else None + return manager.produce_target(step, target_paths, only_if_ready=only_if_ready) + + +@query.field("step") +async def resolve_step(_, info, *, name): + manager: Manager = info.context["g"].manager + step = manager.get_step(name) + return step.as_dict() if step is not None else {} + + +@query.field("container") +async def resolve_container(_, info, *, name, step): + manager: Manager = info.context["g"].manager + container_id = manager.get_container_with_name(name) + step = manager.get_step(step) + if step is None or container_id is None: + return {} + container = step.get_container(container_id) + return container.as_dict() if container is not None else {} + + +@query.field("targets") +async def resolve_targets(_, info, *, pathspec): + manager: Manager = info.context["g"].manager + targets = manager.get_all_targets() + result = [ + { + "name": k, + "containers": [ + {"name": k2, "targets": [t.as_dict() for t in v2 if t.joined_path() == pathspec]} + for k2, v2 in v.items() + ], + } + for k, v in targets.items() + ] + clean_step_list(result) + return result + + +@mutation.field("upload_b64") +async def resolve_upload_b64(_, info, *, input: str, container: str): # noqa: A002 + g = info.context["g"] + g.manager.set_input(container, b64decode(input)) + logging.info(f"Saved file for container {container}") + return True + + +@mutation.field("upload_file") +async def resolve_upload_file(_, info, *, file, container: str): + g = info.context["g"] + g.manager.set_input(container, file.read()) + logging.info(f"Saved file for container {container}") + return True + + +@info.field("ranks") +async def resolve_ranks(_, info): + return [x.as_dict() for x in Rank.ranks()] + + +@info.field("kinds") +async def resolve_root_kinds(_, info): + manager = info.context["g"].manager + return [k.as_dict() for k in manager.kinds()] + + +@info.field("model") +async def resolve_root_model(_, info): + manager = info.context["g"].manager + return manager.get_model() + + +@info.field("steps") +async def resolve_root_steps(_, info): + manager: Manager = info.context["g"].manager + return [s.as_dict() for s in manager.steps()] + + +@step.field("containers") +async def resolve_step_containers(step_obj, info): + if "containers" in step_obj: + return step_obj["containers"] + + manager: Manager = info.context["g"].manager + step = manager.get_step(step_obj["name"]) + if step is None: + return [] + containers = [step.get_container(c) for c in manager.containers()] + return [c.as_dict() for c in containers if c is not None] + + +@container.field("targets") +async def resolve_container_targets(container_obj, info): + if "targets" in container_obj: + return container_obj["targets"] + + manager: Manager = info.context["g"].manager + targets = manager.get_targets(container_obj["_step"], container_obj["name"]) + return [t.as_dict() for t in targets] + + +DEFAULT_BINDABLES = (query, mutation, info, step, container, upload_scalar) + + +class SchemaGen: + jenv: Environment + + def __init__(self): + local_folder = Path(__file__).parent.resolve() + self.jenv = Environment(loader=FileSystemLoader(str(local_folder))) + self.jenv.filters["rank_param"] = self._rank_to_arguments + self.jenv.filters["snake_case"] = str_to_snake_case + + def get_schema(self, manager: Manager) -> GraphQLSchema: + structure = manager.pipeline_artifact_structure() + str_schema = self._generate_schema(structure) + bindable_gen = BindableGen(structure) + bindables = [*DEFAULT_BINDABLES, *bindable_gen.get_bindables()] + schema = make_executable_schema(str_schema, *bindables) # type: ignore + return schema + + @staticmethod + def _rank_to_arguments(rank: Rank): + if rank.depth == 0: + return "" + params = ", ".join([f"param{i+1}: String!" for i in range(rank.depth)]) + return f"({params})" + + def _generate_schema(self, structure: Dict[Rank, List[Step]]) -> str: + template = self.jenv.get_template("schema.graphql.tpl") + render = template.render(structure=structure) + return render + + +class BindableGen: + structure: Dict[Rank, List[Step]] + + def __init__(self, structure: Dict[Rank, List[Step]]): + self.structure = structure + + def get_bindables(self) -> List[ObjectType]: + return [self.get_query_bindable(), *self.get_rank_bindables()] + + def get_query_bindable(self) -> QueryType: + query_obj = QueryType() + for rank in self.structure.keys(): + query_obj.set_field(rank.name.lower(), self.rank_handle) + + return query_obj + + def get_rank_bindables(self) -> List[ObjectType]: + bindables = [] + for rank, steps in self.structure.items(): + rank_obj = ObjectType(rank.name.capitalize()) + bindables.append(rank_obj) + for step in steps: + handle = self.gen_step_handle(step) + rank_obj.set_field(str_to_snake_case(step.name), handle) + + return bindables + + @staticmethod + def rank_handle(_, info, **params): + if not params: + return {"_target": None} + + params_arr = [] + i = 1 + while True: + if f"param{i}" in params: + params_arr.append(params[f"param{i}"]) + i += 1 + else: + return {"_target": "/".join(params_arr)} + + @staticmethod + def gen_step_handle(step: Step): + def rank_step_handle(obj, info, *, only_if_ready=False): + manager: Manager = info.context["g"].manager + return manager.produce_target(step.name, obj["_target"], only_if_ready=only_if_ready) + + return rank_step_handle + + +schema_gen = SchemaGen() + + +class SchemafulManager(Manager): + _schema: Optional[GraphQLSchema] + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._schema = None + + @property + def schema(self) -> GraphQLSchema: + if self._schema is None: + self._schema = schema_gen.get_schema(self) + return self._schema diff --git a/python/revng/daemon/templates/base.html b/python/revng/daemon/templates/base.html new file mode 100644 index 000000000..57f41862b --- /dev/null +++ b/python/revng/daemon/templates/base.html @@ -0,0 +1,30 @@ + + + + + + + + + {% block title %}{% endblock %} + + + + {% block head %}{% endblock %} + + + +{% block body %}{% endblock %} + + + diff --git a/python/revng/daemon/templates/index.html b/python/revng/daemon/templates/index.html new file mode 100644 index 000000000..818ea2705 --- /dev/null +++ b/python/revng/daemon/templates/index.html @@ -0,0 +1,161 @@ + +{% extends "base.html" %} +{% block title %}Index{% endblock %} +{% block body %} +

revng API Debug

+
+
+ GraphQL Sandbox +
+
+ Welcome to revng!
+ Working directory is {{ g.workdir }} +
+ +
+ + + +
+ +
+ + + + + + + + + +
NameProduceReady
+
+
+ +{% endblock %} diff --git a/python/revng/daemon/util.py b/python/revng/daemon/util.py new file mode 100644 index 000000000..03d52c022 --- /dev/null +++ b/python/revng/daemon/util.py @@ -0,0 +1,83 @@ +# +# This file is distributed under the MIT License. See LICENSE.md for details. +# + +import os +from base64 import b64encode +from pathlib import Path +from tempfile import mkdtemp +from typing import Dict, List + +from xdg import xdg_data_home + + +def clean_double_dict(dictionary: Dict[str, Dict[str, List]]): + keys_to_delete = [] + for key in dictionary.keys(): + clean_dict(dictionary[key]) + if not dictionary[key]: + keys_to_delete.append(key) + + for key in keys_to_delete: + dictionary.pop(key) + + +def clean_dict(dictionary: Dict[str, List]): + keys_to_delete = [] + for key in dictionary.keys(): + if not dictionary[key]: + keys_to_delete.append(key) + + for key in keys_to_delete: + dictionary.pop(key) + + +def clean_step_list(step_list: List): + for step in step_list: + clean_container_list(step["containers"]) + + for step in step_list[:]: + if len(step["containers"]) == 0: + step_list.remove(step) + + +def clean_container_list(container_list: List): + for container in container_list[:]: + if len(container["targets"]) == 0: + container_list.remove(container) + + +def str_to_snake_case(string: str) -> str: + ret = [] + for idx, char in enumerate(string): + if char.isupper(): + if (idx > 0 and string[idx - 1].isupper()) or idx == 0: + ret.append(char.lower()) + else: + ret += ["_", char.lower()] + else: + ret.append(char) + return "".join(ret) + + +def b64e(string: str) -> str: + ret = b64encode(string.encode("utf-8")) + return ret.decode("utf-8") + + +def project_workdir() -> Path: + data_dir = os.getenv("REVNG_DATA_DIR") + project_id = os.getenv("REVNG_PROJECT_ID") + + if data_dir is None and project_id is None: + workdir = Path(mkdtemp()) + elif data_dir is not None and project_id is None: + workdir = Path(data_dir) + elif project_id is not None: + if data_dir is not None: + workdir = Path(data_dir) / b64e(project_id) + else: + workdir = xdg_data_home() / "revng" / b64e(project_id) + + workdir.mkdir(parents=True, exist_ok=True) + return workdir