Introduce the revng.daemon GraphQL API

This commit introduces the `revng.daemon` Python module, a Flask-powered
web application that exposes the functionality from `revng.api` across a
GraphQL API

python/CMakeLists.txt

@@ -158,3 +158,18 @@ foreach(HEADER ${REQUIRED_PIPELINE_C_HEADERS})
   configure_file("${PIPELINE_C_HEADERS_SOURCE_DIR}/${HEADER}"
                  "${PIPELINE_C_HEADERS_BUILD_DIR}/${HEADER}" COPYONLY)
 endforeach()
+
+#
+# Install revng.daemon
+#
+set(REVNG_DAEMON_MODULE_FILES
+    revng/daemon/__init__.py
+    revng/daemon/api.py
+    revng/daemon/demo_webpage.py
+    revng/daemon/schema.py
+    revng/daemon/schema.graphql.tpl
+    revng/daemon/util.py
+    revng/daemon/templates/base.html
+    revng/daemon/templates/index.html)
+python_module(TARGET_NAME revng-python-daemon MODULE_FILES
+              ${REVNG_DAEMON_MODULE_FILES})

python/requirements.txt

@@ -9,3 +9,11 @@ grandiso
 
 # revng.api
 cffi
+
+# revng.daemon
+Flask
+Werkzeug
+ariadne
+aiodataloader
+Jinja2
+xdg

python/revng/daemon/__init__.py

@@ -0,0 +1,58 @@
+#
+# This file is distributed under the MIT License. See LICENSE.md for details.
+#
+
+import atexit
+import logging
+import secrets
+from pathlib import Path
+from typing import Optional
+
+from flask import Flask, g
+
+from revng.api._capi import initialize as capi_initialize
+
+from .api import api_blueprint
+from .demo_webpage import demo_blueprint
+from .schema import SchemafulManager
+from .util import project_workdir
+
+workdir: Path = project_workdir()
+manager: Optional[SchemafulManager] = None
+
+
+app = Flask(__name__)
+app.register_blueprint(api_blueprint)
+app.register_blueprint(demo_blueprint)
+
+app.secret_key = secrets.token_hex(16)
+
+
+def cleanup():
+    if manager is not None:
+        store_result = manager.store_containers()
+        if not store_result:
+            logging.warning("Failed to store manager's containers")
+
+
+atexit.register(cleanup)
+
+
+@app.before_first_request
+def init():
+    global manager
+    capi_initialize()
+    manager = SchemafulManager(workdir=str(workdir.resolve()))
+
+
+@app.before_request
+def init_global_object():
+    assert manager, "Manager not initialized"
+
+    g.workdir = workdir
+    g.manager = manager
+
+    # Safety checks
+    assert g.workdir is not None
+    assert g.workdir.exists() and g.workdir.is_dir()
+    assert g.manager is not None

python/revng/daemon/api.py

@@ -0,0 +1,74 @@
+#
+# This file is distributed under the MIT License. See LICENSE.md for details.
+#
+
+import json
+from typing import TYPE_CHECKING, cast
+
+from flask import Blueprint, current_app, jsonify, request
+
+from ariadne import graphql
+from ariadne.constants import PLAYGROUND_HTML
+from ariadne.contrib.tracing.apollotracing import ApolloTracingExtension
+from ariadne.file_uploads import FilesDict, combine_multipart_data
+
+from .schema import SchemafulManager
+
+if TYPE_CHECKING:
+    from flask.ctx import _AppCtxGlobals
+
+    class FlaskGlobals(_AppCtxGlobals):
+        manager: SchemafulManager
+
+    g = FlaskGlobals()
+else:
+    from flask import g
+
+
+api_blueprint = Blueprint("api", __name__)
+
+
+def json_response(data=None):
+    if data is None:
+        data = {}
+    return jsonify(data)
+
+
+def json_error(message, http_code=404):
+    data = {
+        "error": message,
+    }
+    return jsonify(data), http_code
+
+
+# When navigating to http://<server>/graphql show a graphql sandbox
+@api_blueprint.route("/graphql", methods=["GET"])
+def graphql_playground():
+    return PLAYGROUND_HTML, 200
+
+
+@api_blueprint.route("/graphql", methods=["POST"])
+async def graphql_server():
+    if request.content_type.startswith("multipart/form-data;"):
+        operations = request.form.get("operations")
+        req_map = request.form.get("map")
+        if operations is not None and req_map is not None:
+            request_files = cast(FilesDict, request.files)
+            data = combine_multipart_data(
+                json.loads(operations), json.loads(req_map), request_files
+            )
+        else:
+            return json_error("Invalid form data")
+    else:
+        data = request.get_json()
+
+    success, result = await graphql(
+        g.manager.schema,
+        data,
+        context_value={"g": g, "request": request},
+        extensions=[ApolloTracingExtension],
+        debug=current_app.config["DEBUG"],
+    )
+
+    status_code = 200 if success else 400
+    return jsonify(result), status_code

python/revng/daemon/demo_webpage.py

@@ -0,0 +1,12 @@
+#
+# This file is distributed under the MIT License. See LICENSE.md for details.
+#
+
+from flask import Blueprint, render_template
+
+demo_blueprint = Blueprint("demo", __name__)
+
+
+@demo_blueprint.route("/")
+def index():
+    return render_template("index.html")

python/revng/daemon/schema.graphql.tpl

@@ -0,0 +1,69 @@
+{#
+  This file is distributed under the MIT License. See LICENSE.md for details.
+#}
+type Query {
+    info: Info!
+    step(name: String!): Step
+    container(name: String!, step: String!): Container
+    targets(pathspec: String!): [Step!]!
+    produce(step: String!, container: String!, target_list: String!, only_if_ready: Boolean): String
+    produce_artifacts(step: String!, paths: String, only_if_ready: Boolean): String
+
+    {%- for rank in structure.keys() %}
+    {{ rank.name }}{{ rank | rank_param }}: {{ rank.name | capitalize }}!
+    {%- endfor %}
+}
+
+type Mutation {
+    upload_b64(input: String!, container: String!): Boolean!
+    upload_file(file: Upload, container: String!): Boolean!
+}
+
+type Info {
+    kinds: [Kind!]!
+    ranks: [Rank!]!
+    steps: [Step!]!
+    model: String!
+}
+
+type Kind {
+    name: ID
+    rank: String!
+    parent: String
+}
+
+type Rank {
+    name: ID
+    depth: Int!
+    parent: String
+}
+
+type Step {
+    name: ID
+    parent: String
+    containers: [Container!]!
+}
+
+type Container {
+    name: ID
+    mime: String
+    targets: [Target!]!
+}
+
+type Target {
+    serialized: String
+    exact: Boolean
+    path_components: [String!]!
+    kind: String
+    ready: Boolean
+}
+
+{% for rank, steps in structure.items() %}
+type {{ rank.name | capitalize }} {
+{%- for step in steps %}
+    {{ step.name | snake_case }}(only_if_ready: Boolean): String!
+{%- endfor %}
+}
+{% endfor %}
+
+scalar Upload

python/revng/daemon/schema.py

@@ -0,0 +1,241 @@
+#
+# This file is distributed under the MIT License. See LICENSE.md for details.
+#
+
+import logging
+from base64 import b64decode
+from pathlib import Path
+from typing import Dict, List, Optional
+
+from ariadne import MutationType, ObjectType, QueryType, make_executable_schema, upload_scalar
+from graphql.type.schema import GraphQLSchema
+from jinja2 import Environment, FileSystemLoader
+
+from revng.api.manager import Manager
+from revng.api.rank import Rank
+from revng.api.step import Step
+
+from .util import clean_step_list, str_to_snake_case
+
+query = QueryType()
+mutation = MutationType()
+info = ObjectType("Info")
+step = ObjectType("Step")
+container = ObjectType("Container")
+
+
+@query.field("info")
+async def resolve_root(_, info):
+    return {}
+
+
+@query.field("produce")
+async def resolve_produce(obj, info, *, step, container, target_list, only_if_ready=False):
+    manager: Manager = info.context["g"].manager
+    targets = target_list.split(",")
+    return manager.produce_target(step, targets, container, only_if_ready)
+
+
+@query.field("produce_artifacts")
+async def resolve_produce_artifacts(obj, info, *, step, paths=None, only_if_ready=False):
+    manager: Manager = info.context["g"].manager
+    target_paths = paths.split(",") if paths is not None else None
+    return manager.produce_target(step, target_paths, only_if_ready=only_if_ready)
+
+
+@query.field("step")
+async def resolve_step(_, info, *, name):
+    manager: Manager = info.context["g"].manager
+    step = manager.get_step(name)
+    return step.as_dict() if step is not None else {}
+
+
+@query.field("container")
+async def resolve_container(_, info, *, name, step):
+    manager: Manager = info.context["g"].manager
+    container_id = manager.get_container_with_name(name)
+    step = manager.get_step(step)
+    if step is None or container_id is None:
+        return {}
+    container = step.get_container(container_id)
+    return container.as_dict() if container is not None else {}
+
+
+@query.field("targets")
+async def resolve_targets(_, info, *, pathspec):
+    manager: Manager = info.context["g"].manager
+    targets = manager.get_all_targets()
+    result = [
+        {
+            "name": k,
+            "containers": [
+                {"name": k2, "targets": [t.as_dict() for t in v2 if t.joined_path() == pathspec]}
+                for k2, v2 in v.items()
+            ],
+        }
+        for k, v in targets.items()
+    ]
+    clean_step_list(result)
+    return result
+
+
+@mutation.field("upload_b64")
+async def resolve_upload_b64(_, info, *, input: str, container: str):  # noqa: A002
+    g = info.context["g"]
+    g.manager.set_input(container, b64decode(input))
+    logging.info(f"Saved file for container {container}")
+    return True
+
+
+@mutation.field("upload_file")
+async def resolve_upload_file(_, info, *, file, container: str):
+    g = info.context["g"]
+    g.manager.set_input(container, file.read())
+    logging.info(f"Saved file for container {container}")
+    return True
+
+
+@info.field("ranks")
+async def resolve_ranks(_, info):
+    return [x.as_dict() for x in Rank.ranks()]
+
+
+@info.field("kinds")
+async def resolve_root_kinds(_, info):
+    manager = info.context["g"].manager
+    return [k.as_dict() for k in manager.kinds()]
+
+
+@info.field("model")
+async def resolve_root_model(_, info):
+    manager = info.context["g"].manager
+    return manager.get_model()
+
+
+@info.field("steps")
+async def resolve_root_steps(_, info):
+    manager: Manager = info.context["g"].manager
+    return [s.as_dict() for s in manager.steps()]
+
+
+@step.field("containers")
+async def resolve_step_containers(step_obj, info):
+    if "containers" in step_obj:
+        return step_obj["containers"]
+
+    manager: Manager = info.context["g"].manager
+    step = manager.get_step(step_obj["name"])
+    if step is None:
+        return []
+    containers = [step.get_container(c) for c in manager.containers()]
+    return [c.as_dict() for c in containers if c is not None]
+
+
+@container.field("targets")
+async def resolve_container_targets(container_obj, info):
+    if "targets" in container_obj:
+        return container_obj["targets"]
+
+    manager: Manager = info.context["g"].manager
+    targets = manager.get_targets(container_obj["_step"], container_obj["name"])
+    return [t.as_dict() for t in targets]
+
+
+DEFAULT_BINDABLES = (query, mutation, info, step, container, upload_scalar)
+
+
+class SchemaGen:
+    jenv: Environment
+
+    def __init__(self):
+        local_folder = Path(__file__).parent.resolve()
+        self.jenv = Environment(loader=FileSystemLoader(str(local_folder)))
+        self.jenv.filters["rank_param"] = self._rank_to_arguments
+        self.jenv.filters["snake_case"] = str_to_snake_case
+
+    def get_schema(self, manager: Manager) -> GraphQLSchema:
+        structure = manager.pipeline_artifact_structure()
+        str_schema = self._generate_schema(structure)
+        bindable_gen = BindableGen(structure)
+        bindables = [*DEFAULT_BINDABLES, *bindable_gen.get_bindables()]
+        schema = make_executable_schema(str_schema, *bindables)  # type: ignore
+        return schema
+
+    @staticmethod
+    def _rank_to_arguments(rank: Rank):
+        if rank.depth == 0:
+            return ""
+        params = ", ".join([f"param{i+1}: String!" for i in range(rank.depth)])
+        return f"({params})"
+
+    def _generate_schema(self, structure: Dict[Rank, List[Step]]) -> str:
+        template = self.jenv.get_template("schema.graphql.tpl")
+        render = template.render(structure=structure)
+        return render
+
+
+class BindableGen:
+    structure: Dict[Rank, List[Step]]
+
+    def __init__(self, structure: Dict[Rank, List[Step]]):
+        self.structure = structure
+
+    def get_bindables(self) -> List[ObjectType]:
+        return [self.get_query_bindable(), *self.get_rank_bindables()]
+
+    def get_query_bindable(self) -> QueryType:
+        query_obj = QueryType()
+        for rank in self.structure.keys():
+            query_obj.set_field(rank.name.lower(), self.rank_handle)
+
+        return query_obj
+
+    def get_rank_bindables(self) -> List[ObjectType]:
+        bindables = []
+        for rank, steps in self.structure.items():
+            rank_obj = ObjectType(rank.name.capitalize())
+            bindables.append(rank_obj)
+            for step in steps:
+                handle = self.gen_step_handle(step)
+                rank_obj.set_field(str_to_snake_case(step.name), handle)
+
+        return bindables
+
+    @staticmethod
+    def rank_handle(_, info, **params):
+        if not params:
+            return {"_target": None}
+
+        params_arr = []
+        i = 1
+        while True:
+            if f"param{i}" in params:
+                params_arr.append(params[f"param{i}"])
+                i += 1
+            else:
+                return {"_target": "/".join(params_arr)}
+
+    @staticmethod
+    def gen_step_handle(step: Step):
+        def rank_step_handle(obj, info, *, only_if_ready=False):
+            manager: Manager = info.context["g"].manager
+            return manager.produce_target(step.name, obj["_target"], only_if_ready=only_if_ready)
+
+        return rank_step_handle
+
+
+schema_gen = SchemaGen()
+
+
+class SchemafulManager(Manager):
+    _schema: Optional[GraphQLSchema]
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self._schema = None
+
+    @property
+    def schema(self) -> GraphQLSchema:
+        if self._schema is None:
+            self._schema = schema_gen.get_schema(self)
+        return self._schema

python/revng/daemon/templates/base.html

@@ -0,0 +1,30 @@
+<!--
+  This file is distributed under the MIT License. See LICENSE.md for details.
+-->
+<!doctype html>
+<html>
+<head>
+  <meta charset="utf-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1">
+  <link href="https://cdn.jsdelivr.net/npm/bootstrap@5.1.3/dist/css/bootstrap.min.css" rel="stylesheet" integrity="sha384-1BmE4kWBq78iYhFldvKuhfTAU6auU8tT94WrHftjDbrCEXSU1oBoqyl2QvZ6jIW3" crossorigin="anonymous">
+  <link href="https:////cdn.datatables.net/1.11.3/css/jquery.dataTables.min.css" rel="stylesheet" crossorigin="anonymous">
+  <title>{% block title %}{% endblock %}</title>
+  <script src="https://code.jquery.com/jquery-3.6.0.js" integrity="sha256-H+K7U5CnXl1h5ywQfKtSj8PCmoN9aaq30gDh27Xc0jk=" crossorigin="anonymous"></script>
+  <script src="https://cdn.datatables.net/1.11.3/js/jquery.dataTables.min.js" crossorigin="anonymous"></script>
+  <script>
+  jQuery.fn.outerHTML = function() {
+    return jQuery('<div />').append(this.eq(0).clone()).html();
+  };
+  </script>
+  {% block head %}{% endblock %}
+</head>
+<body class="text-center">
+<style>
+  html, body {
+    height: 100%;
+  }
+</style>
+{% block body %}{% endblock %}
+<script src="https://cdn.jsdelivr.net/npm/bootstrap@5.1.3/dist/js/bootstrap.bundle.min.js" integrity="sha384-ka7Sk0Gln4gmtz2MlQnikT1wXgYsOg+OMhuP+IlRH9sENBO0LRn5q+8nbTov4+1p" crossorigin="anonymous"></script>
+</body>
+</html>

python/revng/daemon/templates/index.html

@@ -0,0 +1,161 @@
+<!--
+  This file is distributed under the MIT License. See LICENSE.md for details.
+-->
+{% extends "base.html" %}
+{% block title %}Index{% endblock %}
+{% block body %}
+<h1>revng API Debug</h1>
+<div class="container text-start">
+  <div>
+    <a href="/graphql" target="_blank">GraphQL Sandbox</a>
+  </div>
+  <div>
+    Welcome to revng!<br>
+    Working directory is {{ g.workdir }}
+  </div>
+
+  <div>
+    <label for="input-file">Input</label>
+    <input id="input-file" type="file">
+    <button class="btn btn-primary" onclick="setInputFile()">Set input file</button>
+  </div>
+
+  <div id="targets">
+    <table id="targets-table">
+      <thead>
+        <tr>
+          <td>Name</td>
+          <td>Produce</td>
+          <td>Ready</td>
+        </tr>
+      </thead>
+      <tbody></tbody>
+    </table>
+  </div>
+</div>
+<script>
+  "use strict";
+  function wrap_gql(query) {
+    return JSON.stringify({
+      operationName: null,
+      variables: {},
+      query: query,
+    });
+  }
+
+  async function gql(query) {
+    return fetch("/graphql", {
+      body: wrap_gql(query),
+      method: 'POST',
+      mode: 'cors',
+      headers: new Headers({'Content-Type': 'application/json'})
+    }).then(res => res.json());
+  }
+
+  async function produce(step_name, container_name, kind_name, exact, path_components) {
+    let target_path = `${path_components.join("/")}:${kind_name}`
+    let query = `
+    {
+      produce(step: "${step_name}", container: "${container_name}", target_list: "${target_path}")
+    }
+    `;
+
+    let response = await gql(query);
+    console.log("Response: ", response);
+  }
+
+  async function loadTargets() {
+    let targets_query = `
+      {
+        info {
+          steps {
+            name
+            containers {
+              name
+              targets {
+                kind
+                ready
+                serialized
+                exact
+                path_components
+              }
+            }
+          }
+        }
+      }
+    `;
+    let targets = await gql(targets_query);
+
+    for (let step of targets.data.info.steps) {
+      for (let container of step.containers) {
+        for (let target of container.targets) {
+          let target_name = target.path_components.join("/");
+          let target_fullname = `${step.name}/${container.name}/${target_name}:${target.kind}`;
+
+          let target_properties = {
+            step: step.name,
+            container: container.name,
+            kind: target.kind,
+            exact: true,
+            path_components: target.path_components,
+          }
+
+          let ready = target.ready ? "Yes" : "No";
+
+          // TODO: passing the properties twice is ugly
+          $('#targets-table').DataTable().row.add([
+            target_fullname,
+            target_properties,
+            ready,
+          ]);
+        }
+      }
+    }
+    $('#targets-table').DataTable().draw();
+  }
+
+  async function setInputFile() {
+    const file_reader = new FileReader();
+    let input_file = $("#input-file")[0];
+    file_reader.readAsBinaryString(input_file.files[0])
+    await new Promise(function(resolve, reject) {
+      file_reader.addEventListener('loadend', function() {
+        resolve();
+      });
+    });
+    let input_query = `
+      mutation {
+        upload_b64(input: "${btoa(file_reader.result)}", container: "input")
+      }
+    `;
+
+    let response = await gql(input_query);
+
+    // TODO: do not reload the page, reload the targets instead
+    window.location.reload();
+  }
+
+  $(function(){
+    $('#targets-table').DataTable({
+      autoWidth: true,
+      columns: [
+        null,
+        {
+          render: function(data, type, row) {
+            let produce_button = $("<button>");
+            produce_button.text("Produce");
+            produce_button.addClass(["produce-btn", "btn", "btn-primary", "btn-sm"]);
+            return produce_button.outerHTML();
+          },
+        },
+        null
+      ],
+    })
+    $("#targets-table tbody").on("click", "button.produce-btn", async function() {
+      let data = $("#targets-table").DataTable().row($(this).parents("tr")).data()[1];
+      await produce(data.step, data.container, data.kind, data.exact, data.path_components);
+    });
+    loadTargets();
+  });
+</script>
+{% endblock %}

python/revng/daemon/util.py

@@ -0,0 +1,83 @@
+#
+# This file is distributed under the MIT License. See LICENSE.md for details.
+#
+
+import os
+from base64 import b64encode
+from pathlib import Path
+from tempfile import mkdtemp
+from typing import Dict, List
+
+from xdg import xdg_data_home
+
+
+def clean_double_dict(dictionary: Dict[str, Dict[str, List]]):
+    keys_to_delete = []
+    for key in dictionary.keys():
+        clean_dict(dictionary[key])
+        if not dictionary[key]:
+            keys_to_delete.append(key)
+
+    for key in keys_to_delete:
+        dictionary.pop(key)
+
+
+def clean_dict(dictionary: Dict[str, List]):
+    keys_to_delete = []
+    for key in dictionary.keys():
+        if not dictionary[key]:
+            keys_to_delete.append(key)
+
+    for key in keys_to_delete:
+        dictionary.pop(key)
+
+
+def clean_step_list(step_list: List):
+    for step in step_list:
+        clean_container_list(step["containers"])
+
+    for step in step_list[:]:
+        if len(step["containers"]) == 0:
+            step_list.remove(step)
+
+
+def clean_container_list(container_list: List):
+    for container in container_list[:]:
+        if len(container["targets"]) == 0:
+            container_list.remove(container)
+
+
+def str_to_snake_case(string: str) -> str:
+    ret = []
+    for idx, char in enumerate(string):
+        if char.isupper():
+            if (idx > 0 and string[idx - 1].isupper()) or idx == 0:
+                ret.append(char.lower())
+            else:
+                ret += ["_", char.lower()]
+        else:
+            ret.append(char)
+    return "".join(ret)
+
+
+def b64e(string: str) -> str:
+    ret = b64encode(string.encode("utf-8"))
+    return ret.decode("utf-8")
+
+
+def project_workdir() -> Path:
+    data_dir = os.getenv("REVNG_DATA_DIR")
+    project_id = os.getenv("REVNG_PROJECT_ID")
+
+    if data_dir is None and project_id is None:
+        workdir = Path(mkdtemp())
+    elif data_dir is not None and project_id is None:
+        workdir = Path(data_dir)
+    elif project_id is not None:
+        if data_dir is not None:
+            workdir = Path(data_dir) / b64e(project_id)
+        else:
+            workdir = xdg_data_home() / "revng" / b64e(project_id)
+
+    workdir.mkdir(parents=True, exist_ok=True)
+    return workdir