From d3f3cd37657f2e53eab1ef64e4051985d50ed6fc Mon Sep 17 00:00:00 2001 From: Evan Downing <2077950+evandowning@users.noreply.github.com> Date: Tue, 4 Mar 2025 10:51:53 -0500 Subject: [PATCH] Add graphdb indexing (#114) * Add indexes for fields * Add indexing schema --- common/src/buttercup/common/queues.py | 4 +- compose.yaml | 3 ++ program-model/README.md | 12 ++++-- program-model/conf/schema.json | 54 +++++++++++++++++++++++++++ program-model/dev.md | 33 ---------------- program-model/load_data.groovy | 5 +++ 6 files changed, 72 insertions(+), 39 deletions(-) create mode 100644 program-model/conf/schema.json delete mode 100644 program-model/dev.md diff --git a/common/src/buttercup/common/queues.py b/common/src/buttercup/common/queues.py index a30a4d06..689e0037 100644 --- a/common/src/buttercup/common/queues.py +++ b/common/src/buttercup/common/queues.py @@ -64,8 +64,8 @@ DELETE_TASK_TIMEOUT_MS = int(os.getenv("DELETE_TASK_TIMEOUT_MS", 5 * 60 * 1000)) CRASH_TASK_TIMEOUT_MS = int(os.getenv("CRASH_TASK_TIMEOUT_MS", 4 * 60 * 1000)) PATCH_TASK_TIMEOUT_MS = int(os.getenv("PATCH_TASK_TIMEOUT_MS", 10 * 60 * 1000)) CONFIRMED_VULNERABILITIES_TASK_TIMEOUT_MS = int(os.getenv("CONFIRMED_VULNERABILITIES_TASK_TIMEOUT_MS", 10 * 60 * 1000)) -INDEX_TASK_TIMEOUT_MS = int(os.getenv("INDEX_TASK_TIMEOUT_MS", 10 * 60 * 1000)) -INDEX_OUTPUT_TASK_TIMEOUT_MS = int(os.getenv("INDEX_OUTPUT_TASK_TIMEOUT_MS", 10 * 60 * 1000)) +INDEX_TASK_TIMEOUT_MS = int(os.getenv("INDEX_TASK_TIMEOUT_MS", 30 * 60 * 1000)) +INDEX_OUTPUT_TASK_TIMEOUT_MS = int(os.getenv("INDEX_OUTPUT_TASK_TIMEOUT_MS", 3 * 60 * 1000)) TRACED_VULNERABILITIES_TASK_TIMEOUT_MS = int(os.getenv("TRACED_VULNERABILITIES_TASK_TIMEOUT_MS", 10 * 60 * 1000)) logger = logging.getLogger(__name__) diff --git a/compose.yaml b/compose.yaml index 31915787..675d3cd5 100644 --- a/compose.yaml +++ b/compose.yaml @@ -73,6 +73,8 @@ services: - janusgraph.storage.backend=cql - janusgraph.storage.hostname=jg-cassandra - janusgraph.storage.cql.keyspace=janusgraph + - janusgraph.schema.init.strategy=json + - janusgraph.schema.init.json.file=/opt/janusgraph/conf/schema.json - _JAVA_OPTIONS=-Xmx2g -Xms2g healthcheck: test: ["CMD-SHELL", "bin/gremlin.sh", "-e", "scripts/remote-connect.groovy"] @@ -84,6 +86,7 @@ services: volumes: - ./crs_scratch:/crs_scratch - ./program-model/conf/janusgraph-server.yaml:/opt/janusgraph/conf/janusgraph-server.yaml + - ./program-model/conf/schema.json:/opt/janusgraph/conf/schema.json depends_on: graphdb-storage: condition: service_healthy diff --git a/program-model/README.md b/program-model/README.md index 5f4fe18b..1ff7a816 100644 --- a/program-model/README.md +++ b/program-model/README.md @@ -93,19 +93,23 @@ See [challenges.md](challenges.md). ## Development -See [dev.md](dev.md). - Sync, reformat, lint, and test before committing changes to this directory: ```shell just all ``` +## JanusGraph Indexing References + +* +* +* +* +* + ## FAQs * Why does this use a ubuntu base image? * Since Kythe uses OSS Fuzz to build and index the challenge source code, we have to use the same base image as ClusterFuzz. * Why does this use Python 3.10? * ClusterFuzz uses Python 3.10. -* How do I build Kythe? - * Follow the instructions in [dev.md](dev.md). diff --git a/program-model/conf/schema.json b/program-model/conf/schema.json new file mode 100644 index 00000000..a225cf2d --- /dev/null +++ b/program-model/conf/schema.json @@ -0,0 +1,54 @@ +{ + "edgeLabels": [ + { + "label": "/kythe/edge/defines", + "multiplicity": "MULTI", + "unidirected": false + } + ], + "propertyKeys": [ + { + "key": "path", + "className": "java.lang.String", + "cardinality": "SINGLE" + }, + { + "key": "/kythe/code", + "className": "java.lang.String", + "cardinality": "SINGLE" + }, + { + "key": "/kythe/text", + "className": "java.lang.String", + "cardinality": "SINGLE" + }, + { + "key": "/kythe/node/kind", + "className": "java.lang.String", + "cardinality": "SINGLE" + } + ], + "compositeIndexes": [ + { + "name": "kindCompositeIndex", + "typeClass": "org.apache.tinkerpop.gremlin.structure.Vertex", + "keys": [ + { + "propertyKey": "/kythe/node/kind" + } + ] + } + ], + "mixedIndexes": [ + { + "name": "codeMixedIndex", + "typeClass": "org.apache.tinkerpop.gremlin.structure.Vertex", + "indexBackend": "search", + "keys": [ + { + "propertyKey": "/kythe/code" + } + ] + } + ] +} \ No newline at end of file diff --git a/program-model/dev.md b/program-model/dev.md deleted file mode 100644 index 39462253..00000000 --- a/program-model/dev.md +++ /dev/null @@ -1,33 +0,0 @@ -# Dev - -Documentation for developing Program Model. - -## Kythe - -Takes about 3 hours to build. - -### Install Bazel - -Follow [instructions](https://bazel.build/install/ubuntu#install-on-ubuntu). You will need to install a specific version of Bazel depending on what the `build` output below outputs. - -### Install Dependencies - -```shell -sudo apt install flex bison asciidoc graphviz source-highlight clang -``` - -### Download and build Kythe - -From [documentation](https://kythe.io/getting-started/#build-a-release-of-kythe-using-bazel-and-unpack-it-in-optkythe) - -```shell -git clone git@github.com:trailofbits/aixcc-kythe.git - -cd aixcc-kythe/ - -bazel build //kythe/release - -mkdir ../opt/ - -tar -zxf bazel-bin/kythe/release/kythe.tar.gz --directory ../opt/ -``` diff --git a/program-model/load_data.groovy b/program-model/load_data.groovy index 86f1ade9..7a9a0136 100644 --- a/program-model/load_data.groovy +++ b/program-model/load_data.groovy @@ -6,6 +6,11 @@ // 1 hour in milliseconds :remote config timeout 3600000 +println("Printing schema...") + +mgmt = graph.openManagement() +mgmt.printSchema() + println("Starting graph creation...") g.io('/crs_scratch/graph.xml').read().iterate()