Add graphdb indexing (#114)

* Add indexes for fields

* Add indexing schema

common/src/buttercup/common/queues.py

@@ -64,8 +64,8 @@ DELETE_TASK_TIMEOUT_MS = int(os.getenv("DELETE_TASK_TIMEOUT_MS", 5 * 60 * 1000))
 CRASH_TASK_TIMEOUT_MS = int(os.getenv("CRASH_TASK_TIMEOUT_MS", 4 * 60 * 1000))
 PATCH_TASK_TIMEOUT_MS = int(os.getenv("PATCH_TASK_TIMEOUT_MS", 10 * 60 * 1000))
 CONFIRMED_VULNERABILITIES_TASK_TIMEOUT_MS = int(os.getenv("CONFIRMED_VULNERABILITIES_TASK_TIMEOUT_MS", 10 * 60 * 1000))
-INDEX_TASK_TIMEOUT_MS = int(os.getenv("INDEX_TASK_TIMEOUT_MS", 10 * 60 * 1000))
-INDEX_OUTPUT_TASK_TIMEOUT_MS = int(os.getenv("INDEX_OUTPUT_TASK_TIMEOUT_MS", 10 * 60 * 1000))
+INDEX_TASK_TIMEOUT_MS = int(os.getenv("INDEX_TASK_TIMEOUT_MS", 30 * 60 * 1000))
+INDEX_OUTPUT_TASK_TIMEOUT_MS = int(os.getenv("INDEX_OUTPUT_TASK_TIMEOUT_MS", 3 * 60 * 1000))
 TRACED_VULNERABILITIES_TASK_TIMEOUT_MS = int(os.getenv("TRACED_VULNERABILITIES_TASK_TIMEOUT_MS", 10 * 60 * 1000))
 
 logger = logging.getLogger(__name__)

compose.yaml

@@ -73,6 +73,8 @@ services:
       - janusgraph.storage.backend=cql
       - janusgraph.storage.hostname=jg-cassandra
       - janusgraph.storage.cql.keyspace=janusgraph
+      - janusgraph.schema.init.strategy=json
+      - janusgraph.schema.init.json.file=/opt/janusgraph/conf/schema.json
       - _JAVA_OPTIONS=-Xmx2g -Xms2g
     healthcheck:
       test: ["CMD-SHELL", "bin/gremlin.sh", "-e", "scripts/remote-connect.groovy"]
@@ -84,6 +86,7 @@ services:
     volumes:
       - ./crs_scratch:/crs_scratch
       - ./program-model/conf/janusgraph-server.yaml:/opt/janusgraph/conf/janusgraph-server.yaml
+      - ./program-model/conf/schema.json:/opt/janusgraph/conf/schema.json
     depends_on:
       graphdb-storage:
         condition: service_healthy

program-model/README.md

@@ -93,19 +93,23 @@ See [challenges.md](challenges.md).
 
 ## Development
 
-See [dev.md](dev.md).
-
 Sync, reformat, lint, and test before committing changes to this directory:
 
 ```shell
 just all
 ```
 
+## JanusGraph Indexing References
+
+* <https://user3141592.medium.com/single-vs-composite-indexes-in-relational-databases-58d0eb045cbe>
+* <https://docs.janusgraph.org/schema/index-management/index-performance/>
+* <https://docs.janusgraph.org/schema/schema-init-strategies/>
+* <https://docs.janusgraph.org/configs/configuration-reference/#schema>
+* <https://docs.janusgraph.org/v0.3/basics/schema/#:~:text=.util.UUID%20)-,Property%20Key%20Cardinality,all%20elements%20in%20the%20graph.>
+
 ## FAQs
 
 * Why does this use a ubuntu base image?
   * Since Kythe uses OSS Fuzz to build and index the challenge source code, we have to use the same base image as ClusterFuzz.
 * Why does this use Python 3.10?
   * ClusterFuzz uses Python 3.10.
-* How do I build Kythe?
-  * Follow the instructions in [dev.md](dev.md).

program-model/conf/schema.json

@@ -0,0 +1,54 @@
+{
+    "edgeLabels": [
+        {
+            "label": "/kythe/edge/defines",
+            "multiplicity": "MULTI",
+            "unidirected": false
+        }
+    ],
+    "propertyKeys": [
+        {
+            "key": "path",
+            "className": "java.lang.String",
+            "cardinality": "SINGLE"
+        },
+        {
+            "key": "/kythe/code",
+            "className": "java.lang.String",
+            "cardinality": "SINGLE"
+        },
+        {
+            "key": "/kythe/text",
+            "className": "java.lang.String",
+            "cardinality": "SINGLE"
+        },
+        {
+            "key": "/kythe/node/kind",
+            "className": "java.lang.String",
+            "cardinality": "SINGLE"
+        }
+    ],
+    "compositeIndexes": [
+        {
+            "name": "kindCompositeIndex",
+            "typeClass": "org.apache.tinkerpop.gremlin.structure.Vertex",
+            "keys": [
+                {
+                    "propertyKey": "/kythe/node/kind"
+                }
+            ]
+        }
+    ],
+    "mixedIndexes": [
+        {
+            "name": "codeMixedIndex",
+            "typeClass": "org.apache.tinkerpop.gremlin.structure.Vertex",
+            "indexBackend": "search",
+            "keys": [
+                {
+                    "propertyKey": "/kythe/code"
+                }
+            ]
+        }
+    ]
+}

program-model/dev.md

@@ -1,33 +0,0 @@
-# Dev
-
-Documentation for developing Program Model.
-
-## Kythe
-
-Takes about 3 hours to build.
-
-### Install Bazel
-
-Follow [instructions](https://bazel.build/install/ubuntu#install-on-ubuntu). You will need to install a specific version of Bazel depending on what the `build` output below outputs.
-
-### Install Dependencies
-
-```shell
-sudo apt install flex bison asciidoc graphviz source-highlight clang
-```
-
-### Download and build Kythe
-
-From [documentation](https://kythe.io/getting-started/#build-a-release-of-kythe-using-bazel-and-unpack-it-in-optkythe)
-
-```shell
-git clone git@github.com:trailofbits/aixcc-kythe.git
-
-cd aixcc-kythe/
-
-bazel build //kythe/release
-
-mkdir ../opt/
-
-tar -zxf bazel-bin/kythe/release/kythe.tar.gz --directory ../opt/
-```

program-model/load_data.groovy

@@ -6,6 +6,11 @@
 // 1 hour in milliseconds
 :remote config timeout 3600000
 
+println("Printing schema...")
+
+mgmt = graph.openManagement()
+mgmt.printSchema()
+
 println("Starting graph creation...")
 
 g.io('/crs_scratch/graph.xml').read().iterate()