mirror of
https://github.com/trailofbits/buttercup
synced 2026-06-21 14:11:39 +00:00
Merge branch 'main' into update-deps
This commit is contained in:
Michael D Brown
@@ -1,3 +0,0 @@
|
||||
[submodule "program-model/aixcc-kythe"]
|
||||
path = program-model/aixcc-kythe
|
||||
url = git@github.com:trailofbits/aixcc-kythe.git
|
||||
|
||||
@@ -1,30 +1,3 @@
|
||||
BUILDER_IMAGE_NAME := "kythe_builder"
|
||||
|
||||
build-kythe-builder:
|
||||
docker build -t {{BUILDER_IMAGE_NAME}} -f ./program-model/kythe_builder/Dockerfile .
|
||||
|
||||
build-kythe-tar-gz: build-kythe-builder
|
||||
docker run -v {{justfile_directory()}}/program-model/scripts/gzs/:/out {{BUILDER_IMAGE_NAME}}
|
||||
|
||||
download-kythe:
|
||||
#!/usr/bin/env bash
|
||||
set -euxo pipefail
|
||||
if [[ ! -f "program-model/scripts/gzs/kythe-v0.0.67.tar.gz" ]]
|
||||
then
|
||||
mkdir -p program-model/scripts/gzs/
|
||||
gh release download v0.0.2 -R github.com/trailofbits/aixcc-kythe -D program-model/scripts/gzs/
|
||||
fi
|
||||
|
||||
install-gh:
|
||||
(type -p wget >/dev/null || (sudo apt update && sudo apt-get install wget -y)) \
|
||||
&& sudo mkdir -p -m 755 /etc/apt/keyrings \
|
||||
&& out=$(mktemp) && wget -nv -O$out https://cli.github.com/packages/githubcli-archive-keyring.gpg \
|
||||
&& cat $out | sudo tee /etc/apt/keyrings/githubcli-archive-keyring.gpg > /dev/null \
|
||||
&& sudo chmod go+r /etc/apt/keyrings/githubcli-archive-keyring.gpg \
|
||||
&& echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null \
|
||||
&& sudo apt update \
|
||||
&& sudo apt install gh -y
|
||||
|
||||
download-cscope:
|
||||
mkdir -p cscope/
|
||||
docker pull ghcr.io/trailofbits/buttercup-cscope:main
|
||||
@@ -35,12 +8,6 @@ download-cscope:
|
||||
install-cscope:
|
||||
cd cscope/cscope/ && autoreconf -i -s && ./configure && make && sudo make install
|
||||
|
||||
build-indexer-image:
|
||||
docker build -t indexer -f ./program-model/upload_worker.Dockerfile .
|
||||
|
||||
run-indexer: download-kythe
|
||||
docker compose --profile=development up --build indexer-run
|
||||
|
||||
lint-python COMPONENT:
|
||||
cd {{ COMPONENT }} && uv sync --all-extras && uv run ruff format && uv run ruff check --fix && uv run mypy
|
||||
|
||||
|
||||
@@ -1,3 +0,0 @@
|
||||
[submodule "kythe"]
|
||||
path = kythe
|
||||
url = git@github.com:kythe/kythe.git
|
||||
+6
-112
@@ -1,58 +1,10 @@
|
||||
# Program Model
|
||||
|
||||
Indexes a program into a graph database.
|
||||
|
||||
## Requirements
|
||||
|
||||
`n2-highmem-8` [instance](https://cloud.google.com/compute/docs/general-purpose-machines#n2-high-mem).
|
||||
|
||||
```shell
|
||||
sudo apt-get install -y codequery
|
||||
```
|
||||
Indexes a program's source code to be queried by `seed-gen` and `patcher`.
|
||||
|
||||
## Setup
|
||||
|
||||
Set up working directory and tasks directory.
|
||||
|
||||
```shell
|
||||
cd afc-crs-trail-of-bits/
|
||||
|
||||
cp env.template .env
|
||||
|
||||
sudo mkdir /crs_scratch/ && sudo chown `whoami`:`whoami` /crs_scratch && sudo mount --bind ./crs_scratch /crs_scratch
|
||||
|
||||
sudo mkdir /tasks_storage && sudo chown `whoami`:`whoami` /tasks_storage && sudo mount --bind ./tasks_storage /tasks_storage
|
||||
```
|
||||
|
||||
Set up Github token.
|
||||
|
||||
```shell
|
||||
gh auth login
|
||||
GitHub.com
|
||||
SSH
|
||||
No
|
||||
Paste the authentication token
|
||||
|
||||
gh auth token | docker login ghcr.io -u USERNAME --password-stdin
|
||||
```
|
||||
|
||||
Build the [Kythe](https://github.com/trailofbits/aixcc-kythe) docker image and push to `aixcc-finals`.
|
||||
Create your own personal PAT, and give it `write:packages` permissions -- do not configure SSO for any organization.
|
||||
Log into `ghcr.io` like above.
|
||||
|
||||
```shell
|
||||
git clone git@github.com:trailofbits/aixcc-kythe.git
|
||||
cd aixcc-kythe/
|
||||
|
||||
# Takes roughly 70 minutes to build and 15 minutes to copy the .tar.gz file.
|
||||
docker build -t aixcc-kythe -f aixcc.Dockerfile .
|
||||
docker tag aixcc-kythe ghcr.io/aixcc-finals/buttercup-kythe:main
|
||||
docker push ghcr.io/aixcc-finals/buttercup-kythe:main
|
||||
```
|
||||
|
||||
You should see the `buttercup-kythe` package [here](https://github.com/orgs/aixcc-finals/packages?visibility=private).
|
||||
|
||||
Build [cscope](https://github.com/trailofbits/aixcc-cscope) docker image and push to `aixcc-finals`.
|
||||
Build the `cscope` docker image and push to `trailofbits`.
|
||||
|
||||
```shell
|
||||
git clone git@github.com:trailofbits/aixcc-cscope.git
|
||||
@@ -60,74 +12,16 @@ cd aixcc-cscope/
|
||||
git checkout buttercup
|
||||
|
||||
docker build -t aixcc-cscope -f aixcc.Dockerfile .
|
||||
docker tag aixcc-cscope ghcr.io/aixcc-finals/buttercup-cscope:main
|
||||
docker push ghcr.io/aixcc-finals/buttercup-cscope:main
|
||||
docker tag aixcc-cscope ghcr.io/trailofbits/buttercup-cscope:main
|
||||
docker push ghcr.io/trailofbits/buttercup-cscope:main
|
||||
```
|
||||
|
||||
You should see the `buttercup-cscope` package [here](https://github.com/orgs/aixcc-finals/packages?visibility=private).
|
||||
|
||||
## Usage
|
||||
|
||||
Start up CRS.
|
||||
|
||||
```shell
|
||||
cd afc-crs-trail-of-bits/
|
||||
|
||||
docker compose up -d --build --remove-orphans
|
||||
|
||||
docker compose logs -f program-model
|
||||
```
|
||||
|
||||
Send task to CRS.
|
||||
|
||||
```shell
|
||||
git clone git@github.com:aixcc-finals/generate-challenge-task.git
|
||||
|
||||
cd generate-challenge-task/
|
||||
|
||||
# Libpng Challenge (c language)
|
||||
./generate-challenge-task.sh -p libpng -c 127.0.0.1:8000 -t "https://github.com/aixcc-finals/example-libpng.git" -b 0cc367aaeaac3f888f255cee5d394968996f736e -r 2c894c66108f0724331a9e5b4826e351bf2d094b
|
||||
./task_crs.sh
|
||||
|
||||
# Antlr4 Challenge (java language)
|
||||
./generate-challenge-task.sh -p antlr4-java -c 127.0.0.1:8000 -t "https://github.com/antlr/antlr4.git" -b master
|
||||
./task_crs.sh
|
||||
```
|
||||
|
||||
## API
|
||||
|
||||
See [api.md](api.md).
|
||||
|
||||
## Testing on Challenges
|
||||
|
||||
See [challenges.md](challenges.md).
|
||||
You should see the `buttercup-cscope` [package](https://github.com/orgs/trailofbits/packages)
|
||||
|
||||
## Development
|
||||
|
||||
Sync, reformat, lint, and test before committing changes to this directory:
|
||||
Sync, reformat, lint, and test before committing changes to this directory.
|
||||
|
||||
```shell
|
||||
just all
|
||||
```
|
||||
|
||||
Note: To run tests locally, you may need to run this beforehand
|
||||
|
||||
```shell
|
||||
sudo mount --bind ./crs_scratch /crs_scratch
|
||||
just download-kythe
|
||||
```
|
||||
|
||||
## JanusGraph Indexing References
|
||||
|
||||
* <https://user3141592.medium.com/single-vs-composite-indexes-in-relational-databases-58d0eb045cbe>
|
||||
* <https://docs.janusgraph.org/schema/index-management/index-performance/>
|
||||
* <https://docs.janusgraph.org/schema/schema-init-strategies/>
|
||||
* <https://docs.janusgraph.org/configs/configuration-reference/#schema>
|
||||
* <https://docs.janusgraph.org/v0.3/basics/schema/#:~:text=.util.UUID%20)-,Property%20Key%20Cardinality,all%20elements%20in%20the%20graph.>
|
||||
|
||||
## FAQs
|
||||
|
||||
* Why does this use a ubuntu base image?
|
||||
* Since Kythe uses OSS Fuzz to build and index the challenge source code, we have to use the same base image as ClusterFuzz.
|
||||
* Why does this use Python 3.10?
|
||||
* ClusterFuzz uses Python 3.10.
|
||||
|
||||
Submodule program-model/aixcc-kythe deleted from 2cddc663d1
@@ -1,25 +0,0 @@
|
||||
# Program Model API
|
||||
|
||||
Kythe [schema](https://kythe.io/docs/schema/).
|
||||
|
||||
## Getters
|
||||
|
||||
`get_function_body(source_path: Path | None, function_name: str) -> List[str]`: Get function body(s) by name.
|
||||
|
||||
`get_distinct_edge_types() -> List[str]`: Get distinct edge types.
|
||||
|
||||
`get_distinct_node_types() -> List[Node]`: Get distinct node types.
|
||||
|
||||
## Setters
|
||||
|
||||
`set_node_property(node: Node, property_name: str, property_value: str) -> bool`: Set a property on a node.
|
||||
|
||||
`set_edge_property(edge: Edge, property_name: str, property_value: str) -> bool`: Set a property on an edge.
|
||||
|
||||
## Example Usage
|
||||
|
||||
```shell
|
||||
cd afc-crs-trail-of-bits/program-model
|
||||
|
||||
uv run mock/api.py
|
||||
```
|
||||
@@ -1,3 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
protoc -Iaixcc-kythe/ --python_out=./src/buttercup/program_model/data --pyi_out=./src/buttercup/program_model/data aixcc-kythe/kythe/proto/storage.proto
|
||||
@@ -1,24 +0,0 @@
|
||||
# Challenges
|
||||
|
||||
Details of challenges tested.
|
||||
|
||||
## [libpng](https://github.com/aixcc-finals/example-libpng)
|
||||
|
||||
* 6 minutes to write graphml file
|
||||
|
||||
* 12 minutes to load into JanusGraph
|
||||
|
||||
* Graph dimensions:
|
||||
* Nodes: 291,910
|
||||
* Edges: 1,991,311
|
||||
|
||||
**TODO:** Reconcile the differences in the number of nodes and edges.
|
||||
|
||||
### Full-scan
|
||||
|
||||
* **base:** `2c894c66108f0724331a9e5b4826e351bf2d094b`
|
||||
|
||||
### Delta-scan
|
||||
|
||||
* **base:** `0cc367aaeaac3f888f255cee5d394968996f736e`
|
||||
* **delta:** `2c894c66108f0724331a9e5b4826e351bf2d094b`
|
||||
@@ -1,44 +0,0 @@
|
||||
host: 0.0.0.0
|
||||
port: 8182
|
||||
evaluationTimeout: 3600000
|
||||
channelizer: org.apache.tinkerpop.gremlin.server.channel.WebSocketChannelizer
|
||||
graphManager: org.janusgraph.graphdb.management.JanusGraphManager
|
||||
graphs: {
|
||||
#graph: conf/janusgraph-inmemory.properties
|
||||
graph: conf/janusgraph-cql.properties
|
||||
}
|
||||
scriptEngines: {
|
||||
gremlin-groovy: {
|
||||
plugins: { org.janusgraph.graphdb.tinkerpop.plugin.JanusGraphGremlinPlugin: {},
|
||||
org.apache.tinkerpop.gremlin.server.jsr223.GremlinServerGremlinPlugin: {},
|
||||
org.apache.tinkerpop.gremlin.tinkergraph.jsr223.TinkerGraphGremlinPlugin: {},
|
||||
org.apache.tinkerpop.gremlin.jsr223.ImportGremlinPlugin: {classImports: [java.lang.Math], methodImports: [java.lang.Math#*]},
|
||||
org.apache.tinkerpop.gremlin.jsr223.ScriptFileGremlinPlugin: {files: [scripts/empty-sample.groovy]}
|
||||
}
|
||||
}
|
||||
}
|
||||
# serializers:
|
||||
# - { className: org.apache.tinkerpop.gremlin.util.ser.GraphBinaryMessageSerializerV1, config: { ioRegistries: [org.janusgraph.graphdb.tinkerpop.JanusGraphIoRegistry] }}
|
||||
# - { className: org.apache.tinkerpop.gremlin.util.ser.GraphBinaryMessageSerializerV1, config: { serializeResultToString: true }}
|
||||
# - { className: org.apache.tinkerpop.gremlin.util.ser.GraphSONMessageSerializerV3, config: { ioRegistries: [org.janusgraph.graphdb.tinkerpop.JanusGraphIoRegistry] }}
|
||||
processors:
|
||||
- { className: org.apache.tinkerpop.gremlin.server.op.session.SessionOpProcessor, config: { sessionTimeout: 28800000 }}
|
||||
- { className: org.apache.tinkerpop.gremlin.server.op.traversal.TraversalOpProcessor, config: { cacheExpirationTime: 600000, cacheMaxSize: 1000 }}
|
||||
metrics: {
|
||||
consoleReporter: {enabled: true, interval: 180000},
|
||||
csvReporter: {enabled: true, interval: 180000, fileName: /tmp/gremlin-server-metrics.csv},
|
||||
jmxReporter: {enabled: true},
|
||||
slf4jReporter: {enabled: true, interval: 180000},
|
||||
graphiteReporter: {enabled: false, interval: 180000}}
|
||||
maxInitialLineLength: 4096
|
||||
maxHeaderSize: 8192
|
||||
maxChunkSize: 8192
|
||||
|
||||
# NOTE(Evan): We need to increase the maxContentLength to 2MB to allow for larger Kythe entries.
|
||||
# 2MB = 2 * 1024 * 1024
|
||||
maxContentLength: 2097152
|
||||
|
||||
maxAccumulationBufferComponents: 1024
|
||||
resultIterationBatchSize: 64
|
||||
writeBufferLowWaterMark: 32768
|
||||
writeBufferHighWaterMark: 65536
|
||||
@@ -1,414 +0,0 @@
|
||||
{
|
||||
"edgeLabels": [
|
||||
{
|
||||
"label": "/kythe/edge/aliases/root",
|
||||
"multiplicity": "MULTI",
|
||||
"unidirected": false
|
||||
},
|
||||
{
|
||||
"label": "/kythe/edge/aliases",
|
||||
"multiplicity": "MULTI",
|
||||
"unidirected": false
|
||||
},
|
||||
{
|
||||
"label": "/kythe/edge/childof/context",
|
||||
"multiplicity": "MULTI",
|
||||
"unidirected": false
|
||||
},
|
||||
{
|
||||
"label": "/kythe/edge/childof",
|
||||
"multiplicity": "MULTI",
|
||||
"unidirected": false
|
||||
},
|
||||
{
|
||||
"label": "/kythe/edge/completedby",
|
||||
"multiplicity": "MULTI",
|
||||
"unidirected": false
|
||||
},
|
||||
{
|
||||
"label": "/kythe/edge/defines/binding",
|
||||
"multiplicity": "MULTI",
|
||||
"unidirected": false
|
||||
},
|
||||
{
|
||||
"label": "/kythe/edge/defines",
|
||||
"multiplicity": "MULTI",
|
||||
"unidirected": false
|
||||
},
|
||||
{
|
||||
"label": "/kythe/edge/documents",
|
||||
"multiplicity": "MULTI",
|
||||
"unidirected": false
|
||||
},
|
||||
{
|
||||
"label": "/kythe/edge/extends/public",
|
||||
"multiplicity": "MULTI",
|
||||
"unidirected": false
|
||||
},
|
||||
{
|
||||
"label": "/kythe/edge/extends/protected",
|
||||
"multiplicity": "MULTI",
|
||||
"unidirected": false
|
||||
},
|
||||
{
|
||||
"label": "/kythe/edge/extends/private",
|
||||
"multiplicity": "MULTI",
|
||||
"unidirected": false
|
||||
},
|
||||
{
|
||||
"label": "/kythe/edge/influences",
|
||||
"multiplicity": "MULTI",
|
||||
"unidirected": false
|
||||
},
|
||||
{
|
||||
"label": "/kythe/edge/instantiates",
|
||||
"multiplicity": "MULTI",
|
||||
"unidirected": false
|
||||
},
|
||||
{
|
||||
"label": "/kythe/edge/overrides/root",
|
||||
"multiplicity": "MULTI",
|
||||
"unidirected": false
|
||||
},
|
||||
{
|
||||
"label": "/kythe/edge/overrides",
|
||||
"multiplicity": "MULTI",
|
||||
"unidirected": false
|
||||
},
|
||||
{
|
||||
"label": "/kythe/edge/param.0",
|
||||
"multiplicity": "MULTI",
|
||||
"unidirected": false
|
||||
},
|
||||
{
|
||||
"label": "/kythe/edge/param.1",
|
||||
"multiplicity": "MULTI",
|
||||
"unidirected": false
|
||||
},
|
||||
{
|
||||
"label": "/kythe/edge/param.2",
|
||||
"multiplicity": "MULTI",
|
||||
"unidirected": false
|
||||
},
|
||||
{
|
||||
"label": "/kythe/edge/param.3",
|
||||
"multiplicity": "MULTI",
|
||||
"unidirected": false
|
||||
},
|
||||
{
|
||||
"label": "/kythe/edge/param.4",
|
||||
"multiplicity": "MULTI",
|
||||
"unidirected": false
|
||||
},
|
||||
{
|
||||
"label": "/kythe/edge/param.5",
|
||||
"multiplicity": "MULTI",
|
||||
"unidirected": false
|
||||
},
|
||||
{
|
||||
"label": "/kythe/edge/param.6",
|
||||
"multiplicity": "MULTI",
|
||||
"unidirected": false
|
||||
},
|
||||
{
|
||||
"label": "/kythe/edge/param.7",
|
||||
"multiplicity": "MULTI",
|
||||
"unidirected": false
|
||||
},
|
||||
{
|
||||
"label": "/kythe/edge/param.8",
|
||||
"multiplicity": "MULTI",
|
||||
"unidirected": false
|
||||
},
|
||||
{
|
||||
"label": "/kythe/edge/param.9",
|
||||
"multiplicity": "MULTI",
|
||||
"unidirected": false
|
||||
},
|
||||
{
|
||||
"label": "/kythe/edge/param.10",
|
||||
"multiplicity": "MULTI",
|
||||
"unidirected": false
|
||||
},
|
||||
{
|
||||
"label": "/kythe/edge/param.11",
|
||||
"multiplicity": "MULTI",
|
||||
"unidirected": false
|
||||
},
|
||||
{
|
||||
"label": "/kythe/edge/param.12",
|
||||
"multiplicity": "MULTI",
|
||||
"unidirected": false
|
||||
},
|
||||
{
|
||||
"label": "/kythe/edge/param.13",
|
||||
"multiplicity": "MULTI",
|
||||
"unidirected": false
|
||||
},
|
||||
{
|
||||
"label": "/kythe/edge/param.14",
|
||||
"multiplicity": "MULTI",
|
||||
"unidirected": false
|
||||
},
|
||||
{
|
||||
"label": "/kythe/edge/param.15",
|
||||
"multiplicity": "MULTI",
|
||||
"unidirected": false
|
||||
},
|
||||
{
|
||||
"label": "/kythe/edge/param.16",
|
||||
"multiplicity": "MULTI",
|
||||
"unidirected": false
|
||||
},
|
||||
{
|
||||
"label": "/kythe/edge/ref/call/implicit",
|
||||
"multiplicity": "MULTI",
|
||||
"unidirected": false
|
||||
},
|
||||
{
|
||||
"label": "/kythe/edge/ref/call",
|
||||
"multiplicity": "MULTI",
|
||||
"unidirected": false
|
||||
},
|
||||
{
|
||||
"label": "/kythe/edge/ref/doc",
|
||||
"multiplicity": "MULTI",
|
||||
"unidirected": false
|
||||
},
|
||||
{
|
||||
"label": "/kythe/edge/ref/expands/transitive",
|
||||
"multiplicity": "MULTI",
|
||||
"unidirected": false
|
||||
},
|
||||
{
|
||||
"label": "/kythe/edge/ref/expands",
|
||||
"multiplicity": "MULTI",
|
||||
"unidirected": false
|
||||
},
|
||||
{
|
||||
"label": "/kythe/edge/ref/includes",
|
||||
"multiplicity": "MULTI",
|
||||
"unidirected": false
|
||||
},
|
||||
{
|
||||
"label": "/kythe/edge/ref/id",
|
||||
"multiplicity": "MULTI",
|
||||
"unidirected": false
|
||||
},
|
||||
{
|
||||
"label": "/kythe/edge/ref/implicit",
|
||||
"multiplicity": "MULTI",
|
||||
"unidirected": false
|
||||
},
|
||||
{
|
||||
"label": "/kythe/edge/ref/includes",
|
||||
"multiplicity": "MULTI",
|
||||
"unidirected": false
|
||||
},
|
||||
{
|
||||
"label": "/kythe/edge/ref/init/implicit",
|
||||
"multiplicity": "MULTI",
|
||||
"unidirected": false
|
||||
},
|
||||
{
|
||||
"label": "/kythe/edge/ref/init",
|
||||
"multiplicity": "MULTI",
|
||||
"unidirected": false
|
||||
},
|
||||
{
|
||||
"label": "/kythe/edge/ref/queries",
|
||||
"multiplicity": "MULTI",
|
||||
"unidirected": false
|
||||
},
|
||||
{
|
||||
"label": "/kythe/edge/ref/writes/implicit",
|
||||
"multiplicity": "MULTI",
|
||||
"unidirected": false
|
||||
},
|
||||
{
|
||||
"label": "/kythe/edge/ref/writes",
|
||||
"multiplicity": "MULTI",
|
||||
"unidirected": false
|
||||
},
|
||||
{
|
||||
"label": "/kythe/edge/ref",
|
||||
"multiplicity": "MULTI",
|
||||
"unidirected": false
|
||||
},
|
||||
{
|
||||
"label": "/kythe/edge/specializes",
|
||||
"multiplicity": "MULTI",
|
||||
"unidirected": false
|
||||
},
|
||||
{
|
||||
"label": "/kythe/edge/tparam.0",
|
||||
"multiplicity": "MULTI",
|
||||
"unidirected": false
|
||||
},
|
||||
{
|
||||
"label": "/kythe/edge/tparam.1",
|
||||
"multiplicity": "MULTI",
|
||||
"unidirected": false
|
||||
},
|
||||
{
|
||||
"label": "/kythe/edge/tparam.2",
|
||||
"multiplicity": "MULTI",
|
||||
"unidirected": false
|
||||
},
|
||||
{
|
||||
"label": "/kythe/edge/tparam.3",
|
||||
"multiplicity": "MULTI",
|
||||
"unidirected": false
|
||||
},
|
||||
{
|
||||
"label": "/kythe/edge/tparam.4",
|
||||
"multiplicity": "MULTI",
|
||||
"unidirected": false
|
||||
},
|
||||
{
|
||||
"label": "/kythe/edge/tparam.5",
|
||||
"multiplicity": "MULTI",
|
||||
"unidirected": false
|
||||
},
|
||||
{
|
||||
"label": "/kythe/edge/tparam.6",
|
||||
"multiplicity": "MULTI",
|
||||
"unidirected": false
|
||||
},
|
||||
{
|
||||
"label": "/kythe/edge/tparam.7",
|
||||
"multiplicity": "MULTI",
|
||||
"unidirected": false
|
||||
},
|
||||
{
|
||||
"label": "/kythe/edge/tparam.8",
|
||||
"multiplicity": "MULTI",
|
||||
"unidirected": false
|
||||
},
|
||||
{
|
||||
"label": "/kythe/edge/tparam.9",
|
||||
"multiplicity": "MULTI",
|
||||
"unidirected": false
|
||||
},
|
||||
{
|
||||
"label": "/kythe/edge/typed",
|
||||
"multiplicity": "MULTI",
|
||||
"unidirected": false
|
||||
},
|
||||
{
|
||||
"label": "/kythe/edge/undefines",
|
||||
"multiplicity": "MULTI",
|
||||
"unidirected": false
|
||||
}
|
||||
],
|
||||
"propertyKeys": [
|
||||
{
|
||||
"key": "task_id",
|
||||
"className": "java.lang.String",
|
||||
"cardinality": "SINGLE"
|
||||
},
|
||||
{
|
||||
"key": "corpus",
|
||||
"className": "java.lang.String",
|
||||
"cardinality": "SINGLE"
|
||||
},
|
||||
{
|
||||
"key": "language",
|
||||
"className": "java.lang.String",
|
||||
"cardinality": "SINGLE"
|
||||
},
|
||||
{
|
||||
"key": "path",
|
||||
"className": "java.lang.String",
|
||||
"cardinality": "SINGLE"
|
||||
},
|
||||
{
|
||||
"key": "root",
|
||||
"className": "java.lang.String",
|
||||
"cardinality": "SINGLE"
|
||||
},
|
||||
{
|
||||
"key": "signature",
|
||||
"className": "java.lang.String",
|
||||
"cardinality": "SINGLE"
|
||||
},
|
||||
{
|
||||
"key": "/kythe/code",
|
||||
"className": "java.lang.String",
|
||||
"cardinality": "SINGLE"
|
||||
},
|
||||
{
|
||||
"key": "/kythe/visibility",
|
||||
"className": "java.lang.String",
|
||||
"cardinality": "SINGLE"
|
||||
},
|
||||
{
|
||||
"key": "/kythe/complete",
|
||||
"className": "java.lang.String",
|
||||
"cardinality": "SINGLE"
|
||||
},
|
||||
{
|
||||
"key": "/kythe/tag/static",
|
||||
"className": "java.lang.String",
|
||||
"cardinality": "SINGLE"
|
||||
},
|
||||
{
|
||||
"key": "/kythe/tag/deprecated",
|
||||
"className": "java.lang.String",
|
||||
"cardinality": "SINGLE"
|
||||
},
|
||||
{
|
||||
"key": "/kythe/text",
|
||||
"className": "java.lang.String",
|
||||
"cardinality": "SINGLE"
|
||||
},
|
||||
{
|
||||
"key": "/kythe/node/kind",
|
||||
"className": "java.lang.String",
|
||||
"cardinality": "SINGLE"
|
||||
},
|
||||
{
|
||||
"key": "/kythe/loc/start",
|
||||
"className": "java.lang.String",
|
||||
"cardinality": "SINGLE"
|
||||
},
|
||||
{
|
||||
"key": "/kythe/subkind",
|
||||
"className": "java.lang.String",
|
||||
"cardinality": "SINGLE"
|
||||
},
|
||||
{
|
||||
"key": "/kythe/param/default",
|
||||
"className": "java.lang.String",
|
||||
"cardinality": "SINGLE"
|
||||
},
|
||||
{
|
||||
"key": "/kythe/loc/end",
|
||||
"className": "java.lang.String",
|
||||
"cardinality": "SINGLE"
|
||||
}
|
||||
],
|
||||
"compositeIndexes": [
|
||||
{
|
||||
"name": "kindCompositeIndex",
|
||||
"typeClass": "org.apache.tinkerpop.gremlin.structure.Vertex",
|
||||
"keys": [
|
||||
{
|
||||
"propertyKey": "/kythe/node/kind"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"mixedIndexes": [
|
||||
{
|
||||
"name": "codeMixedIndex",
|
||||
"typeClass": "org.apache.tinkerpop.gremlin.structure.Vertex",
|
||||
"indexBackend": "search",
|
||||
"keys": [
|
||||
{
|
||||
"propertyKey": "/kythe/code"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -21,22 +21,3 @@ test:
|
||||
|
||||
# To run tests
|
||||
uv run pytest -svv --runintegration
|
||||
|
||||
install-gh:
|
||||
(type -p wget >/dev/null || (sudo apt update && sudo apt-get install wget -y)) \
|
||||
&& sudo mkdir -p -m 755 /etc/apt/keyrings \
|
||||
&& out=$(mktemp) && wget -nv -O$out https://cli.github.com/packages/githubcli-archive-keyring.gpg \
|
||||
&& cat $out | sudo tee /etc/apt/keyrings/githubcli-archive-keyring.gpg > /dev/null \
|
||||
&& sudo chmod go+r /etc/apt/keyrings/githubcli-archive-keyring.gpg \
|
||||
&& echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null \
|
||||
&& sudo apt update \
|
||||
&& sudo apt install gh -y
|
||||
|
||||
download-kythe:
|
||||
mkdir -p scripts/gzs/kythe
|
||||
docker pull ghcr.io/aixcc-finals/buttercup-kythe:main
|
||||
docker create --name temp-kythe ghcr.io/aixcc-finals/buttercup-kythe:main
|
||||
docker cp temp-kythe:/kythe-v0.0.67.tar.gz scripts/gzs/
|
||||
docker rm temp-kythe
|
||||
tar xvzf scripts/gzs/kythe-v0.0.67.tar.gz -C scripts/gzs/kythe/ --strip-components=1
|
||||
rm scripts/gzs/kythe-v0.0.67.tar.gz
|
||||
|
||||
@@ -1,17 +0,0 @@
|
||||
FROM gcr.io/oss-fuzz-base/base-clang
|
||||
|
||||
ENV KYTHE_VERSION=v0.0.67
|
||||
RUN apt update && apt -y install bison flex uuid-dev asciidoc graphviz source-highlight
|
||||
RUN apt -y install git
|
||||
COPY program-model/aixcc-kythe /kythe
|
||||
RUN apt install apt-transport-https curl gnupg -y
|
||||
RUN curl -fsSL https://bazel.build/bazel-release.pub.gpg | gpg --dearmor >bazel-archive-keyring.gpg
|
||||
RUN mv bazel-archive-keyring.gpg /usr/share/keyrings
|
||||
RUN echo "deb [arch=amd64 signed-by=/usr/share/keyrings/bazel-archive-keyring.gpg] https://storage.googleapis.com/bazel-apt stable jdk1.8" | tee /etc/apt/sources.list.d/bazel.list
|
||||
RUN apt update && apt -y install bazel-7.1.0 build-essential
|
||||
|
||||
COPY program-model/kythe_builder/entrypoint.sh /entrypoint.sh
|
||||
RUN chmod +x /entrypoint.sh
|
||||
WORKDIR /kythe
|
||||
|
||||
ENTRYPOINT ["/entrypoint.sh"]
|
||||
@@ -1,8 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -euxo pipefail
|
||||
|
||||
cp -r ./ $OUT/kythe
|
||||
pushd $OUT/kythe
|
||||
bazel-7.1.0 build //kythe/release --sandbox_debug
|
||||
cp bazel-bin/kythe/release/kythe-v0.0.67.tar.gz ./
|
||||
@@ -1,21 +0,0 @@
|
||||
// We need to configure the console to have state:
|
||||
// https://tinkerpop.apache.org/docs/current/reference/#console-sessions
|
||||
:remote connect tinkerpop.server conf/remote.yaml session
|
||||
:remote console
|
||||
|
||||
// 1 hour in milliseconds
|
||||
:remote config timeout 3600000
|
||||
|
||||
println("Printing schema...")
|
||||
|
||||
mgmt = graph.openManagement()
|
||||
mgmt.printSchema()
|
||||
|
||||
println("Starting graph creation...")
|
||||
|
||||
g.io('/crs_scratch/graph.xml').read().iterate()
|
||||
|
||||
println("Graph creation complete.")
|
||||
|
||||
:remote console
|
||||
:exit
|
||||
@@ -1,72 +0,0 @@
|
||||
from buttercup.program_model.api import Graph
|
||||
from pathlib import Path
|
||||
import logging
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
logging.basicConfig(
|
||||
format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
|
||||
level=logging.INFO,
|
||||
handlers=[
|
||||
logging.StreamHandler(),
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
def main():
|
||||
with Graph(url="ws://localhost:8182/gremlin") as graph:
|
||||
# # Get examples of each type of node
|
||||
# logger.info("Getting examples of each type of node")
|
||||
# nodes = graph.get_distinct_node_types()
|
||||
# logger.info(f"Found {len(nodes)} distinct node types")
|
||||
# for node in nodes:
|
||||
# logger.info(node)
|
||||
# logger.info("=" * 80)
|
||||
|
||||
# # Get examples of each type of edge
|
||||
# logger.info("Getting examples of each type of edge")
|
||||
# edges = graph.get_distinct_edge_types()
|
||||
# logger.info(f"Found {len(edges)} distinct edge types")
|
||||
# for edge in edges:
|
||||
# logger.info(edge)
|
||||
# logger.info("=" * 80)
|
||||
|
||||
function_name = "png_handle_iCCP"
|
||||
file_path = Path("pngrutil.c")
|
||||
|
||||
logger.info(f"Getting function bodies with name: {function_name}")
|
||||
bodies = graph.get_function_body(function_name)
|
||||
if not bodies:
|
||||
logger.info("No bodies found")
|
||||
for body in bodies:
|
||||
logger.info(body)
|
||||
logger.info("=" * 80)
|
||||
|
||||
logger.info(
|
||||
f"Getting function bodies with name '{function_name}' from file path '{file_path}'"
|
||||
)
|
||||
bodies = graph.get_function_body(
|
||||
function_name=function_name, source_path=file_path
|
||||
)
|
||||
if not bodies:
|
||||
logger.info("No bodies found")
|
||||
for body in bodies:
|
||||
logger.info(body)
|
||||
logger.info("=" * 80)
|
||||
|
||||
file_path = Path("pngmem.c")
|
||||
|
||||
logger.info(
|
||||
f"Getting function bodies with name '{function_name}' from file path '{file_path}'"
|
||||
)
|
||||
bodies = graph.get_function_body(
|
||||
function_name=function_name, source_path=file_path
|
||||
)
|
||||
if not bodies:
|
||||
logger.info("No bodies found")
|
||||
for body in bodies:
|
||||
logger.info(body)
|
||||
logger.info("=" * 80)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,9 +0,0 @@
|
||||
task {
|
||||
message_id: "my-message-id"
|
||||
message_time: 123
|
||||
task_type: TASK_TYPE_FULL
|
||||
task_id: "example-libpng"
|
||||
sources {
|
||||
url: "https://github.com/buttercup-project/buttercup"
|
||||
}
|
||||
}
|
||||
@@ -1,30 +0,0 @@
|
||||
import argparse
|
||||
from redis import Redis
|
||||
from buttercup.common.queues import QueueFactory, QueueNames
|
||||
from buttercup.common.datastructures.msg_pb2 import IndexRequest
|
||||
|
||||
|
||||
def main():
|
||||
prsr = argparse.ArgumentParser("trigger Program Model manually")
|
||||
prsr.add_argument("--redis_url", default="redis://127.0.0.1:6379")
|
||||
prsr.add_argument("--build_type", required=True)
|
||||
prsr.add_argument("--package_name", required=True)
|
||||
prsr.add_argument("--sanitizer", required=True)
|
||||
prsr.add_argument("--task_dir", required=True)
|
||||
prsr.add_argument("--task_id", required=True)
|
||||
args = prsr.parse_args()
|
||||
|
||||
redis = Redis.from_url(args.redis_url)
|
||||
queue = QueueFactory(redis).create(QueueNames.INDEX)
|
||||
req = IndexRequest(
|
||||
build_type=args.build_type,
|
||||
package_name=args.package_name,
|
||||
sanitizer=args.sanitizer,
|
||||
task_dir=args.task_dir,
|
||||
task_id=args.task_id,
|
||||
)
|
||||
queue.push(req)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -6,7 +6,6 @@ readme = "README.md"
|
||||
authors = [{ name = "Trail of Bits", email = "opensource@trailofbits" }]
|
||||
requires-python = ">=3.10,<3.13"
|
||||
dependencies = [
|
||||
"gremlinpython==3.7.3",
|
||||
"common",
|
||||
"protobuf (>=3.20, <=3.20.3)",
|
||||
"tree-sitter>=0.24.0",
|
||||
@@ -28,12 +27,6 @@ requires = ["hatchling"]
|
||||
build-backend = "hatchling.build"
|
||||
|
||||
|
||||
[tool.ruff]
|
||||
extend-exclude = [
|
||||
"aixcc-kythe",
|
||||
"src/buttercup/program_model/data/kythe/proto/",
|
||||
]
|
||||
|
||||
[dependency-groups]
|
||||
dev = ["pytest>=8.3.4", "ruff>=0.9.2", "mypy>=1.15.0"]
|
||||
|
||||
|
||||
@@ -1,9 +0,0 @@
|
||||
ARG BASE_IMAGE=myorg/myapp:latest
|
||||
FROM ${BASE_IMAGE}
|
||||
|
||||
ENV FUZZING_LANGUAGE=c++
|
||||
|
||||
COPY ./compile_and_extract /usr/local/bin/compile_and_extract
|
||||
RUN chmod +x /usr/local/bin/compile_and_extract
|
||||
COPY gzs/kythe/ ${SRC}/kythe
|
||||
ENV KYTHE_RELEASE_DIR=${SRC}/kythe/
|
||||
@@ -1,12 +0,0 @@
|
||||
#!/bin/bash -eu
|
||||
|
||||
|
||||
mkdir -p /tmp/compdbs
|
||||
export KYTHE_ROOT_DIRECTORY=$SRC
|
||||
export CFLAGS="$CFLAGS -gen-cdb-fragment-path /tmp/compdbs"
|
||||
compile
|
||||
echo "[" > /tmp/compdbs.json
|
||||
cat /tmp/compdbs/*.json >> /tmp/compdbs.json
|
||||
echo "{}]" >> /tmp/compdbs.json
|
||||
|
||||
$KYTHE_RELEASE_DIR/tools/runextractor compdb -path /tmp/compdbs.json -extractor $KYTHE_RELEASE_DIR/extractors/cxx_extractor || true
|
||||
@@ -36,26 +36,18 @@ def main() -> None:
|
||||
sleep_time=command.sleep_time,
|
||||
redis=redis,
|
||||
wdir=settings.scratch_dir,
|
||||
script_dir=command.script_dir,
|
||||
kythe_dir=command.kythe_dir,
|
||||
python=command.python,
|
||||
allow_pull=command.allow_pull,
|
||||
base_image_url=command.base_image_url,
|
||||
graphdb_url=settings.graphdb_url,
|
||||
graphdb_enabled=settings.graphdb_enabled,
|
||||
) as program_model:
|
||||
program_model.serve()
|
||||
elif isinstance(command, ProcessCommand):
|
||||
task = prepare_task(command) # type: ignore[unreachable]
|
||||
with ProgramModel(
|
||||
wdir=settings.scratch_dir,
|
||||
script_dir=command.script_dir,
|
||||
kythe_dir=command.kythe_dir,
|
||||
python=command.python,
|
||||
allow_pull=command.allow_pull,
|
||||
base_image_url=command.base_image_url,
|
||||
graphdb_url=settings.graphdb_url,
|
||||
graphdb_enabled=settings.graphdb_enabled,
|
||||
) as program_model:
|
||||
program_model.process_task(task)
|
||||
|
||||
|
||||
@@ -1,5 +0,0 @@
|
||||
from buttercup.program_model.api.graph import Graph
|
||||
|
||||
__all__ = [
|
||||
"Graph",
|
||||
]
|
||||
|
||||
@@ -1,212 +0,0 @@
|
||||
from buttercup.program_model.graph import Node, Edge, decode_value, encode_value
|
||||
import logging
|
||||
from dataclasses import dataclass
|
||||
from typing import List, Dict, Any
|
||||
from pathlib import Path
|
||||
from gremlin_python.structure.graph import Graph as GremlinGraph
|
||||
from gremlin_python.process.traversal import T, TextP
|
||||
from gremlin_python.driver.driver_remote_connection import (
|
||||
DriverRemoteConnection,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class Graph:
|
||||
"""Program Model Graph API class."""
|
||||
|
||||
url: str = "ws://graphdb:8182/gremlin"
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
self.graph = GremlinGraph()
|
||||
self.connection = DriverRemoteConnection(self.url, "g")
|
||||
self.g = self.graph.traversal().withRemote(self.connection)
|
||||
|
||||
def __enter__(self): # type: ignore[no-untyped-def]
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None:
|
||||
self.connection.close()
|
||||
|
||||
def _decode_properties(self, obj: dict) -> dict:
|
||||
"""Decode properties from the graph."""
|
||||
properties = {}
|
||||
for k, v in obj.items():
|
||||
if k != T.id and k != T.label:
|
||||
properties[k] = decode_value(v).decode("utf-8")
|
||||
return properties
|
||||
|
||||
def _decode_node(self, node: dict) -> Node:
|
||||
"""Decode a node from the graph."""
|
||||
return Node(
|
||||
id=node[T.id],
|
||||
label=node[T.label],
|
||||
properties=self._decode_properties(node),
|
||||
)
|
||||
|
||||
def _decode_edge(self, edge: dict) -> Edge:
|
||||
"""Decode an edge from the graph."""
|
||||
return Edge(
|
||||
id=edge[T.id],
|
||||
source_id=edge[T.outV],
|
||||
target_id=edge[T.inV],
|
||||
properties=self._decode_properties(edge),
|
||||
)
|
||||
|
||||
def get_distinct_node_types(self) -> List[Node]:
|
||||
"""Get distinct node types."""
|
||||
nodes: List[Node] = []
|
||||
for k in (
|
||||
self.g.V()
|
||||
.has("/kythe/node/kind")
|
||||
.values("/kythe/node/kind")
|
||||
.dedup()
|
||||
.toList()
|
||||
):
|
||||
n = self.g.V().has("/kythe/node/kind", k).limit(1).elementMap().toList()[0]
|
||||
nodes.append(self._decode_node(n))
|
||||
return nodes
|
||||
|
||||
def get_distinct_edge_types(self) -> List[str]:
|
||||
"""Get distinct edge types."""
|
||||
edges: List[str] = []
|
||||
for l in self.g.E().label().dedup().toList(): # noqa: E741
|
||||
edges.append(l)
|
||||
return edges
|
||||
|
||||
def _get_function_nodes(
|
||||
self, function_name: str, source_path: str | None = None
|
||||
) -> Any:
|
||||
"""Retrieves all function nodes matching the given function name and optionally source path."""
|
||||
if source_path is None:
|
||||
return (
|
||||
self.g.V()
|
||||
.has("/kythe/node/kind", encode_value(b"function"))
|
||||
.has(
|
||||
"/kythe/code",
|
||||
TextP.containing(function_name),
|
||||
)
|
||||
.elementMap()
|
||||
.toList()
|
||||
)
|
||||
else:
|
||||
return (
|
||||
self.g.V()
|
||||
.has("/kythe/node/kind", encode_value(b"function"))
|
||||
.has(
|
||||
"/kythe/code",
|
||||
TextP.containing(function_name),
|
||||
)
|
||||
.has("path", source_path)
|
||||
.elementMap()
|
||||
.toList()
|
||||
)
|
||||
|
||||
def _get_node_anchors(self, node_id: str) -> Any:
|
||||
"""Get all anchors for a given node."""
|
||||
return self.g.V(node_id).in_("/kythe/edge/defines").elementMap().toList()
|
||||
|
||||
def _get_function_node_body(self, function_node: Dict) -> List[str]:
|
||||
"""Get the body of a function node."""
|
||||
|
||||
# Get anchor nodes for this function node
|
||||
anchors = self._get_node_anchors(function_node[T.id])
|
||||
logger.debug(f"Found {len(anchors)} anchors")
|
||||
|
||||
files: List[Dict] = []
|
||||
|
||||
# For each anchor found
|
||||
for a in anchors:
|
||||
logger.debug(f"Anchor: {a}")
|
||||
|
||||
# Get start and end bytes of the function
|
||||
a_plain = self._decode_node(a)
|
||||
start_str = a_plain.properties.get("/kythe/loc/start")
|
||||
if start_str is None:
|
||||
logger.warning(f"No start byte offset found for anchor: {a}")
|
||||
continue
|
||||
start = int(start_str)
|
||||
end_str = a_plain.properties.get("/kythe/loc/end")
|
||||
if end_str is None:
|
||||
logger.warning(f"No end byte offset found for anchor: {a}")
|
||||
continue
|
||||
end = int(end_str)
|
||||
logger.debug(f"Start byte offset: {start}, End byte offset: {end}")
|
||||
|
||||
# Get the file nodes matching the anchor's file path
|
||||
file_path = a_plain.properties.get("path")
|
||||
if file_path is None:
|
||||
logger.warning(f"No file path found for anchor: {a}")
|
||||
continue
|
||||
files.extend(
|
||||
self.g.V()
|
||||
.has("/kythe/node/kind", encode_value(b"file"))
|
||||
.has("path", encode_value(file_path.encode("utf-8")))
|
||||
.elementMap()
|
||||
.toList()
|
||||
)
|
||||
logger.debug(f"Found {len(files)} files")
|
||||
|
||||
bodies: List[str] = []
|
||||
|
||||
# For each file found, return the function body
|
||||
for fn in files:
|
||||
fn_plain = self._decode_node(fn)
|
||||
fn_plain_text = fn_plain.properties.get("/kythe/text")
|
||||
if fn_plain_text is None:
|
||||
logger.warning(f"No text found for function: {fn}")
|
||||
continue
|
||||
bodies.append(fn_plain_text[start:end])
|
||||
|
||||
return bodies
|
||||
|
||||
def get_function_body(
|
||||
self, function_name: str, source_path: Path | None = None
|
||||
) -> List[str]:
|
||||
"""Get function bodies by name."""
|
||||
|
||||
function_name_encode = encode_value(function_name.encode("utf-8"))
|
||||
source_path_encode = (
|
||||
encode_value(str(source_path).encode("utf-8")) if source_path else None
|
||||
)
|
||||
|
||||
if source_path is not None:
|
||||
logger.debug(
|
||||
f"Searching for functions with name: {function_name} from file path: {source_path}"
|
||||
)
|
||||
logger.debug(f"Encoded source path: {source_path_encode}")
|
||||
else:
|
||||
logger.debug(f"Searching for functions with name: {function_name}")
|
||||
logger.debug(f"Encoded function name: {function_name_encode}")
|
||||
|
||||
# Retrieve all functions matching this function name (and optionally source path)
|
||||
functions = self._get_function_nodes(
|
||||
function_name=function_name_encode, source_path=source_path_encode
|
||||
)
|
||||
logger.debug(f"Found {len(functions)} functions")
|
||||
|
||||
bodies: List[str] = []
|
||||
|
||||
# For each function found
|
||||
for f in functions:
|
||||
logger.debug(f"Function: {f}")
|
||||
bodies.extend(self._get_function_node_body(f))
|
||||
|
||||
return bodies
|
||||
|
||||
# TODO(Evan)
|
||||
def set_node_property(
|
||||
self, node: Node, property_name: str, property_value: str
|
||||
) -> bool:
|
||||
"""Set a property on a node."""
|
||||
logger.error("Not implemented")
|
||||
return False
|
||||
|
||||
# TODO(Evan)
|
||||
def set_edge_property(
|
||||
self, edge: Edge, property_name: str, property_value: str
|
||||
) -> bool:
|
||||
"""Set a property on an edge."""
|
||||
logger.error("Not implemented")
|
||||
return False
|
||||
@@ -1,52 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
# Generated by the protocol buffer compiler. DO NOT EDIT!
|
||||
# source: kythe/proto/storage.proto
|
||||
"""Generated protocol buffer code."""
|
||||
from google.protobuf.internal import builder as _builder
|
||||
from google.protobuf import descriptor as _descriptor
|
||||
from google.protobuf import descriptor_pool as _descriptor_pool
|
||||
from google.protobuf import symbol_database as _symbol_database
|
||||
# @@protoc_insertion_point(imports)
|
||||
|
||||
_sym_db = _symbol_database.Default()
|
||||
|
||||
|
||||
|
||||
|
||||
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x19kythe/proto/storage.proto\x12\x0bkythe.proto\"X\n\x05VName\x12\x11\n\tsignature\x18\x01 \x01(\t\x12\x0e\n\x06\x63orpus\x18\x02 \x01(\t\x12\x0c\n\x04root\x18\x03 \x01(\t\x12\x0c\n\x04path\x18\x04 \x01(\t\x12\x10\n\x08language\x18\x05 \x01(\t\"\\\n\tVNameMask\x12\x11\n\tsignature\x18\x01 \x01(\x08\x12\x0e\n\x06\x63orpus\x18\x02 \x01(\x08\x12\x0c\n\x04root\x18\x03 \x01(\x08\x12\x0c\n\x04path\x18\x04 \x01(\x08\x12\x10\n\x08language\x18\x05 \x01(\x08\"\x89\x01\n\x05\x45ntry\x12\"\n\x06source\x18\x01 \x01(\x0b\x32\x12.kythe.proto.VName\x12\x11\n\tedge_kind\x18\x02 \x01(\t\x12\"\n\x06target\x18\x03 \x01(\x0b\x32\x12.kythe.proto.VName\x12\x11\n\tfact_name\x18\x04 \x01(\t\x12\x12\n\nfact_value\x18\x05 \x01(\x0c\".\n\x07\x45ntries\x12#\n\x07\x65ntries\x18\x01 \x03(\x0b\x32\x12.kythe.proto.Entry\"D\n\x0bReadRequest\x12\"\n\x06source\x18\x01 \x01(\x0b\x32\x12.kythe.proto.VName\x12\x11\n\tedge_kind\x18\x02 \x01(\t\"\xcc\x01\n\x0cWriteRequest\x12\"\n\x06source\x18\x01 \x01(\x0b\x32\x12.kythe.proto.VName\x12\x30\n\x06update\x18\x02 \x03(\x0b\x32 .kythe.proto.WriteRequest.Update\x1a\x66\n\x06Update\x12\x11\n\tedge_kind\x18\x01 \x01(\t\x12\"\n\x06target\x18\x02 \x01(\x0b\x32\x12.kythe.proto.VName\x12\x11\n\tfact_name\x18\x03 \x01(\t\x12\x12\n\nfact_value\x18\x04 \x01(\x0c\"\x0c\n\nWriteReply\"Y\n\x0bScanRequest\x12\"\n\x06target\x18\x01 \x01(\x0b\x32\x12.kythe.proto.VName\x12\x11\n\tedge_kind\x18\x02 \x01(\t\x12\x13\n\x0b\x66\x61\x63t_prefix\x18\x03 \x01(\t\"-\n\x0c\x43ountRequest\x12\r\n\x05index\x18\x01 \x01(\x03\x12\x0e\n\x06shards\x18\x02 \x01(\x03\"\x1d\n\nCountReply\x12\x0f\n\x07\x65ntries\x18\x01 \x01(\x03\"-\n\x0cShardRequest\x12\r\n\x05index\x18\x01 \x01(\x03\x12\x0e\n\x06shards\x18\x02 \x01(\x03\"N\n\x10VNameRewriteRule\x12\x0f\n\x07pattern\x18\x01 \x01(\t\x12)\n\x06v_name\x18\x02 \x01(\x0b\x32\x12.kythe.proto.VNameR\x05vname\"@\n\x11VNameRewriteRules\x12+\n\x04rule\x18\x01 \x03(\x0b\x32\x1d.kythe.proto.VNameRewriteRuleBH\n\x1f\x63om.google.devtools.kythe.protoZ%kythe.io/kythe/proto/storage_go_protob\x06proto3')
|
||||
|
||||
_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals())
|
||||
_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'kythe.proto.storage_pb2', globals())
|
||||
if _descriptor._USE_C_DESCRIPTORS == False:
|
||||
|
||||
DESCRIPTOR._options = None
|
||||
DESCRIPTOR._serialized_options = b'\n\037com.google.devtools.kythe.protoZ%kythe.io/kythe/proto/storage_go_proto'
|
||||
_VNAME._serialized_start=42
|
||||
_VNAME._serialized_end=130
|
||||
_VNAMEMASK._serialized_start=132
|
||||
_VNAMEMASK._serialized_end=224
|
||||
_ENTRY._serialized_start=227
|
||||
_ENTRY._serialized_end=364
|
||||
_ENTRIES._serialized_start=366
|
||||
_ENTRIES._serialized_end=412
|
||||
_READREQUEST._serialized_start=414
|
||||
_READREQUEST._serialized_end=482
|
||||
_WRITEREQUEST._serialized_start=485
|
||||
_WRITEREQUEST._serialized_end=689
|
||||
_WRITEREQUEST_UPDATE._serialized_start=587
|
||||
_WRITEREQUEST_UPDATE._serialized_end=689
|
||||
_WRITEREPLY._serialized_start=691
|
||||
_WRITEREPLY._serialized_end=703
|
||||
_SCANREQUEST._serialized_start=705
|
||||
_SCANREQUEST._serialized_end=794
|
||||
_COUNTREQUEST._serialized_start=796
|
||||
_COUNTREQUEST._serialized_end=841
|
||||
_COUNTREPLY._serialized_start=843
|
||||
_COUNTREPLY._serialized_end=872
|
||||
_SHARDREQUEST._serialized_start=874
|
||||
_SHARDREQUEST._serialized_end=919
|
||||
_VNAMEREWRITERULE._serialized_start=921
|
||||
_VNAMEREWRITERULE._serialized_end=999
|
||||
_VNAMEREWRITERULES._serialized_start=1001
|
||||
_VNAMEREWRITERULES._serialized_end=1065
|
||||
# @@protoc_insertion_point(module_scope)
|
||||
@@ -1,131 +0,0 @@
|
||||
from google.protobuf.internal import containers as _containers
|
||||
from google.protobuf import descriptor as _descriptor
|
||||
from google.protobuf import message as _message
|
||||
from typing import ClassVar as _ClassVar, Iterable as _Iterable, Mapping as _Mapping, Optional as _Optional, Union as _Union
|
||||
|
||||
DESCRIPTOR: _descriptor.FileDescriptor
|
||||
|
||||
class CountReply(_message.Message):
|
||||
__slots__ = ["entries"]
|
||||
ENTRIES_FIELD_NUMBER: _ClassVar[int]
|
||||
entries: int
|
||||
def __init__(self, entries: _Optional[int] = ...) -> None: ...
|
||||
|
||||
class CountRequest(_message.Message):
|
||||
__slots__ = ["index", "shards"]
|
||||
INDEX_FIELD_NUMBER: _ClassVar[int]
|
||||
SHARDS_FIELD_NUMBER: _ClassVar[int]
|
||||
index: int
|
||||
shards: int
|
||||
def __init__(self, index: _Optional[int] = ..., shards: _Optional[int] = ...) -> None: ...
|
||||
|
||||
class Entries(_message.Message):
|
||||
__slots__ = ["entries"]
|
||||
ENTRIES_FIELD_NUMBER: _ClassVar[int]
|
||||
entries: _containers.RepeatedCompositeFieldContainer[Entry]
|
||||
def __init__(self, entries: _Optional[_Iterable[_Union[Entry, _Mapping]]] = ...) -> None: ...
|
||||
|
||||
class Entry(_message.Message):
|
||||
__slots__ = ["edge_kind", "fact_name", "fact_value", "source", "target"]
|
||||
EDGE_KIND_FIELD_NUMBER: _ClassVar[int]
|
||||
FACT_NAME_FIELD_NUMBER: _ClassVar[int]
|
||||
FACT_VALUE_FIELD_NUMBER: _ClassVar[int]
|
||||
SOURCE_FIELD_NUMBER: _ClassVar[int]
|
||||
TARGET_FIELD_NUMBER: _ClassVar[int]
|
||||
edge_kind: str
|
||||
fact_name: str
|
||||
fact_value: bytes
|
||||
source: VName
|
||||
target: VName
|
||||
def __init__(self, source: _Optional[_Union[VName, _Mapping]] = ..., edge_kind: _Optional[str] = ..., target: _Optional[_Union[VName, _Mapping]] = ..., fact_name: _Optional[str] = ..., fact_value: _Optional[bytes] = ...) -> None: ...
|
||||
|
||||
class ReadRequest(_message.Message):
|
||||
__slots__ = ["edge_kind", "source"]
|
||||
EDGE_KIND_FIELD_NUMBER: _ClassVar[int]
|
||||
SOURCE_FIELD_NUMBER: _ClassVar[int]
|
||||
edge_kind: str
|
||||
source: VName
|
||||
def __init__(self, source: _Optional[_Union[VName, _Mapping]] = ..., edge_kind: _Optional[str] = ...) -> None: ...
|
||||
|
||||
class ScanRequest(_message.Message):
|
||||
__slots__ = ["edge_kind", "fact_prefix", "target"]
|
||||
EDGE_KIND_FIELD_NUMBER: _ClassVar[int]
|
||||
FACT_PREFIX_FIELD_NUMBER: _ClassVar[int]
|
||||
TARGET_FIELD_NUMBER: _ClassVar[int]
|
||||
edge_kind: str
|
||||
fact_prefix: str
|
||||
target: VName
|
||||
def __init__(self, target: _Optional[_Union[VName, _Mapping]] = ..., edge_kind: _Optional[str] = ..., fact_prefix: _Optional[str] = ...) -> None: ...
|
||||
|
||||
class ShardRequest(_message.Message):
|
||||
__slots__ = ["index", "shards"]
|
||||
INDEX_FIELD_NUMBER: _ClassVar[int]
|
||||
SHARDS_FIELD_NUMBER: _ClassVar[int]
|
||||
index: int
|
||||
shards: int
|
||||
def __init__(self, index: _Optional[int] = ..., shards: _Optional[int] = ...) -> None: ...
|
||||
|
||||
class VName(_message.Message):
|
||||
__slots__ = ["corpus", "language", "path", "root", "signature"]
|
||||
CORPUS_FIELD_NUMBER: _ClassVar[int]
|
||||
LANGUAGE_FIELD_NUMBER: _ClassVar[int]
|
||||
PATH_FIELD_NUMBER: _ClassVar[int]
|
||||
ROOT_FIELD_NUMBER: _ClassVar[int]
|
||||
SIGNATURE_FIELD_NUMBER: _ClassVar[int]
|
||||
corpus: str
|
||||
language: str
|
||||
path: str
|
||||
root: str
|
||||
signature: str
|
||||
def __init__(self, signature: _Optional[str] = ..., corpus: _Optional[str] = ..., root: _Optional[str] = ..., path: _Optional[str] = ..., language: _Optional[str] = ...) -> None: ...
|
||||
|
||||
class VNameMask(_message.Message):
|
||||
__slots__ = ["corpus", "language", "path", "root", "signature"]
|
||||
CORPUS_FIELD_NUMBER: _ClassVar[int]
|
||||
LANGUAGE_FIELD_NUMBER: _ClassVar[int]
|
||||
PATH_FIELD_NUMBER: _ClassVar[int]
|
||||
ROOT_FIELD_NUMBER: _ClassVar[int]
|
||||
SIGNATURE_FIELD_NUMBER: _ClassVar[int]
|
||||
corpus: bool
|
||||
language: bool
|
||||
path: bool
|
||||
root: bool
|
||||
signature: bool
|
||||
def __init__(self, signature: bool = ..., corpus: bool = ..., root: bool = ..., path: bool = ..., language: bool = ...) -> None: ...
|
||||
|
||||
class VNameRewriteRule(_message.Message):
|
||||
__slots__ = ["pattern", "v_name"]
|
||||
PATTERN_FIELD_NUMBER: _ClassVar[int]
|
||||
V_NAME_FIELD_NUMBER: _ClassVar[int]
|
||||
pattern: str
|
||||
v_name: VName
|
||||
def __init__(self, pattern: _Optional[str] = ..., v_name: _Optional[_Union[VName, _Mapping]] = ...) -> None: ...
|
||||
|
||||
class VNameRewriteRules(_message.Message):
|
||||
__slots__ = ["rule"]
|
||||
RULE_FIELD_NUMBER: _ClassVar[int]
|
||||
rule: _containers.RepeatedCompositeFieldContainer[VNameRewriteRule]
|
||||
def __init__(self, rule: _Optional[_Iterable[_Union[VNameRewriteRule, _Mapping]]] = ...) -> None: ...
|
||||
|
||||
class WriteReply(_message.Message):
|
||||
__slots__: list[str] = []
|
||||
def __init__(self) -> None: ...
|
||||
|
||||
class WriteRequest(_message.Message):
|
||||
__slots__ = ["source", "update"]
|
||||
class Update(_message.Message):
|
||||
__slots__ = ["edge_kind", "fact_name", "fact_value", "target"]
|
||||
EDGE_KIND_FIELD_NUMBER: _ClassVar[int]
|
||||
FACT_NAME_FIELD_NUMBER: _ClassVar[int]
|
||||
FACT_VALUE_FIELD_NUMBER: _ClassVar[int]
|
||||
TARGET_FIELD_NUMBER: _ClassVar[int]
|
||||
edge_kind: str
|
||||
fact_name: str
|
||||
fact_value: bytes
|
||||
target: VName
|
||||
def __init__(self, edge_kind: _Optional[str] = ..., target: _Optional[_Union[VName, _Mapping]] = ..., fact_name: _Optional[str] = ..., fact_value: _Optional[bytes] = ...) -> None: ...
|
||||
SOURCE_FIELD_NUMBER: _ClassVar[int]
|
||||
UPDATE_FIELD_NUMBER: _ClassVar[int]
|
||||
source: VName
|
||||
update: _containers.RepeatedCompositeFieldContainer[WriteRequest.Update]
|
||||
def __init__(self, source: _Optional[_Union[VName, _Mapping]] = ..., update: _Optional[_Iterable[_Union[WriteRequest.Update, _Mapping]]] = ...) -> None: ...
|
||||
@@ -1,40 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
# Generated by the protocol buffer compiler. DO NOT EDIT!
|
||||
# NO CHECKED-IN PROTOBUF GENCODE
|
||||
# source: kythe/proto/storage_service.proto
|
||||
# Protobuf Python Version: 5.28.1
|
||||
"""Generated protocol buffer code."""
|
||||
|
||||
from google.protobuf import descriptor as _descriptor
|
||||
from google.protobuf import descriptor_pool as _descriptor_pool
|
||||
from google.protobuf import runtime_version as _runtime_version
|
||||
from google.protobuf import symbol_database as _symbol_database
|
||||
from google.protobuf.internal import builder as _builder
|
||||
|
||||
_runtime_version.ValidateProtobufRuntimeVersion(
|
||||
_runtime_version.Domain.PUBLIC, 5, 28, 1, "", "kythe/proto/storage_service.proto"
|
||||
)
|
||||
# @@protoc_insertion_point(imports)
|
||||
|
||||
_sym_db = _symbol_database.Default()
|
||||
|
||||
|
||||
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(
|
||||
b'\n!kythe/proto/storage_service.proto\x12\x0bkythe.proto\x1a\x19kythe/proto/storage.proto2\xbf\x01\n\nGraphStore\x12\x38\n\x04Read\x12\x18.kythe.proto.ReadRequest\x1a\x12.kythe.proto.Entry"\x00\x30\x01\x12\x38\n\x04Scan\x12\x18.kythe.proto.ScanRequest\x1a\x12.kythe.proto.Entry"\x00\x30\x01\x12=\n\x05Write\x12\x19.kythe.proto.WriteRequest\x1a\x17.kythe.proto.WriteReply"\x00\x32\x8e\x01\n\x11ShardedGraphStore\x12=\n\x05\x43ount\x12\x19.kythe.proto.CountRequest\x1a\x17.kythe.proto.CountReply"\x00\x12:\n\x05Shard\x12\x19.kythe.proto.ShardRequest\x1a\x12.kythe.proto.Entry"\x00\x30\x01\x42P\n\x1f\x63om.google.devtools.kythe.protoZ-kythe.io/kythe/proto/storage_service_go_protob\x06proto3'
|
||||
)
|
||||
|
||||
_globals = globals()
|
||||
_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
|
||||
_builder.BuildTopDescriptorsAndMessages(
|
||||
DESCRIPTOR, "kythe.proto.storage_service_pb2", _globals
|
||||
)
|
||||
if not _descriptor._USE_C_DESCRIPTORS:
|
||||
_globals["DESCRIPTOR"]._loaded_options = None
|
||||
_globals[
|
||||
"DESCRIPTOR"
|
||||
]._serialized_options = b"\n\037com.google.devtools.kythe.protoZ-kythe.io/kythe/proto/storage_service_go_proto"
|
||||
_globals["_GRAPHSTORE"]._serialized_start = 78
|
||||
_globals["_GRAPHSTORE"]._serialized_end = 269
|
||||
_globals["_SHARDEDGRAPHSTORE"]._serialized_start = 272
|
||||
_globals["_SHARDEDGRAPHSTORE"]._serialized_end = 414
|
||||
# @@protoc_insertion_point(module_scope)
|
||||
@@ -1,3 +0,0 @@
|
||||
from google.protobuf import descriptor as _descriptor
|
||||
|
||||
DESCRIPTOR: _descriptor.FileDescriptor
|
||||
@@ -1,334 +0,0 @@
|
||||
import urllib.parse
|
||||
import logging
|
||||
import uuid
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Generator, Dict, Set
|
||||
from buttercup.program_model.data.kythe.proto.storage_pb2 import Entry, VName
|
||||
from buttercup.program_model.utils.varint import decode_stream
|
||||
from multiprocessing import Pool
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Any, List, TextIO
|
||||
from io import BytesIO
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
ENTRY_CHUNK_SIZE = 50
|
||||
|
||||
|
||||
def encode_value(value: bytes) -> str:
|
||||
"""Encode bytes as hex string"""
|
||||
return value.hex()
|
||||
|
||||
|
||||
def decode_value(value: str) -> bytes:
|
||||
"""Decode hex string as bytes"""
|
||||
return bytes.fromhex(value)
|
||||
|
||||
|
||||
@dataclass(frozen=True, repr=False)
|
||||
class KytheURI:
|
||||
"""Kythe entries described here: https://kythe.io/docs/kythe-storage.html#_entry"""
|
||||
|
||||
corpus: str
|
||||
language: str
|
||||
path: str
|
||||
root: str
|
||||
signature: str
|
||||
|
||||
def __str__(self) -> str:
|
||||
lst = [self.corpus, self.language, self.path, self.root, self.signature]
|
||||
uristr = "/".join([urllib.parse.quote(x, safe="") for x in lst])
|
||||
return uristr
|
||||
|
||||
@staticmethod
|
||||
def from_vname(vn: VName) -> Any:
|
||||
return KytheURI(
|
||||
signature=vn.signature,
|
||||
corpus=vn.corpus,
|
||||
root=vn.root,
|
||||
path=vn.path,
|
||||
language=vn.language,
|
||||
)
|
||||
|
||||
|
||||
class ToGraphML(ABC):
|
||||
"""Class to convert a Kythe entry to a GraphML string."""
|
||||
|
||||
@abstractmethod
|
||||
def to_graphml(self) -> str:
|
||||
"""Convert the entry to a GraphML string."""
|
||||
pass
|
||||
|
||||
|
||||
@dataclass
|
||||
class Node(ToGraphML):
|
||||
"""Node in the graph."""
|
||||
|
||||
id: str
|
||||
label: str | None = None
|
||||
properties: Dict[str, str] = field(default_factory=dict)
|
||||
|
||||
def __str__(self) -> str:
|
||||
properties_str = ",".join([f"[{k}: {v}]" for k, v in self.properties.items()])
|
||||
return f"Node(id={self.id}, label={self.label}, properties={properties_str}"
|
||||
|
||||
def to_graphml(self) -> str:
|
||||
"""Convert node to a GraphML string."""
|
||||
content = []
|
||||
content.append(f'<node id="{self.id}">')
|
||||
for key, value in self.properties.items():
|
||||
content.append(f'<data key="{key}">{value}</data>')
|
||||
content.append("</node>")
|
||||
return "".join(content)
|
||||
|
||||
|
||||
@dataclass
|
||||
class Edge(ToGraphML):
|
||||
"""Edge in the graph."""
|
||||
|
||||
id: str
|
||||
source_id: str
|
||||
target_id: str
|
||||
properties: Dict[str, str] = field(default_factory=dict)
|
||||
|
||||
def __str__(self) -> str:
|
||||
properties_str = ",".join([f"[{k}: {v}]" for k, v in self.properties.items()])
|
||||
return f"Edge(id={self.id}, source_id={self.source_id}, target_id={self.target_id}, properties={properties_str}"
|
||||
|
||||
def to_graphml(self) -> str:
|
||||
"""Convert edge to a GraphML string."""
|
||||
content = []
|
||||
content.append(
|
||||
f'<edge id="{self.id}" source="{self.source_id}" target="{self.target_id}">'
|
||||
)
|
||||
for key, value in self.properties.items():
|
||||
content.append(f'<data key="{key}">{value}</data>')
|
||||
content.append("</edge>")
|
||||
return "".join(content)
|
||||
|
||||
|
||||
@dataclass
|
||||
class WriteResult:
|
||||
"""Result of writing a fragment."""
|
||||
|
||||
nodes: list[Node]
|
||||
edges: list[Edge]
|
||||
node_props: list[str]
|
||||
edge_props: list[str]
|
||||
|
||||
|
||||
def chunk_data(
|
||||
generator: Generator[Any, None, None], chunk_size: int
|
||||
) -> Generator[list[Any], None, None]:
|
||||
"""Split a generator into chunks of specified size."""
|
||||
chunk = []
|
||||
for item in generator:
|
||||
chunk.append(item)
|
||||
if len(chunk) == chunk_size:
|
||||
yield chunk
|
||||
chunk = []
|
||||
if chunk:
|
||||
yield chunk
|
||||
|
||||
|
||||
class GraphWriter:
|
||||
"""Class to write a GraphML file."""
|
||||
|
||||
def __init__(self, task_id: str):
|
||||
self.task_id: str = task_id
|
||||
|
||||
def is_edge(self, ent: Entry) -> bool:
|
||||
"""Check if the entry is an edge."""
|
||||
return ent.edge_kind != ""
|
||||
|
||||
def convert_node(self, nd: VName) -> Node:
|
||||
"""Converts a Kythe node to a GraphML node."""
|
||||
uri = str(KytheURI.from_vname(nd))
|
||||
return Node(
|
||||
id=uri,
|
||||
properties={
|
||||
"corpus": encode_value(nd.corpus.encode("utf-8")),
|
||||
"language": encode_value(nd.language.encode("utf-8")),
|
||||
"path": encode_value(nd.path.encode("utf-8")),
|
||||
"root": encode_value(nd.root.encode("utf-8")),
|
||||
"signature": encode_value(nd.signature.encode("utf-8")),
|
||||
},
|
||||
)
|
||||
|
||||
def entry_to_graphml(
|
||||
self,
|
||||
entry: Entry,
|
||||
node_props: list[str],
|
||||
edge_props: list[str],
|
||||
edges: list[Edge],
|
||||
nodes: list[Node],
|
||||
) -> None:
|
||||
source_node = self.convert_node(entry.source)
|
||||
nodes.append(source_node)
|
||||
source_node.properties["task_id"] = encode_value(self.task_id.encode("utf-8"))
|
||||
key = entry.fact_name
|
||||
value = encode_value(entry.fact_value)
|
||||
if self.is_edge(entry):
|
||||
target_node = self.convert_node(entry.target)
|
||||
nodes.append(target_node)
|
||||
target_node.properties["task_id"] = encode_value(
|
||||
self.task_id.encode("utf-8")
|
||||
)
|
||||
edge_id = str(uuid.uuid4())
|
||||
edge = Edge(
|
||||
id=edge_id,
|
||||
source_id=source_node.id,
|
||||
target_id=target_node.id,
|
||||
)
|
||||
edge.properties["labelE"] = entry.edge_kind
|
||||
edge.properties["task_id"] = encode_value(self.task_id.encode("utf-8"))
|
||||
edges.append(edge)
|
||||
else:
|
||||
source_node.properties[key] = value
|
||||
node_props.append(key)
|
||||
nodes.append(source_node)
|
||||
|
||||
def write_entry(self, entries: list[bytes]) -> WriteResult:
|
||||
node_props: list[str] = list()
|
||||
edge_props: list[str] = list()
|
||||
edges: list[Edge] = list()
|
||||
nodes: list[Node] = list()
|
||||
|
||||
for bts in entries:
|
||||
try:
|
||||
ent = Entry()
|
||||
ent.ParseFromString(bts)
|
||||
except Exception as e:
|
||||
logger.error("Error parsing entry: %s", e)
|
||||
continue
|
||||
|
||||
self.entry_to_graphml(ent, node_props, edge_props, edges, nodes)
|
||||
|
||||
return WriteResult(
|
||||
nodes=nodes,
|
||||
edges=edges,
|
||||
node_props=node_props,
|
||||
edge_props=edge_props,
|
||||
)
|
||||
|
||||
|
||||
@dataclass(repr=False)
|
||||
class GraphStorage:
|
||||
"""Class to interact between Kythe and an output file."""
|
||||
|
||||
def __init__(self, task_id: str):
|
||||
self.task_id: str = task_id
|
||||
self.node_properties: Set[str] = set(
|
||||
["corpus", "language", "path", "root", "signature", "task_id"]
|
||||
)
|
||||
self.edge_properties: Set[str] = set(["labelE", "task_id"])
|
||||
|
||||
def process_stream(self, fl: BytesIO, outfile: TextIO) -> None:
|
||||
"""Process a stream of Kythe entries and output them to a GraphML file."""
|
||||
|
||||
try:
|
||||
fw = GraphWriter(self.task_id)
|
||||
with Pool() as p:
|
||||
nodes: dict[str, Node] = dict()
|
||||
edges: list[Edge] = list()
|
||||
|
||||
for res in p.imap_unordered(
|
||||
fw.write_entry,
|
||||
chunk_data(self.iterate_over_entries(fl), ENTRY_CHUNK_SIZE),
|
||||
chunksize=3,
|
||||
):
|
||||
for node_props in res.node_props:
|
||||
self.node_properties.add(node_props)
|
||||
for edge_props in res.edge_props:
|
||||
self.edge_properties.add(edge_props)
|
||||
for node in res.nodes:
|
||||
if node.id in nodes:
|
||||
nodes[node.id].properties.update(node.properties)
|
||||
else:
|
||||
nodes[node.id] = node
|
||||
for edge in res.edges:
|
||||
edges.append(edge)
|
||||
self.to_graphml(outfile, nodes, edges)
|
||||
|
||||
except Exception as e:
|
||||
logger.error("Exception occurred: %s", e)
|
||||
raise e
|
||||
|
||||
def iterate_over_entries(self, fl: BytesIO) -> Generator[bytes, None, None]:
|
||||
"""Iterate over entries in the stream."""
|
||||
|
||||
# Indexers emit a delimited stream of entry protobufs
|
||||
# From: https://kythe.io/examples/#indexing-compilations
|
||||
while True:
|
||||
try:
|
||||
yield self.next_entry(fl)
|
||||
# This is the end of the stream. This error is expected.
|
||||
except EOFError:
|
||||
break
|
||||
except Exception as e:
|
||||
raise e
|
||||
|
||||
def next_entry(self, fl: BytesIO) -> bytes:
|
||||
"""Parse a Kythe entry from the stream."""
|
||||
|
||||
# Read the size of the entry protobuf
|
||||
sz = decode_stream(fl)
|
||||
bts = bytes()
|
||||
while len(bts) < sz:
|
||||
rd = fl.read(sz - len(bts))
|
||||
if len(rd) <= 0:
|
||||
return b""
|
||||
bts += rd
|
||||
return bts
|
||||
|
||||
def parse_entry(self, bts: bytes) -> Entry:
|
||||
"""Parse a Kythe entry from a bytes object."""
|
||||
ent = Entry()
|
||||
ent.ParseFromString(bts)
|
||||
return ent
|
||||
|
||||
def parse_entries(self, fl: BytesIO) -> Generator[Entry, None, None]:
|
||||
"""Parse a stream of Kythe entries from a file."""
|
||||
while True:
|
||||
bts = self.next_entry(fl)
|
||||
if bts == b"":
|
||||
break
|
||||
yield self.parse_entry(bts)
|
||||
|
||||
def to_graphml(
|
||||
self, outfile: TextIO, nodes: Dict[str, Node], edges: List[Edge]
|
||||
) -> None:
|
||||
"""Convert graph to a GraphML file.
|
||||
From: https://tinkerpop.apache.org/docs/3.7.3/dev/io/
|
||||
"""
|
||||
outfile.write('<?xml version="1.0" encoding="UTF-8"?>\n')
|
||||
outfile.write(
|
||||
'<graphml xmlns="http://graphml.graphdrawing.org/xmlns" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://graphml.graphdrawing.org/xmlns http://graphml.graphdrawing.org/xmlns/1.1/graphml.xsd">\n'
|
||||
)
|
||||
outfile.write('<graph id="G" edgedefault="directed">\n')
|
||||
|
||||
# Output node properties
|
||||
for p in self.node_properties:
|
||||
outfile.write(
|
||||
f'<key id="{p}" for="node" attr.name="{p}" attr.type="string" />\n'
|
||||
)
|
||||
|
||||
# Output edge properties
|
||||
for p in self.edge_properties:
|
||||
outfile.write(
|
||||
f'<key id="{p}" for="edge" attr.name="{p}" attr.type="string" />\n'
|
||||
)
|
||||
|
||||
# Output node contents
|
||||
id_set = set()
|
||||
for id_name, node in nodes.items():
|
||||
id_set.add(id_name)
|
||||
outfile.write(node.to_graphml())
|
||||
|
||||
# Output edge contents
|
||||
for edge in edges:
|
||||
if edge.source_id in id_set and edge.target_id in id_set:
|
||||
outfile.write(edge.to_graphml())
|
||||
|
||||
outfile.write("</graph>")
|
||||
outfile.write("</graphml>")
|
||||
@@ -1,128 +0,0 @@
|
||||
import logging
|
||||
import uuid
|
||||
import subprocess
|
||||
from dataclasses import dataclass
|
||||
from buttercup.common.challenge_task import ChallengeTask
|
||||
import os
|
||||
from argparse import ArgumentParser
|
||||
from pathlib import Path
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class IndexConf:
|
||||
scriptdir: Path
|
||||
python: str
|
||||
allow_pull: bool
|
||||
base_image_url: str
|
||||
wdir: Path
|
||||
|
||||
|
||||
class Indexer:
|
||||
def __init__(self, conf: IndexConf):
|
||||
self.conf = conf
|
||||
|
||||
def build_image(self, task: ChallengeTask) -> str | None:
|
||||
logger.debug(f"Building image for {task.task_meta.task_id}")
|
||||
res = task.build_image(pull_latest_base_image=self.conf.allow_pull)
|
||||
if not res.success:
|
||||
return None
|
||||
base_image_name = f"{self.conf.base_image_url}/{task.project_name}"
|
||||
|
||||
buildid = task.task_meta.task_id
|
||||
emitted_image = f"kyther_indexer_image_{task.project_name}_{buildid}"
|
||||
wdir = f"{self.conf.scriptdir}"
|
||||
command = [
|
||||
"docker",
|
||||
"build",
|
||||
"-t",
|
||||
emitted_image,
|
||||
"--build-arg",
|
||||
f"BASE_IMAGE={base_image_name}",
|
||||
".",
|
||||
]
|
||||
result = subprocess.run(command, check=True, cwd=wdir, capture_output=True)
|
||||
if result.returncode != 0:
|
||||
logger.error(
|
||||
f"Failed to build image for {task.task_meta.task_id}. Return code: {result.returncode}"
|
||||
)
|
||||
return None
|
||||
logger.debug(f"Finished building image for {task.task_meta.task_id}")
|
||||
return emitted_image
|
||||
|
||||
def index_target(self, task: ChallengeTask) -> str | None:
|
||||
logger.debug(f"Started indexing target {task.task_meta.task_id}")
|
||||
emitted_image = self.build_image(task)
|
||||
if emitted_image is None:
|
||||
return None
|
||||
|
||||
indexuid = str(uuid.uuid4())
|
||||
output_dir = f"{self.conf.wdir}/output_{indexuid}"
|
||||
os.makedirs(output_dir, exist_ok=True)
|
||||
workdir = task.workdir_from_dockerfile().absolute().as_posix()
|
||||
|
||||
# Log directory information
|
||||
logger.debug(f"Task source path: {task.get_source_path().absolute()}")
|
||||
logger.debug(f"Work directory: {workdir}")
|
||||
logger.debug(f"Output directory: {output_dir}")
|
||||
|
||||
# Verify directories exist
|
||||
if not task.get_source_path().exists():
|
||||
logger.error(
|
||||
f"Source path does not exist: {task.get_source_path().absolute()}"
|
||||
)
|
||||
return None
|
||||
|
||||
command = [
|
||||
"docker",
|
||||
"run",
|
||||
"-v",
|
||||
f"{task.get_source_path().absolute()}:{workdir}",
|
||||
"-v",
|
||||
f"{output_dir}:/kythe_out",
|
||||
"-e",
|
||||
"KYTHE_OUTPUT_DIRECTORY=/kythe_out",
|
||||
"-e",
|
||||
f"KYTHE_CORPUS={task.task_meta.task_id}",
|
||||
emitted_image,
|
||||
"compile_and_extract",
|
||||
]
|
||||
result = subprocess.run(command, check=True, capture_output=True)
|
||||
if result.returncode != 0:
|
||||
logger.error(
|
||||
f"Failed to index target {task.task_meta.task_id}. Return code: {result.returncode}"
|
||||
)
|
||||
return None
|
||||
logger.debug(f"Finished indexing target {task.task_meta.task_id}")
|
||||
return output_dir
|
||||
|
||||
|
||||
def main() -> None:
|
||||
prsr = ArgumentParser()
|
||||
prsr.add_argument("--scriptdir", type=str, required=True)
|
||||
prsr.add_argument("--python", default="python3")
|
||||
prsr.add_argument("--allow_pull", type=bool, default=True)
|
||||
prsr.add_argument("--base_image_url", type=str, default="gcr.io/oss-fuzz")
|
||||
prsr.add_argument("--wdir", type=str, default="/tmp")
|
||||
prsr.add_argument("--task_dir", type=str, required=True)
|
||||
args = prsr.parse_args()
|
||||
|
||||
conf = IndexConf(
|
||||
scriptdir=args.scriptdir,
|
||||
python=args.python,
|
||||
allow_pull=args.allow_pull,
|
||||
base_image_url=args.base_image_url,
|
||||
wdir=args.wdir,
|
||||
)
|
||||
|
||||
task = ChallengeTask(
|
||||
args.task_dir,
|
||||
)
|
||||
with task.get_rw_copy(work_dir=args.wdir, delete=False) as local_task:
|
||||
indexer = Indexer(conf)
|
||||
print(indexer.index_target(local_task))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,41 +0,0 @@
|
||||
import subprocess
|
||||
import logging
|
||||
import os
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class KytheConf:
|
||||
kythe_dir: Path
|
||||
|
||||
|
||||
class KytheTool:
|
||||
def __init__(self, conf: KytheConf):
|
||||
self.conf = conf
|
||||
|
||||
def merge_kythe_output(self, input_dir: Path, output_kzip: Path) -> bool:
|
||||
logger.debug(f"Merging kythe output from {input_dir} to {output_kzip}")
|
||||
merge_path = os.path.join(self.conf.kythe_dir, "tools/kzip")
|
||||
|
||||
total = []
|
||||
for fl in os.listdir(input_dir):
|
||||
if fl.endswith(".kzip"):
|
||||
total.append(os.path.join(input_dir, fl))
|
||||
|
||||
command = [merge_path, "merge", "--output", str(output_kzip)] + total
|
||||
subprocess.run(command, check=True)
|
||||
logger.debug(f"Finished merging kythe output from {input_dir} to {output_kzip}")
|
||||
return True
|
||||
|
||||
def cxx_index(self, input_kzip: Path, output_bin: Path) -> bool:
|
||||
logger.debug(f"Indexing kythe output from {input_kzip} to {output_bin}")
|
||||
indexer_path = os.path.join(self.conf.kythe_dir, "indexers/cxx_indexer")
|
||||
command = [indexer_path, str(input_kzip), "-o", str(output_bin)]
|
||||
subprocess.run(command, check=True)
|
||||
logger.debug(
|
||||
f"Finished indexing kythe output from {input_kzip} to {output_bin}"
|
||||
)
|
||||
return True
|
||||
@@ -1,7 +1,5 @@
|
||||
import logging
|
||||
import os
|
||||
import stat
|
||||
import uuid
|
||||
from typing import Any
|
||||
from dataclasses import dataclass, field
|
||||
from buttercup.common.queues import (
|
||||
@@ -10,9 +8,6 @@ from buttercup.common.queues import (
|
||||
QueueNames,
|
||||
GroupNames,
|
||||
)
|
||||
from buttercup.program_model.indexer import Indexer, IndexConf
|
||||
from buttercup.program_model.kythe import KytheTool, KytheConf
|
||||
from buttercup.program_model.graph import GraphStorage
|
||||
from buttercup.program_model.codequery import CodeQueryPersistent
|
||||
from buttercup.common.datastructures.msg_pb2 import IndexRequest, IndexOutput
|
||||
from buttercup.common.challenge_task import ChallengeTask
|
||||
@@ -22,10 +17,7 @@ from pathlib import Path
|
||||
from opentelemetry import trace
|
||||
from opentelemetry.trace import Status, StatusCode
|
||||
from redis import Redis
|
||||
import subprocess
|
||||
import tempfile
|
||||
import buttercup.common.node_local as node_local
|
||||
from io import BytesIO
|
||||
from buttercup.common.telemetry import set_crs_attributes, CRSActionCategory
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -39,10 +31,6 @@ class ProgramModel:
|
||||
output_queue: ReliableQueue | None = field(init=False, default=None)
|
||||
registry: TaskRegistry | None = field(init=False, default=None)
|
||||
wdir: Path | None = None
|
||||
script_dir: Path | None = None
|
||||
kythe_dir: Path | None = None
|
||||
graphdb_url: str = "ws://graphdb:8182/gremlin"
|
||||
graphdb_enabled: bool = True
|
||||
python: str | None = None
|
||||
allow_pull: bool = True
|
||||
base_image_url: str = field(
|
||||
@@ -53,10 +41,6 @@ class ProgramModel:
|
||||
"""Post-initialization setup."""
|
||||
if self.wdir is not None:
|
||||
self.wdir = Path(self.wdir).resolve()
|
||||
if self.script_dir is not None:
|
||||
self.script_dir = Path(self.script_dir).resolve()
|
||||
if self.kythe_dir is not None:
|
||||
self.kythe_dir = Path(self.kythe_dir).resolve()
|
||||
|
||||
if self.redis is not None:
|
||||
logger.debug("Using Redis for task queues")
|
||||
@@ -75,153 +59,6 @@ class ProgramModel:
|
||||
"""Cleanup resources used by the program model"""
|
||||
pass
|
||||
|
||||
def index_kythe(
|
||||
self, task_id: str, output_id: str, output_dir: Path, td: Path
|
||||
) -> Path:
|
||||
"""Index using kythe. Returns path to the index binary file."""
|
||||
if self.kythe_dir is None:
|
||||
raise ValueError("Kythe directory is not initialized")
|
||||
ktool = KytheTool(KytheConf(self.kythe_dir))
|
||||
merged_kzip = Path(td) / f"kythe_output_merge_{output_id}.kzip"
|
||||
ktool.merge_kythe_output(output_dir, merged_kzip)
|
||||
|
||||
# Convert the merged kzip file into a binary file
|
||||
bin_file = Path(td) / f"kythe_output_cxx_{output_id}.bin"
|
||||
try:
|
||||
ktool.cxx_index(merged_kzip, bin_file)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to index program {task_id} to binary: {bin_file}")
|
||||
raise e
|
||||
|
||||
return bin_file
|
||||
|
||||
def store_graphml(
|
||||
self, task_id: str, output_id: str, bin_file: Path, td: Path
|
||||
) -> Path:
|
||||
"""Store the program into a graphml file. Returns path to the graphml file."""
|
||||
graphml_file = Path(td) / f"kythe_output_graphml_{output_id}.xml"
|
||||
with open(graphml_file, "w") as fw, open(bin_file, "rb") as fr:
|
||||
gs = GraphStorage(task_id=task_id)
|
||||
buf = BytesIO(fr.read())
|
||||
gs.process_stream(buf, fw)
|
||||
return graphml_file
|
||||
|
||||
def load_graphml(self, graphml_file: Path) -> None:
|
||||
"""Load graphml file into graph database."""
|
||||
# TODO(Evan): This needs to wait until JanusGraph is ready. For some reason, even if the container is running and healthy, it's not ready to accept connections.
|
||||
from gremlin_python.process.anonymous_traversal import traversal
|
||||
from gremlin_python.driver.driver_remote_connection import (
|
||||
DriverRemoteConnection,
|
||||
)
|
||||
|
||||
g = traversal().withRemote(DriverRemoteConnection(self.graphdb_url, "g"))
|
||||
g.io(str(graphml_file)).read().iterate()
|
||||
|
||||
def process_task_kythe(self, args: IndexRequest) -> bool:
|
||||
"""Process a single task for indexing a program"""
|
||||
# Convert path strings to Path objects
|
||||
with tempfile.TemporaryDirectory(dir=self.wdir) as td:
|
||||
logger.debug(f"Running indexer for {args.task_id} | {args.task_dir}")
|
||||
|
||||
# Change permissions so that JanusGraph can read from the temporary directory
|
||||
current = os.stat(td).st_mode
|
||||
janus_user = stat.S_IRGRP | stat.S_IROTH | stat.S_IXGRP | stat.S_IXOTH
|
||||
os.chmod(td, current | janus_user)
|
||||
|
||||
tsk = ChallengeTask(
|
||||
read_only_task_dir=args.task_dir,
|
||||
python_path=self.python,
|
||||
)
|
||||
|
||||
with tsk.get_rw_copy(work_dir=td) as local_tsk:
|
||||
# Apply the diff if it exists
|
||||
logger.debug(f"Applying diff for {args.task_id}")
|
||||
if not local_tsk.apply_patch_diff():
|
||||
logger.debug(f"No diffs for {args.task_id}")
|
||||
|
||||
# Index the task
|
||||
try:
|
||||
if self.script_dir is None:
|
||||
raise ValueError("Script directory is not initialized")
|
||||
if self.python is None:
|
||||
raise ValueError("Python is not initialized")
|
||||
if self.kythe_dir is None:
|
||||
raise ValueError("Kythe directory is not initialized")
|
||||
indexer_conf = IndexConf(
|
||||
scriptdir=self.script_dir,
|
||||
python=self.python,
|
||||
allow_pull=self.allow_pull,
|
||||
base_image_url=self.base_image_url,
|
||||
wdir=Path(td),
|
||||
)
|
||||
indexer = Indexer(indexer_conf)
|
||||
output_dir = indexer.index_target(local_tsk)
|
||||
|
||||
# Because docker is running as root, we need to chown the output directory to the current user
|
||||
subprocess.run(
|
||||
[
|
||||
"sudo",
|
||||
"chown",
|
||||
"-R",
|
||||
f"{os.getuid()}:{os.getgid()}",
|
||||
local_tsk.local_task_dir,
|
||||
],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to index task {args.task_id}: {e}")
|
||||
return False
|
||||
if output_dir is None:
|
||||
logger.error(f"Failed to index task {args.task_id}")
|
||||
return False
|
||||
logger.debug(f"Successfully indexed task {args.task_id}")
|
||||
|
||||
output_id = str(uuid.uuid4())
|
||||
|
||||
try:
|
||||
bin_file = self.index_kythe(
|
||||
args.task_id, output_id, Path(output_dir), Path(td)
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to index files for {args.task_id}: {e}")
|
||||
return False
|
||||
logger.debug(
|
||||
f"Successfully indexed and merged kythe output for {args.task_id} to {bin_file}"
|
||||
)
|
||||
|
||||
# Store the program into a graphml file
|
||||
try:
|
||||
graphml_file = Path(td) / f"kythe_output_graphml_{output_id}.xml"
|
||||
with open(graphml_file, "w") as fw, open(bin_file, "rb") as fr:
|
||||
gs = GraphStorage(task_id=args.task_id)
|
||||
buf = BytesIO(fr.read())
|
||||
gs.process_stream(buf, fw)
|
||||
logger.debug(
|
||||
f"Successfully stored program {args.task_id} in graphml file: {graphml_file}"
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
f"Failed to store program {args.task_id} in graphml file {graphml_file}: {e}"
|
||||
)
|
||||
return False
|
||||
logger.debug(
|
||||
f"Successfully stored program {args.task_id} in graphml file: {graphml_file}"
|
||||
)
|
||||
|
||||
# Load graphml file into graph database
|
||||
try:
|
||||
logger.debug("Loading graphml file into JanusGraph...")
|
||||
self.load_graphml(graphml_file)
|
||||
except Exception:
|
||||
logger.exception(
|
||||
f"Failed to load graphml file {graphml_file} into JanusGraph"
|
||||
)
|
||||
return False
|
||||
logger.debug("Successfully loaded graphml file into JanusGraph")
|
||||
|
||||
return True
|
||||
|
||||
def process_task_codequery(self, args: IndexRequest) -> bool:
|
||||
"""Process a single task for indexing a program"""
|
||||
try:
|
||||
@@ -264,15 +101,10 @@ class ProgramModel:
|
||||
|
||||
def process_task(self, args: IndexRequest) -> bool:
|
||||
"""Process a single task for indexing a program"""
|
||||
# If at least one of the two methods succeeds, return True
|
||||
logger.info(
|
||||
f"Processing task {args.package_name}/{args.task_id}/{args.task_dir}"
|
||||
)
|
||||
rv_code_query: bool = self.process_task_codequery(args)
|
||||
rv_kythe: bool = False
|
||||
if self.graphdb_enabled:
|
||||
rv_kythe = self.process_task_kythe(args)
|
||||
return rv_code_query or rv_kythe
|
||||
return self.process_task_codequery(args)
|
||||
|
||||
def serve_item(self) -> bool:
|
||||
if self.task_queue is None:
|
||||
|
||||
@@ -1,27 +0,0 @@
|
||||
# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT!
|
||||
"""Client and server classes corresponding to protobuf-defined services."""
|
||||
|
||||
import grpc
|
||||
|
||||
|
||||
GRPC_GENERATED_VERSION = "1.68.1"
|
||||
GRPC_VERSION = grpc.__version__
|
||||
_version_not_supported = False
|
||||
|
||||
try:
|
||||
from grpc._utilities import first_version_is_lower
|
||||
|
||||
_version_not_supported = first_version_is_lower(
|
||||
GRPC_VERSION, GRPC_GENERATED_VERSION
|
||||
)
|
||||
except ImportError:
|
||||
_version_not_supported = True
|
||||
|
||||
if _version_not_supported:
|
||||
raise RuntimeError(
|
||||
f"The grpc package installed is at version {GRPC_VERSION},"
|
||||
+ " but the generated code in kythe/proto/storage_pb2_grpc.py depends on"
|
||||
+ f" grpcio>={GRPC_GENERATED_VERSION}."
|
||||
+ f" Please upgrade your grpc module to grpcio>={GRPC_GENERATED_VERSION}"
|
||||
+ f" or downgrade your generated code using grpcio-tools<={GRPC_VERSION}."
|
||||
)
|
||||
@@ -25,18 +25,11 @@ class WorkerSettings(BaseModel):
|
||||
python: Annotated[str, Field(default="python", description="Python path")]
|
||||
|
||||
|
||||
class IndexerSettings(BaseModel):
|
||||
kythe_dir: Annotated[
|
||||
str, Field(default="scripts/gzs/kythe", description="Kythe directory")
|
||||
]
|
||||
script_dir: Annotated[str, Field(default="scripts", description="Script directory")]
|
||||
|
||||
|
||||
class ServeCommand(WorkerSettings, IndexerSettings, BuilderSettings):
|
||||
class ServeCommand(WorkerSettings, BuilderSettings):
|
||||
pass
|
||||
|
||||
|
||||
class ProcessCommand(WorkerSettings, IndexerSettings, BuilderSettings):
|
||||
class ProcessCommand(WorkerSettings, BuilderSettings):
|
||||
task_dir: Annotated[str, Field(description="Task directory", default=...)]
|
||||
task_id: Annotated[str, Field(description="Task ID", default=...)]
|
||||
|
||||
@@ -49,13 +42,6 @@ class Settings(BaseSettings):
|
||||
log_max_line_length: Annotated[
|
||||
int | None, Field(default=None, description="Log max line length")
|
||||
]
|
||||
graphdb_url: Annotated[
|
||||
str,
|
||||
Field(description="Graph database URL", default="ws://graphdb:8182/gremlin"),
|
||||
]
|
||||
graphdb_enabled: Annotated[
|
||||
bool, Field(description="Enable graph database", default=True)
|
||||
]
|
||||
|
||||
serve: CliSubCommand[ServeCommand]
|
||||
process: CliSubCommand[ProcessCommand]
|
||||
|
||||
@@ -1,61 +0,0 @@
|
||||
"""Varint encoder/decoder
|
||||
varint from https://github.com/fmoo/python-varint/blob/master/varint.py
|
||||
"""
|
||||
|
||||
from io import BytesIO
|
||||
|
||||
|
||||
import sys
|
||||
|
||||
if sys.version > "3":
|
||||
|
||||
def _byte(b: int) -> bytes:
|
||||
return bytes((b,))
|
||||
else:
|
||||
|
||||
def _byte(b: int) -> str: # type: ignore[misc]
|
||||
return chr(b)
|
||||
|
||||
|
||||
def encode(number: int) -> bytes:
|
||||
"""Pack `number` into varint bytes"""
|
||||
buf = b""
|
||||
while True:
|
||||
towrite = number & 0x7F
|
||||
number >>= 7
|
||||
if number:
|
||||
buf += _byte(towrite | 0x80)
|
||||
else:
|
||||
buf += _byte(towrite)
|
||||
break
|
||||
return buf
|
||||
|
||||
|
||||
def decode_stream(stream: BytesIO) -> int:
|
||||
"""Read a varint from `stream`"""
|
||||
shift = 0
|
||||
result = 0
|
||||
while True:
|
||||
i = _read_one(stream)
|
||||
result |= (i & 0x7F) << shift
|
||||
shift += 7
|
||||
if not (i & 0x80):
|
||||
break
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def decode_bytes(buf: bytes) -> int:
|
||||
"""Read a varint from from `buf` bytes"""
|
||||
return decode_stream(BytesIO(buf))
|
||||
|
||||
|
||||
def _read_one(stream: BytesIO) -> int:
|
||||
"""Read a byte from the file (as an integer)
|
||||
|
||||
raises EOFError if the stream ends while reading bytes.
|
||||
"""
|
||||
c = stream.read(1)
|
||||
if c == b"":
|
||||
raise EOFError("Unexpected EOF while reading bytes")
|
||||
return ord(c)
|
||||
@@ -2,8 +2,6 @@ import pytest
|
||||
import subprocess
|
||||
import os
|
||||
from unittest.mock import patch
|
||||
from buttercup.program_model.api import Graph
|
||||
from buttercup.program_model.graph import encode_value
|
||||
from buttercup.common.challenge_task import ChallengeTask
|
||||
from buttercup.common.task_meta import TaskMeta
|
||||
from buttercup.program_model.codequery import CodeQuery
|
||||
@@ -37,21 +35,6 @@ def is_cleanup_enabled(request) -> bool:
|
||||
return not request.config.getoption("--no-cleanup")
|
||||
|
||||
|
||||
def cleanup_graphdb(request, task_id: str):
|
||||
"""Clean up the JanusGraph database by dropping vertices and edges associated with a specific task ID."""
|
||||
if not is_cleanup_enabled(request):
|
||||
return
|
||||
|
||||
with Graph(url="ws://localhost:8182/gremlin") as graph:
|
||||
# Drop vertices and edges associated with the task ID
|
||||
graph.g.V().has(
|
||||
"task_id", encode_value(task_id.encode("utf-8"))
|
||||
).drop().iterate()
|
||||
graph.g.E().has(
|
||||
"task_id", encode_value(task_id.encode("utf-8"))
|
||||
).drop().iterate()
|
||||
|
||||
|
||||
def pytest_addoption(parser):
|
||||
parser.addoption(
|
||||
"--runintegration",
|
||||
@@ -460,10 +443,7 @@ def cleanup_module_task_dirs(request):
|
||||
logger.info(f"Cleaning up task IDs: {_task_ids}")
|
||||
logger.info(f"Cleaning up task dirs: {_task_dirs}")
|
||||
|
||||
# TODO(Evan): Uncomment this when you re-enable the graphdb
|
||||
# Clean up task IDs registered by this module
|
||||
# for task_id in _task_ids:
|
||||
# cleanup_graphdb(request, task_id)
|
||||
_task_ids.clear()
|
||||
|
||||
# Clean up temp directories created by this module
|
||||
|
||||
@@ -1,258 +0,0 @@
|
||||
"""Tests for the graph database.
|
||||
NOTE: Splitting this into individual tests for Kythe indexing is difficult because the project needs to exist in OSS Fuzz.
|
||||
"""
|
||||
|
||||
from pathlib import Path
|
||||
from xml.dom import minidom
|
||||
import pytest
|
||||
from typing import Iterator
|
||||
from buttercup.common.challenge_task import ChallengeTask
|
||||
from buttercup.program_model.api import Graph
|
||||
from buttercup.program_model.graph import encode_value
|
||||
from buttercup.program_model.program_model import ProgramModel
|
||||
from buttercup.common.datastructures.msg_pb2 import IndexRequest
|
||||
from gremlin_python.process.anonymous_traversal import traversal
|
||||
from gremlin_python.driver.driver_remote_connection import (
|
||||
DriverRemoteConnection,
|
||||
)
|
||||
|
||||
|
||||
def cleanup_graphdb(request, task_id: str):
|
||||
"""Clean up the JanusGraph database by dropping vertices and edges associated with a specific task ID."""
|
||||
from .conftest import cleanup_graphdb as conftest_cleanup
|
||||
|
||||
conftest_cleanup(request, task_id)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def get_graphml_content() -> str:
|
||||
"""Create a mock graphml file."""
|
||||
|
||||
# Create a test C file with two functions
|
||||
test_c_content = """
|
||||
#include <stdio.h>
|
||||
|
||||
int add(int a, int b) {
|
||||
return a + b;
|
||||
}
|
||||
|
||||
void print_hello(void) {
|
||||
printf("Hello, World!\\n");
|
||||
}
|
||||
"""
|
||||
|
||||
content = '<?xml version="1.0" encoding="UTF-8"?>'
|
||||
content += '<graphml xmlns="http://graphml.graphdrawing.org/xmlns" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://graphml.graphdrawing.org/xmlns http://graphml.graphdrawing.org/xmlns/1.1/graphml.xsd">'
|
||||
content += '<graph id="G" edgedefault="directed">'
|
||||
|
||||
for p in [
|
||||
"task_id",
|
||||
"path",
|
||||
"function",
|
||||
"file",
|
||||
"signature",
|
||||
"root",
|
||||
"corpus",
|
||||
"language",
|
||||
"/kythe/node/kind",
|
||||
"/kythe/code",
|
||||
"/kythe/loc/start",
|
||||
"/kythe/loc/end",
|
||||
"/kythe/text",
|
||||
"/kythe/edge/defines",
|
||||
]:
|
||||
content += f'<key id="{p}" for="node" attr.name="{p}" attr.type="string" />'
|
||||
|
||||
# Create edge key
|
||||
content += '<key id="labelE" for="edge" attr.name="labelE" attr.type="string" />'
|
||||
|
||||
# Create function node
|
||||
content += '<node id="1">'
|
||||
content += f'<data key="task_id">{encode_value(b"unit_test")}</data>'
|
||||
content += f'<data key="path">{encode_value(b"test.c")}</data>'
|
||||
content += '<data key="root"></data>'
|
||||
content += '<data key="corpus"></data>'
|
||||
content += f'<data key="language">{encode_value(b"c")}</data>'
|
||||
content += f'<data key="/kythe/node/kind">{encode_value(b"function")}</data>'
|
||||
content += (
|
||||
f'<data key="/kythe/code">{encode_value(b"int add(int a, int b)")}</data>'
|
||||
)
|
||||
content += "</node>"
|
||||
|
||||
content += '<node id="2">'
|
||||
content += f'<data key="task_id">{encode_value(b"unit_test")}</data>'
|
||||
content += f'<data key="path">{encode_value(b"test.c")}</data>'
|
||||
content += '<data key="root"></data>'
|
||||
content += '<data key="corpus"></data>'
|
||||
content += f'<data key="language">{encode_value(b"c")}</data>'
|
||||
content += f'<data key="/kythe/node/kind">{encode_value(b"function")}</data>'
|
||||
content += (
|
||||
f'<data key="/kythe/code">{encode_value(b"void print_hello(void)")}</data>'
|
||||
)
|
||||
content += "</node>"
|
||||
|
||||
# Create anchor node
|
||||
content += '<node id="3">'
|
||||
content += f'<data key="task_id">{encode_value(b"unit_test")}</data>'
|
||||
content += f'<data key="path">{encode_value(b"test.c")}</data>'
|
||||
content += '<data key="root"></data>'
|
||||
content += '<data key="corpus"></data>'
|
||||
content += f'<data key="language">{encode_value(b"c")}</data>'
|
||||
content += f'<data key="/kythe/node/kind">{encode_value(b"anchor")}</data>'
|
||||
content += f'<data key="/kythe/loc/start">{encode_value(b"21")}</data>'
|
||||
content += f'<data key="/kythe/loc/end">{encode_value(b"62")}</data>'
|
||||
content += "</node>"
|
||||
|
||||
content += '<edge id="1" source="3" target="1">'
|
||||
content += '<data key="labelE">/kythe/edge/defines</data>'
|
||||
content += "</edge>"
|
||||
|
||||
content += '<node id="4">'
|
||||
content += f'<data key="task_id">{encode_value(b"unit_test")}</data>'
|
||||
content += f'<data key="path">{encode_value(b"test.c")}</data>'
|
||||
content += '<data key="root"></data>'
|
||||
content += '<data key="corpus"></data>'
|
||||
content += f'<data key="language">{encode_value(b"c")}</data>'
|
||||
content += f'<data key="/kythe/node/kind">{encode_value(b"anchor")}</data>'
|
||||
content += f'<data key="/kythe/loc/start">{encode_value(b"66")}</data>'
|
||||
content += f'<data key="/kythe/loc/end">{encode_value(b"121")}</data>'
|
||||
content += "</node>"
|
||||
|
||||
content += '<edge id="2" source="4" target="2">'
|
||||
content += '<data key="labelE">/kythe/edge/defines</data>'
|
||||
content += "</edge>"
|
||||
|
||||
# Create file node
|
||||
content += '<node id="5">'
|
||||
content += f'<data key="task_id">{encode_value(b"unit_test")}</data>'
|
||||
content += f'<data key="path">{encode_value(b"test.c")}</data>'
|
||||
content += '<data key="root"></data>'
|
||||
content += '<data key="corpus"></data>'
|
||||
content += f'<data key="language">{encode_value(b"c")}</data>'
|
||||
content += f'<data key="/kythe/node/kind">{encode_value(b"file")}</data>'
|
||||
content += (
|
||||
f'<data key="/kythe/text">{encode_value(test_c_content.encode("utf-8"))}</data>'
|
||||
)
|
||||
content += "</node>"
|
||||
|
||||
content += "</graph>"
|
||||
content += "</graphml>"
|
||||
|
||||
xml_doc = minidom.parseString(content)
|
||||
pretty_xml = xml_doc.toprettyxml(indent=" ")
|
||||
return pretty_xml
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def graphml_db(tmp_path: Path, get_graphml_content: str, request) -> Iterator[bool]:
|
||||
cleanup_graphdb(request, "unit_test")
|
||||
|
||||
data_exists = False
|
||||
with Graph(url="ws://localhost:8182/gremlin") as graph:
|
||||
# Check if the task is already in the database
|
||||
if graph.g.V().has("task_id", encode_value(b"unit_test")).count().next():
|
||||
data_exists = True
|
||||
yield True
|
||||
|
||||
if not data_exists:
|
||||
# Create a mock graph database
|
||||
# Make the temporary directory readable by the gremlin user
|
||||
graphml_path = Path("/crs_scratch/graph.xml")
|
||||
graphml_path.write_text(data=get_graphml_content)
|
||||
g = traversal().withRemote(
|
||||
DriverRemoteConnection("ws://localhost:8182/gremlin", "g")
|
||||
)
|
||||
g.io(str("/crs_scratch/graph.xml")).read().iterate()
|
||||
yield True
|
||||
|
||||
# Clean up the graph database
|
||||
cleanup_graphdb(request, "unit_test")
|
||||
|
||||
|
||||
@pytest.mark.skip(reason="Skipping test because we're not using Kythe")
|
||||
def test_get_function_body(graphml_db: bool):
|
||||
"""Test getting function body from graph database."""
|
||||
assert graphml_db is True
|
||||
|
||||
# Test querying the graph database
|
||||
with Graph(url="ws://localhost:8182/gremlin") as graph:
|
||||
bodies = graph.get_function_body(function_name="add")
|
||||
assert len(bodies) == 1
|
||||
assert b"int add(int a, int b)" in bodies[0]
|
||||
assert b"return a + b;" in bodies[0]
|
||||
|
||||
bodies = graph.get_function_body(
|
||||
function_name="add", source_path=Path("test.c")
|
||||
)
|
||||
assert len(bodies) == 1
|
||||
assert b"int add(int a, int b)" in bodies[0]
|
||||
assert b"return a + b;" in bodies[0]
|
||||
|
||||
bodies = graph.get_function_body(
|
||||
function_name="add", source_path=Path("doesnotexist.c")
|
||||
)
|
||||
assert len(bodies) == 0
|
||||
|
||||
bodies = graph.get_function_body(function_name="print_hello")
|
||||
assert len(bodies) == 1
|
||||
assert b"void print_hello(void)" in bodies[0]
|
||||
assert b'printf("Hello, World!\\n");' in bodies[0]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def libpng_oss_fuzz_graphml_content(
|
||||
libpng_oss_fuzz_task: ChallengeTask, request
|
||||
) -> Iterator[bool]:
|
||||
"""Create a graphml file for libpng task."""
|
||||
|
||||
cleanup_graphdb(request, libpng_oss_fuzz_task.task_meta.task_id)
|
||||
|
||||
data_exists = False
|
||||
with Graph(url="ws://localhost:8182/gremlin") as graph:
|
||||
# Check if the task is already in the database
|
||||
if (
|
||||
graph.g.V()
|
||||
.has(
|
||||
"task_id",
|
||||
encode_value(libpng_oss_fuzz_task.task_meta.task_id.encode("utf-8")),
|
||||
)
|
||||
.count()
|
||||
.next()
|
||||
):
|
||||
data_exists = True
|
||||
|
||||
if not data_exists:
|
||||
index_request = IndexRequest(
|
||||
build_type="",
|
||||
package_name=libpng_oss_fuzz_task.project_name,
|
||||
sanitizer="",
|
||||
task_dir=libpng_oss_fuzz_task.task_dir.as_posix(),
|
||||
task_id=libpng_oss_fuzz_task.task_meta.task_id,
|
||||
)
|
||||
with ProgramModel(
|
||||
wdir=Path("/crs_scratch"),
|
||||
script_dir=Path("scripts"),
|
||||
kythe_dir=Path("scripts/gzs/kythe"),
|
||||
graphdb_url="ws://localhost:8182/gremlin",
|
||||
python="python",
|
||||
) as program_model:
|
||||
if not program_model.process_task_kythe(index_request):
|
||||
yield False
|
||||
|
||||
with Graph(url="ws://localhost:8182/gremlin") as graph:
|
||||
bodies = graph.get_function_body(
|
||||
function_name="png_handle_iCCP", source_path=Path("pngrutil.c")
|
||||
)
|
||||
assert len(bodies) == 2
|
||||
assert b"png_handle_iCCP" in bodies[0]
|
||||
|
||||
cleanup_graphdb(request, libpng_oss_fuzz_task.task_meta.task_id)
|
||||
|
||||
yield True
|
||||
|
||||
|
||||
# @pytest.mark.integration
|
||||
@pytest.mark.skip(reason="Skipping test because we're not using Kythe")
|
||||
def test_libpng_get_function_body(libpng_oss_fuzz_graphml_content: bool):
|
||||
"""Test getting function body from libpng."""
|
||||
assert libpng_oss_fuzz_graphml_content is True
|
||||
Generated
-44
@@ -6,14 +6,6 @@ resolution-markers = [
|
||||
"python_full_version < '3.11'",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "aenum"
|
||||
version = "3.1.16"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/e3/52/6ad8f63ec8da1bf40f96996d25d5b650fdd38f5975f8c813732c47388f18/aenum-3.1.16-py3-none-any.whl", hash = "sha256:9035092855a98e41b66e3d0998bd7b96280e85ceb3a04cc035636138a1943eaf", size = 165627, upload-time = "2025-04-25T03:17:58.89Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "aiohappyeyeballs"
|
||||
version = "2.6.1"
|
||||
@@ -536,22 +528,6 @@ wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/1f/8f/8f9e56c5e82eb2c26e8cde787962e66494312dc8cb261c460e1f3a9c88bc/greenlet-3.2.3-cp312-cp312-win_amd64.whl", hash = "sha256:7454d37c740bb27bdeddfc3f358f26956a07d5220818ceb467a483197d84f849", size = 297817, upload-time = "2025-06-05T16:29:49.244Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "gremlinpython"
|
||||
version = "3.7.3"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "aenum" },
|
||||
{ name = "aiohttp" },
|
||||
{ name = "async-timeout" },
|
||||
{ name = "isodate" },
|
||||
{ name = "nest-asyncio" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/e3/bd/706ef9c0589f2742c9860d877151807ec9b0f1a2904a605502247c9f773c/gremlinpython-3.7.3.tar.gz", hash = "sha256:c8144bd1099fdcf072deff9a15e260efa9a1a065220495f799f6fc1ff6b83ef4", size = 52585, upload-time = "2024-10-29T17:46:39.912Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/c2/64/65c9ab3deb11c5e0607c51fcddff3be33a06720cdd08768375720893e994/gremlinpython-3.7.3-py3-none-any.whl", hash = "sha256:56e0da437be3336971761dfeaab8532f0c583aaab760d8783c4a0bb389cfd341", size = 78436, upload-time = "2024-10-29T17:46:38.344Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "grpcio"
|
||||
version = "1.73.1"
|
||||
@@ -700,15 +676,6 @@ wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/2c/e1/e6716421ea10d38022b952c159d5161ca1193197fb744506875fbb87ea7b/iniconfig-2.1.0-py3-none-any.whl", hash = "sha256:9deba5723312380e77435581c6bf4935c94cbfab9b1ed33ef8d238ea168eb760", size = 6050, upload-time = "2025-03-19T20:10:01.071Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "isodate"
|
||||
version = "0.7.2"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/54/4d/e940025e2ce31a8ce1202635910747e5a87cc3a6a6bb2d00973375014749/isodate-0.7.2.tar.gz", hash = "sha256:4cd1aa0f43ca76f4a6c6c0292a85f40b35ec2e43e315b59f06e6d32171a953e6", size = 29705, upload-time = "2024-10-08T23:04:11.5Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/15/aa/0aca39a37d3c7eb941ba736ede56d689e7be91cab5d9ca846bde3999eba6/isodate-0.7.2-py3-none-any.whl", hash = "sha256:28009937d8031054830160fce6d409ed342816b543597cece116d966c6d99e15", size = 22320, upload-time = "2024-10-08T23:04:09.501Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "jiter"
|
||||
version = "0.10.0"
|
||||
@@ -1027,15 +994,6 @@ wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl", hash = "sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505", size = 4963, upload-time = "2025-04-22T14:54:22.983Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "nest-asyncio"
|
||||
version = "1.6.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/83/f8/51569ac65d696c8ecbee95938f89d4abf00f47d58d48f6fbabfe8f0baefe/nest_asyncio-1.6.0.tar.gz", hash = "sha256:6f172d5449aca15afd6c646851f4e31e02c598d553a667e38cafa997cfec55fe", size = 7418, upload-time = "2024-01-21T14:25:19.227Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/a0/c4/c2971a3ba4c6103a3d10c4b0f24f461ddc027f0f09763220cf35ca1401b3/nest_asyncio-1.6.0-py3-none-any.whl", hash = "sha256:87af6efd6b5e897c81050477ef65c62e2b2f35d51703cae01aff2905b1852e1c", size = 5195, upload-time = "2024-01-21T14:25:17.223Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "numpy"
|
||||
version = "2.2.6"
|
||||
@@ -1374,7 +1332,6 @@ version = "0.1.0"
|
||||
source = { editable = "." }
|
||||
dependencies = [
|
||||
{ name = "common" },
|
||||
{ name = "gremlinpython" },
|
||||
{ name = "protobuf" },
|
||||
{ name = "rapidfuzz" },
|
||||
{ name = "tree-sitter" },
|
||||
@@ -1391,7 +1348,6 @@ dev = [
|
||||
[package.metadata]
|
||||
requires-dist = [
|
||||
{ name = "common", editable = "../common" },
|
||||
{ name = "gremlinpython", specifier = "==3.7.3" },
|
||||
{ name = "protobuf", specifier = ">=3.20,<=3.20.3" },
|
||||
{ name = "rapidfuzz", specifier = "~=3.12.2" },
|
||||
{ name = "tree-sitter", specifier = ">=0.24.0" },
|
||||
|
||||
Reference in New Issue
Block a user