mirror of
https://github.com/jtang613/IDAssist
synced 2026-08-09 12:43:55 +00:00
Add SymGraph raw binary upload support
This commit is contained in:
@@ -98,6 +98,34 @@ class QueryWorker(QThread):
|
||||
self.query_error.emit(str(e))
|
||||
|
||||
|
||||
class UploadWorker(QThread):
|
||||
"""Worker thread for uploading raw binaries to SymGraph."""
|
||||
|
||||
upload_complete = Signal(str, object)
|
||||
upload_error = Signal(str, str)
|
||||
|
||||
def __init__(self, expected_sha256: str, filename: str, file_bytes: bytes):
|
||||
super().__init__()
|
||||
self.expected_sha256 = expected_sha256
|
||||
self.filename = filename
|
||||
self.file_bytes = file_bytes
|
||||
|
||||
def run(self):
|
||||
try:
|
||||
loop = asyncio.new_event_loop()
|
||||
asyncio.set_event_loop(loop)
|
||||
try:
|
||||
result = loop.run_until_complete(
|
||||
symgraph_service.upload_binary(self.filename, self.file_bytes)
|
||||
)
|
||||
self.upload_complete.emit(self.expected_sha256, result)
|
||||
finally:
|
||||
loop.close()
|
||||
except Exception as e:
|
||||
log.log_error(f"Upload error: {e}")
|
||||
self.upload_error.emit(self.expected_sha256, str(e))
|
||||
|
||||
|
||||
class PushWorker(QThread):
|
||||
"""Worker thread for pushing to SymGraph."""
|
||||
|
||||
@@ -112,7 +140,10 @@ class PushWorker(QThread):
|
||||
graph_data: Optional[Dict] = None,
|
||||
visibility: str = "public",
|
||||
fingerprints: Optional[List[Dict[str, str]]] = None,
|
||||
binary_metadata: Optional[Dict[str, Any]] = None
|
||||
binary_metadata: Optional[Dict[str, Any]] = None,
|
||||
upload_filename: Optional[str] = None,
|
||||
upload_bytes: Optional[bytes] = None,
|
||||
ensure_binary_upload: bool = False
|
||||
):
|
||||
super().__init__()
|
||||
self.sha256 = sha256
|
||||
@@ -122,6 +153,9 @@ class PushWorker(QThread):
|
||||
self.visibility = visibility
|
||||
self.fingerprints = fingerprints or [] # List of {'type': str, 'value': str}
|
||||
self.binary_metadata = dict(binary_metadata or {})
|
||||
self.upload_filename = upload_filename or "binary.bin"
|
||||
self.upload_bytes = upload_bytes
|
||||
self.ensure_binary_upload = ensure_binary_upload
|
||||
|
||||
def run(self):
|
||||
try:
|
||||
@@ -131,6 +165,15 @@ class PushWorker(QThread):
|
||||
total_result = PushResult(success=True)
|
||||
target_revision = None
|
||||
|
||||
if self.ensure_binary_upload:
|
||||
if not self.upload_bytes:
|
||||
raise SymGraphAPIError("Unable to access raw binary bytes for upload")
|
||||
upload_result = loop.run_until_complete(
|
||||
symgraph_service.upload_binary(self.upload_filename, self.upload_bytes)
|
||||
)
|
||||
if (upload_result.sha256 or "").lower() != self.sha256.lower():
|
||||
raise SymGraphAPIError("Uploaded binary SHA256 did not match the active binary")
|
||||
|
||||
if self.symbols or self.graph_data:
|
||||
target_revision = loop.run_until_complete(
|
||||
symgraph_service.create_binary_revision(
|
||||
@@ -685,6 +728,7 @@ class SymGraphController(QObject):
|
||||
# Worker threads
|
||||
self.query_worker = None
|
||||
self.push_worker = None
|
||||
self.upload_worker = None
|
||||
self.pull_worker = None
|
||||
self.apply_worker = None
|
||||
self.document_apply_worker = None
|
||||
@@ -700,6 +744,7 @@ class SymGraphController(QObject):
|
||||
self._pending_apply_documents: List[Dict[str, Any]] = []
|
||||
self._pending_apply_summary: Dict[str, int] = {}
|
||||
self._last_binary_sha: Optional[str] = None
|
||||
self._last_has_stored_binary: Optional[bool] = None
|
||||
self._active_query_sha: Optional[str] = None
|
||||
|
||||
# Connect view signals
|
||||
@@ -712,6 +757,7 @@ class SymGraphController(QObject):
|
||||
def _connect_signals(self):
|
||||
"""Connect view signals to controller methods."""
|
||||
self.view.query_requested.connect(self.handle_query)
|
||||
self.view.upload_binary_requested.connect(self.handle_upload_binary)
|
||||
self.view.auto_refresh_changed.connect(self._set_auto_refresh_enabled)
|
||||
self.view.open_binary_requested.connect(self.handle_open_binary)
|
||||
self.view.push_preview_requested.connect(self.handle_push_preview)
|
||||
@@ -759,6 +805,7 @@ class SymGraphController(QObject):
|
||||
self.view.set_binary_info(name, sha256, local_metadata=local_metadata or None)
|
||||
if sha256 != previous_sha:
|
||||
self._last_binary_sha = sha256
|
||||
self._last_has_stored_binary = None
|
||||
self.view.hide_stats()
|
||||
self.view.reset_query_status()
|
||||
self.view.set_open_binary_url(None)
|
||||
@@ -771,6 +818,7 @@ class SymGraphController(QObject):
|
||||
self.view.set_binary_info("<error>", None, local_metadata=None)
|
||||
else:
|
||||
self._last_binary_sha = None
|
||||
self._last_has_stored_binary = None
|
||||
self.view.set_binary_info("<no binary loaded>", None, local_metadata=None)
|
||||
self.view.hide_stats()
|
||||
self.view.reset_query_status()
|
||||
@@ -905,6 +953,23 @@ class SymGraphController(QObject):
|
||||
"""Get SHA256 hash of the original binary."""
|
||||
return get_binary_hash() or None
|
||||
|
||||
def _build_binary_upload_payload(self) -> Optional[tuple[str, bytes]]:
|
||||
"""Get raw binary bytes and filename for SymGraph upload."""
|
||||
if not _IN_IDA:
|
||||
return None
|
||||
|
||||
input_path = ida_nalt.get_input_file_path() or ""
|
||||
if not input_path or not os.path.exists(input_path):
|
||||
return None
|
||||
|
||||
file_name = self._basename_only(input_path) or "binary.bin"
|
||||
try:
|
||||
with open(input_path, "rb") as f:
|
||||
return file_name, f.read()
|
||||
except OSError as e:
|
||||
log.log_error(f"Error reading raw binary for upload: {e}")
|
||||
return None
|
||||
|
||||
def handle_query(self):
|
||||
"""Handle query request."""
|
||||
sha256 = self._get_sha256()
|
||||
@@ -914,6 +979,7 @@ class SymGraphController(QObject):
|
||||
|
||||
log.log_info(f"Querying SymGraph for: {sha256}")
|
||||
self.view.set_query_status("Checking...")
|
||||
self.view.set_storage_status(None)
|
||||
self.view.hide_stats()
|
||||
self.view.set_open_binary_url(None)
|
||||
self.view.set_buttons_enabled(False)
|
||||
@@ -936,12 +1002,16 @@ class SymGraphController(QObject):
|
||||
return
|
||||
|
||||
if result.error:
|
||||
self._last_has_stored_binary = None
|
||||
self.view.set_query_status(f"Error: {result.error}", found=False)
|
||||
self.view.set_storage_status(None)
|
||||
self.view.set_open_binary_url(None)
|
||||
return
|
||||
|
||||
if result.exists:
|
||||
self._last_has_stored_binary = result.has_stored_binary
|
||||
self.view.set_query_status("Found in SymGraph", found=True)
|
||||
self.view.set_storage_status(result.has_stored_binary)
|
||||
self.view.set_open_binary_url(symgraph_service.get_binary_url(self._get_sha256()))
|
||||
if result.stats:
|
||||
self.view.set_stats(
|
||||
@@ -954,8 +1024,12 @@ class SymGraphController(QObject):
|
||||
latest_revision=result.latest_revision,
|
||||
selected_revision=result.selected_revision
|
||||
)
|
||||
else:
|
||||
self.view.hide_stats()
|
||||
else:
|
||||
self._last_has_stored_binary = None
|
||||
self.view.set_query_status("Not found in SymGraph", found=False)
|
||||
self.view.set_storage_status(None)
|
||||
self.view.set_open_binary_url(None)
|
||||
self.view.hide_stats()
|
||||
|
||||
@@ -963,10 +1037,67 @@ class SymGraphController(QObject):
|
||||
"""Handle query error."""
|
||||
if expected_sha != self._active_query_sha or expected_sha != self._get_sha256():
|
||||
return
|
||||
self._last_has_stored_binary = None
|
||||
self.view.set_query_status(f"Error: {error_msg}", found=False)
|
||||
self.view.set_storage_status(None)
|
||||
self.view.set_open_binary_url(None)
|
||||
log.log_error(f"Query error: {error_msg}")
|
||||
|
||||
def handle_upload_binary(self):
|
||||
"""Upload the raw binary to SymGraph storage."""
|
||||
sha256 = self._get_sha256()
|
||||
if not sha256:
|
||||
self._show_error("No Binary", "No binary loaded or unable to compute hash.")
|
||||
return
|
||||
|
||||
if not symgraph_service.has_api_key:
|
||||
self._show_error(
|
||||
"API Key Required",
|
||||
"Upload requires a SymGraph API key.\n\nAdd your API key in Settings > SymGraph"
|
||||
)
|
||||
return
|
||||
|
||||
payload = self._build_binary_upload_payload()
|
||||
if payload is None:
|
||||
self._show_error(
|
||||
"Upload Unavailable",
|
||||
"IDAssist can only upload binaries when the original input file path is still accessible."
|
||||
)
|
||||
return
|
||||
|
||||
filename, file_bytes = payload
|
||||
self.view.set_buttons_enabled(False)
|
||||
self.view.set_query_status("Uploading raw binary...", found=False)
|
||||
self.view.set_storage_status(None)
|
||||
|
||||
self.upload_worker = UploadWorker(sha256, filename, file_bytes)
|
||||
self.upload_worker.upload_complete.connect(self._on_upload_complete)
|
||||
self.upload_worker.upload_error.connect(self._on_upload_error)
|
||||
self.upload_worker.start()
|
||||
|
||||
def _on_upload_complete(self, expected_sha: str, result):
|
||||
if expected_sha != self._get_sha256():
|
||||
return
|
||||
|
||||
returned_sha = (getattr(result, 'sha256', '') or '').lower()
|
||||
if returned_sha != expected_sha.lower():
|
||||
self._on_upload_error(expected_sha, "Uploaded binary SHA256 did not match the active binary")
|
||||
return
|
||||
|
||||
self._last_has_stored_binary = True
|
||||
self.view.set_buttons_enabled(True)
|
||||
self.view.set_storage_status(True)
|
||||
self.view.set_query_status("Raw binary uploaded", found=True)
|
||||
self.handle_query()
|
||||
|
||||
def _on_upload_error(self, expected_sha: str, error_msg: str):
|
||||
if expected_sha != self._get_sha256():
|
||||
return
|
||||
self.view.set_buttons_enabled(True)
|
||||
self.view.set_query_status(f"Error: {error_msg}", found=False)
|
||||
self.view.set_storage_status(None)
|
||||
log.log_error(f"Upload error: {error_msg}")
|
||||
|
||||
def _on_query_finished(self, expected_sha: str):
|
||||
if expected_sha != self._active_query_sha:
|
||||
return
|
||||
@@ -1087,10 +1218,17 @@ class SymGraphController(QObject):
|
||||
'scope': scope,
|
||||
'push_symbols': bool(selected_symbols),
|
||||
'push_graph': graph_data is not None,
|
||||
'visibility': visibility
|
||||
'visibility': visibility,
|
||||
'ensure_binary_upload': self._last_has_stored_binary is not True,
|
||||
}
|
||||
self._pending_push_documents = list(selected_documents)
|
||||
binary_metadata = self._get_original_binary_metadata()
|
||||
upload_payload = self._build_binary_upload_payload() if self._last_has_stored_binary is not True else None
|
||||
if self._last_has_stored_binary is not True and upload_payload is None:
|
||||
self.view.hide_push_progress()
|
||||
self.view.set_buttons_enabled(True)
|
||||
self.view.set_push_status("Error: unable to access raw binary bytes for upload", success=False)
|
||||
return
|
||||
|
||||
# Start push worker
|
||||
self.push_worker = PushWorker(
|
||||
@@ -1100,7 +1238,10 @@ class SymGraphController(QObject):
|
||||
graph_data,
|
||||
visibility=visibility,
|
||||
fingerprints=fingerprints,
|
||||
binary_metadata=binary_metadata
|
||||
binary_metadata=binary_metadata,
|
||||
upload_filename=upload_payload[0] if upload_payload else None,
|
||||
upload_bytes=upload_payload[1] if upload_payload else None,
|
||||
ensure_binary_upload=self._last_has_stored_binary is not True
|
||||
)
|
||||
self.push_worker.push_complete.connect(self._on_push_complete)
|
||||
self.push_worker.push_error.connect(self._on_push_error)
|
||||
@@ -1113,6 +1254,8 @@ class SymGraphController(QObject):
|
||||
self.view.hide_push_progress()
|
||||
|
||||
if result.success:
|
||||
self._last_has_stored_binary = True
|
||||
self.view.set_storage_status(True)
|
||||
msg_parts = []
|
||||
if result.symbols_pushed > 0:
|
||||
msg_parts.append(f"{result.symbols_pushed} symbols")
|
||||
|
||||
@@ -113,6 +113,28 @@ class BinaryStats:
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class BinaryUploadResult:
|
||||
"""Result of uploading a raw binary to SymGraph."""
|
||||
sha256: str
|
||||
binary_id: str
|
||||
file_size: int
|
||||
is_new: bool = False
|
||||
message: Optional[str] = None
|
||||
metadata_extracted: bool = False
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: Dict[str, Any]) -> 'BinaryUploadResult':
|
||||
return cls(
|
||||
sha256=str(data.get('sha256', '') or ''),
|
||||
binary_id=str(data.get('binary_id', '') or ''),
|
||||
file_size=_parse_int(data.get('file_size'), 0),
|
||||
is_new=bool(data.get('is_new', False)),
|
||||
message=data.get('message'),
|
||||
metadata_extracted=bool(data.get('metadata_extracted', False)),
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class BinaryRevision:
|
||||
"""Accessible binary revision metadata."""
|
||||
@@ -522,6 +544,7 @@ class QueryResult:
|
||||
revisions: List[BinaryRevision] = field(default_factory=list)
|
||||
latest_revision: Optional[int] = None
|
||||
selected_revision: Optional[int] = None
|
||||
has_stored_binary: Optional[bool] = None
|
||||
error: Optional[str] = None
|
||||
|
||||
@classmethod
|
||||
@@ -530,7 +553,8 @@ class QueryResult:
|
||||
stats: Optional[BinaryStats] = None,
|
||||
revisions: Optional[List[BinaryRevision]] = None,
|
||||
latest_revision: Optional[int] = None,
|
||||
selected_revision: Optional[int] = None
|
||||
selected_revision: Optional[int] = None,
|
||||
has_stored_binary: Optional[bool] = None
|
||||
) -> 'QueryResult':
|
||||
"""Create a successful query result."""
|
||||
return cls(
|
||||
@@ -538,7 +562,8 @@ class QueryResult:
|
||||
stats=stats,
|
||||
revisions=revisions or [],
|
||||
latest_revision=latest_revision,
|
||||
selected_revision=selected_revision
|
||||
selected_revision=selected_revision,
|
||||
has_stored_binary=has_stored_binary
|
||||
)
|
||||
|
||||
@classmethod
|
||||
|
||||
@@ -22,7 +22,7 @@ except ImportError:
|
||||
from .settings_service import settings_service
|
||||
from .models.symgraph_models import (
|
||||
BinaryStats, BinaryRevision, Symbol, GraphNode, GraphEdge,
|
||||
SymbolExport, GraphExport, QueryResult, PushResult,
|
||||
SymbolExport, GraphExport, QueryResult, PushResult, BinaryUploadResult,
|
||||
DocumentSummary, Document,
|
||||
PullPreviewResult, ConflictEntry, ConflictAction
|
||||
)
|
||||
@@ -322,6 +322,75 @@ class SymGraphService:
|
||||
except httpx.RequestError as e:
|
||||
raise SymGraphNetworkError(f"Network error: {str(e)}")
|
||||
|
||||
async def get_stored_binary_flag(self, sha256: str) -> Optional[bool]:
|
||||
"""Get whether SymGraph has a stored raw binary for this SHA256."""
|
||||
self._check_httpx()
|
||||
self._check_auth()
|
||||
|
||||
url = f"{self.base_url}/api/v1/binaries/{sha256}"
|
||||
log.log_debug(f"Getting binary info: {url}")
|
||||
|
||||
try:
|
||||
async with httpx.AsyncClient(**self._client_kwargs(30.0)) as client:
|
||||
response = await client.get(
|
||||
url,
|
||||
headers=self._get_headers(authenticated=True)
|
||||
)
|
||||
|
||||
if response.status_code == 200:
|
||||
data = response.json() or {}
|
||||
if isinstance(data, dict):
|
||||
if 'has_stored_binary' in data:
|
||||
return bool(data.get('has_stored_binary'))
|
||||
binary = data.get('binary')
|
||||
if isinstance(binary, dict) and 'has_stored_binary' in binary:
|
||||
return bool(binary.get('has_stored_binary'))
|
||||
return None
|
||||
elif response.status_code == 401:
|
||||
raise SymGraphAuthError("Invalid API key")
|
||||
elif response.status_code == 404:
|
||||
return False
|
||||
else:
|
||||
self._raise_api_error("fetching binary info", response)
|
||||
|
||||
except httpx.TimeoutException:
|
||||
raise SymGraphNetworkError(f"Timeout connecting to {self.base_url}")
|
||||
except httpx.RequestError as e:
|
||||
raise SymGraphNetworkError(f"Network error: {str(e)}")
|
||||
|
||||
async def upload_binary(self, filename: str, file_bytes: bytes) -> BinaryUploadResult:
|
||||
"""Upload raw binary bytes to SymGraph storage without triggering analysis."""
|
||||
self._check_httpx()
|
||||
self._check_auth()
|
||||
|
||||
upload_name = (filename or "").strip() or "binary.bin"
|
||||
url = f"{self.base_url}/api/v1/analysis/upload"
|
||||
log.log_debug(f"Uploading raw binary: {upload_name} ({len(file_bytes)} bytes)")
|
||||
|
||||
try:
|
||||
async with httpx.AsyncClient(**self._client_kwargs(120.0)) as client:
|
||||
response = await client.post(
|
||||
url,
|
||||
headers={
|
||||
'Accept': 'application/json',
|
||||
'User-Agent': 'IDAssist-SymGraph/1.0',
|
||||
'X-API-Key': self.api_key or '',
|
||||
},
|
||||
files={'file': (upload_name, file_bytes, 'application/octet-stream')}
|
||||
)
|
||||
|
||||
if response.status_code == 200:
|
||||
return BinaryUploadResult.from_dict(response.json() or {})
|
||||
elif response.status_code == 401:
|
||||
raise SymGraphAuthError("Invalid API key")
|
||||
else:
|
||||
self._raise_api_error("uploading binary", response)
|
||||
|
||||
except httpx.TimeoutException:
|
||||
raise SymGraphNetworkError(f"Timeout connecting to {self.base_url}")
|
||||
except httpx.RequestError as e:
|
||||
raise SymGraphNetworkError(f"Network error: {str(e)}")
|
||||
|
||||
async def list_binary_versions(self, sha256: str) -> List[BinaryRevision]:
|
||||
"""List accessible binary revisions for a binary."""
|
||||
self._check_httpx()
|
||||
@@ -377,6 +446,12 @@ class SymGraphService:
|
||||
|
||||
revisions: List[BinaryRevision] = []
|
||||
latest_revision: Optional[int] = None
|
||||
has_stored_binary: Optional[bool] = None
|
||||
if self.has_api_key:
|
||||
try:
|
||||
has_stored_binary = await self.get_stored_binary_flag(sha256)
|
||||
except SymGraphServiceError as e:
|
||||
log.log_warn(f"Unable to fetch binary storage state for {sha256[:16]}...: {e}")
|
||||
if include_versions and self.has_api_key:
|
||||
try:
|
||||
revisions = await self.list_binary_versions(sha256)
|
||||
@@ -395,7 +470,8 @@ class SymGraphService:
|
||||
stats=stats,
|
||||
revisions=revisions,
|
||||
latest_revision=latest_revision,
|
||||
selected_revision=effective_version
|
||||
selected_revision=effective_version,
|
||||
has_stored_binary=has_stored_binary
|
||||
)
|
||||
|
||||
except SymGraphServiceError as e:
|
||||
|
||||
@@ -48,6 +48,7 @@ class SymGraphTabView(QWidget):
|
||||
)
|
||||
|
||||
query_requested = Signal()
|
||||
upload_binary_requested = Signal()
|
||||
auto_refresh_changed = Signal(bool)
|
||||
open_binary_requested = Signal()
|
||||
pull_preview_requested = Signal()
|
||||
@@ -143,6 +144,9 @@ class SymGraphTabView(QWidget):
|
||||
self.query_button = QPushButton("Refresh")
|
||||
self.query_button.clicked.connect(self.query_requested.emit)
|
||||
remote_header.addWidget(self.query_button)
|
||||
self.upload_binary_button = QPushButton("Upload Binary")
|
||||
self.upload_binary_button.clicked.connect(self.upload_binary_requested.emit)
|
||||
remote_header.addWidget(self.upload_binary_button)
|
||||
self.open_binary_button = QPushButton("Open in SymGraph")
|
||||
self.open_binary_button.setEnabled(False)
|
||||
self.open_binary_button.clicked.connect(self.open_binary_requested.emit)
|
||||
@@ -152,6 +156,8 @@ class SymGraphTabView(QWidget):
|
||||
self.status_label = QLabel("Use Refresh to check whether this binary already exists in SymGraph.")
|
||||
self.status_label.setWordWrap(True)
|
||||
remote_layout.addWidget(self.status_label)
|
||||
self.storage_status_label = QLabel("Stored Binary: Unknown")
|
||||
self.storage_status_label.setStyleSheet("color: palette(mid);")
|
||||
|
||||
self.stats_frame = QFrame()
|
||||
stats_layout = QGridLayout(self.stats_frame)
|
||||
@@ -171,7 +177,8 @@ class SymGraphTabView(QWidget):
|
||||
stats_layout.addWidget(self.edges_stat, 1, 1)
|
||||
stats_layout.addWidget(self.updated_stat, 2, 0)
|
||||
stats_layout.addWidget(self.latest_revision_stat, 2, 1)
|
||||
stats_layout.addWidget(self.accessible_versions_stat, 3, 0, 1, 2)
|
||||
stats_layout.addWidget(self.accessible_versions_stat, 3, 0)
|
||||
stats_layout.addWidget(self.storage_status_label, 3, 1)
|
||||
self.stats_frame.setVisible(False)
|
||||
remote_layout.addWidget(self.stats_frame)
|
||||
|
||||
@@ -915,6 +922,17 @@ class SymGraphTabView(QWidget):
|
||||
self._open_binary_url = url
|
||||
self.open_binary_button.setEnabled(bool(url))
|
||||
|
||||
def set_storage_status(self, stored: Optional[bool]):
|
||||
if stored is True:
|
||||
self.storage_status_label.setText("Stored Binary: Stored")
|
||||
self.storage_status_label.setStyleSheet("color: green;")
|
||||
elif stored is False:
|
||||
self.storage_status_label.setText("Stored Binary: Missing")
|
||||
self.storage_status_label.setStyleSheet("color: red;")
|
||||
else:
|
||||
self.storage_status_label.setText("Stored Binary: Unknown")
|
||||
self.storage_status_label.setStyleSheet("color: palette(mid);")
|
||||
|
||||
def get_open_binary_url(self) -> Optional[str]:
|
||||
return self._open_binary_url
|
||||
|
||||
@@ -1156,6 +1174,7 @@ class SymGraphTabView(QWidget):
|
||||
def set_buttons_enabled(self, enabled: bool):
|
||||
for button in (
|
||||
self.query_button,
|
||||
self.upload_binary_button,
|
||||
self.open_binary_button,
|
||||
self.pull_preview_button,
|
||||
self.fetch_reset_button,
|
||||
|
||||
Reference in New Issue
Block a user