mirror of
https://github.com/SpecterOps/Nemesis
synced 2026-06-08 12:36:42 +00:00
6ec0a3b61a
* upgrade to dapr postgresv2 statestore * actually make it v2 * dapr state table name, cleanup subscriptions/globals * proper exceptions * formatting/lint * Refactor workflow tracking and improve activity input handling - Extract workflow tracking logic into dedicated WorkflowTrackingService - Simplify activity signatures to accept specific parameters instead of generic dicts - Remove unused asyncio event loop references from enrichment modules - Update YaraRuleManager initialization and method names * re-added workflow tracking in the DB * update uvicorn prod options * enrichment work parallelism, convert queues from broadcast to task queues * Refactor pubsub and improve workflow parallelism - Split Dapr pubsub Dapr yaml components into topic-specific queues (alerting, dotnet, dpapi, files, noseyparker, workflow_monitor) - Update all Dapr volume mounts to reference new topic-specific pubsub components - Converted queues to task queues - Use YAML anchors to reduce duplication for file-enrichment replicas - Pass asyncpg pool to enrichment modules instead of creating connections - Add asyncpg_pool parameter throughout chromium and enrichment module analyzers - Update VSCode workspace (removed InspectAssembly, renamed dotnet_api to dotnet_service) - Added curl commands for Jaeger API to performance docs to help with perf troubleshooting - Created common.queues module to centralize pubsub/topic names (eases future refactoring) * worker mods * Workflow performance tuning, fix pubsub config, CLI arg changes - Fix pubsub deleteWhenUnused typo (deletedWhenUnused) - Add LOG_LEVEL environment variable support across services - CLI: Rename --repeat to --times, add --max-files option - Increase files pubsub prefetchCount from 25 to 50 - Add MAX_PARALLEL_WORKFLOWS configuration - Fix DotNetAssemblyAnalysis null handling with field validators - Update dashboard to show cumulative files/findings over time - Add RUST_LOG environment variable support to noseyparker - Update CHANGELOG for 2.1.4 release notes * Dapr 1.16.2 and use db transactions - Upgrade all Dapr containers from 1.16.1 to 1.16.2 - Reduce enrichment parallelism default from 25 to 5 workflows - Reduce healthcheck intervals from 10s to 5s for alerting and document conversion - Fixed DPAPI eventing to use new pubsubs - Refactor file_linking database operations to use atomic upserts and avoid deadlocks - Add WriteOnceViolationError handling in DPAPI masterkey analyzer - Wrap database operations in transactions for enrichment storage and plaintext indexing - Fix postgres notification handler closure variable capture * remove unused start_time * Scheduler persistence, workflow concurrency tuning, and config cleanup - Add volume for Dapr scheduler and init service - Add scheduler dependency to file enrichment service - Add async workflow client libraries - Format and cleanup compose.yaml (spacing, indentation, empty lines) * Migrate file_enrichment to async Dapr client and optimize Dockerfile - Use async DaprClient where possible in file_enrichment - Improve Dockerfile caching - Add asyncpg connection pool helper and fix typo in secret store name - Include VS Code debug configuration for document_conversion - Remove unused dapr_client from DpapiBlobAnalyzer - Clean up activity return types and better handle exceptions * Enrichment tracking for NoseyParker and logging cleanup - Add workflow_id to NoseyParkerInput and NoseyParkerOutput models - Remove workflow lookup query in noseyparker subscription handler - Adjust jaeger_perf_stats.sh output formatting and precision - Add type hints for async functions * noseyparker scanner perf, tracing for update_enrichment_results --------- Co-authored-by: Lee Chagolla-Christensen <lee@localhost>
129 lines
5.6 KiB
C#
129 lines
5.6 KiB
C#
using Dapr;
|
|
using Dapr.Client;
|
|
using ILSpyDecompilerService.Models;
|
|
using ILSpyDecompilerService.Services;
|
|
using Microsoft.AspNetCore.Mvc;
|
|
using Microsoft.Extensions.Configuration;
|
|
using Microsoft.Extensions.Logging;
|
|
using Newtonsoft.Json;
|
|
using System;
|
|
using System.IO;
|
|
using System.Text.Json;
|
|
using System.Threading;
|
|
using System.Threading.Tasks;
|
|
|
|
namespace ILSpyDecompilerService.Controllers
|
|
{
|
|
[ApiController]
|
|
[Route("[controller]")]
|
|
public class DecompilerController : ControllerBase
|
|
{
|
|
private readonly ILogger<DecompilerController> _logger;
|
|
private readonly DaprClient _daprClient;
|
|
private readonly MinioService _minioService;
|
|
private readonly DecompilerEngine _decompilerEngine;
|
|
private readonly AssemblyAnalysisService _assemblyAnalysisService;
|
|
private readonly SemaphoreSlim _processingSemaphore;
|
|
private const string PubSubName = "dotnet";
|
|
private const string InputTopicName = "dotnet_input";
|
|
private const string OutputTopicName = "dotnet_output";
|
|
|
|
public DecompilerController(
|
|
ILogger<DecompilerController> logger,
|
|
DaprClient daprClient,
|
|
MinioService minioService,
|
|
DecompilerEngine decompilerEngine,
|
|
AssemblyAnalysisService assemblyAnalysisService,
|
|
IConfiguration configuration)
|
|
{
|
|
_logger = logger;
|
|
_daprClient = daprClient;
|
|
_minioService = minioService;
|
|
_decompilerEngine = decompilerEngine;
|
|
_assemblyAnalysisService = assemblyAnalysisService;
|
|
|
|
// Get max concurrent processing from environment variable, default to 5
|
|
var maxConcurrentProcessing = configuration.GetValue<int>("MAX_CONCURRENT_PROCESSING", 5);
|
|
_processingSemaphore = new SemaphoreSlim(maxConcurrentProcessing, maxConcurrentProcessing);
|
|
_logger.LogInformation("Maximum concurrent processing set to: {MaxConcurrentProcessing}", maxConcurrentProcessing);
|
|
}
|
|
|
|
[Topic(PubSubName, InputTopicName)]
|
|
[HttpPost("process")]
|
|
public async Task<IActionResult> ProcessDecompilationRequest([FromBody] InputMessage inputMessage)
|
|
{
|
|
var rawObjectJson = JsonConvert.SerializeObject(inputMessage, Formatting.Indented);
|
|
// _logger.LogDebug("Raw input object: {RawObject}", rawObjectJson);
|
|
|
|
// Wait for semaphore to limit concurrent processing
|
|
await _processingSemaphore.WaitAsync();
|
|
|
|
string downloadedFilePath = null;
|
|
string outputDirectory = null;
|
|
string zipFilePath = null;
|
|
|
|
try
|
|
{
|
|
_logger.LogInformation("Processing decompilation request for object ID: {ObjectId}", inputMessage.ObjectId);
|
|
|
|
if (string.IsNullOrEmpty(inputMessage.ObjectId))
|
|
{
|
|
_logger.LogError("Invalid input message - ObjectId is null or empty");
|
|
return BadRequest("ObjectId is required");
|
|
}
|
|
|
|
var objectId = inputMessage.ObjectId;
|
|
|
|
// Download file from Minio
|
|
downloadedFilePath = await _minioService.DownloadFileAsync(objectId);
|
|
_logger.LogDebug("File downloaded to: {downloadedFilePath}", downloadedFilePath);
|
|
|
|
// Perform assembly analysis
|
|
_logger.LogInformation("Starting assembly analysis for object ID: {ObjectId}", objectId);
|
|
var analysisResult = _assemblyAnalysisService.AnalyzeAssembly(downloadedFilePath);
|
|
_logger.LogDebug("Assembly analysis completed for: {ObjectId}", objectId);
|
|
|
|
// Decompile assembly
|
|
outputDirectory = Path.Combine(Path.GetTempPath(), $"{objectId}_source");
|
|
await _decompilerEngine.DecompileAssemblyAsync(downloadedFilePath, outputDirectory);
|
|
_logger.LogDebug("File decompiled to: {outputDirectory}", outputDirectory);
|
|
|
|
// Create ZIP file
|
|
var newObjectId = Guid.NewGuid().ToString();
|
|
zipFilePath = Path.Combine(Path.GetTempPath(), newObjectId);
|
|
await _decompilerEngine.CreateZipFromDirectoryAsync(outputDirectory, zipFilePath);
|
|
|
|
// Upload ZIP to Minio
|
|
await _minioService.UploadFileAsync(zipFilePath, newObjectId);
|
|
_logger.LogDebug("Zip uploaded to: {newObjectId}", newObjectId);
|
|
|
|
// Publish result with both decompilation and analysis data
|
|
var outputMessage = new OutputMessage
|
|
{
|
|
ObjectId = objectId,
|
|
Decompilation = newObjectId,
|
|
Analysis = analysisResult
|
|
};
|
|
|
|
await _daprClient.PublishEventAsync(PubSubName, OutputTopicName, outputMessage);
|
|
|
|
_logger.LogInformation("Successfully processed decompilation request and published result: {NewObjectId}", newObjectId);
|
|
|
|
return Ok(new { success = true, outputId = newObjectId, analysisResult = analysisResult });
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
_logger.LogError(ex, "Failed to process decompilation request for object ID: {ObjectId}", inputMessage?.ObjectId);
|
|
return StatusCode(500, new { success = false, error = ex.Message });
|
|
}
|
|
finally
|
|
{
|
|
// Cleanup temporary files
|
|
_decompilerEngine.CleanupTemporaryFiles(downloadedFilePath, outputDirectory, zipFilePath);
|
|
|
|
// Release semaphore
|
|
_processingSemaphore.Release();
|
|
}
|
|
}
|
|
}
|
|
} |