Files
SpecterOps-Nemesis/projects/dotnet_service/Controllers/DecompilerController.cs
T
Lee Chagolla-Christensen 6ec0a3b61a Workflow performance improvements (#87)
* upgrade to dapr postgresv2 statestore

* actually make it v2

* dapr state table name, cleanup subscriptions/globals

* proper exceptions

* formatting/lint

* Refactor workflow tracking and improve activity input handling

- Extract workflow tracking logic into dedicated WorkflowTrackingService
- Simplify activity signatures to accept specific parameters instead of generic dicts
- Remove unused asyncio event loop references from enrichment modules
- Update YaraRuleManager initialization and method names

* re-added workflow tracking in the DB

* update uvicorn prod options

* enrichment work parallelism, convert queues from broadcast to task queues

* Refactor pubsub and improve workflow parallelism

- Split Dapr pubsub Dapr yaml components into topic-specific queues (alerting, dotnet, dpapi, files, noseyparker, workflow_monitor)
- Update all Dapr volume mounts to reference new topic-specific pubsub components
- Converted queues to task queues
- Use YAML anchors to reduce duplication for file-enrichment replicas
- Pass asyncpg pool to enrichment modules instead of creating connections
- Add asyncpg_pool parameter throughout chromium and enrichment module analyzers
- Update VSCode workspace (removed InspectAssembly, renamed dotnet_api to dotnet_service)
- Added curl commands for Jaeger API to performance docs to help with perf troubleshooting
- Created common.queues module to centralize pubsub/topic names (eases
  future refactoring)

* worker mods

* Workflow performance tuning, fix pubsub config, CLI arg changes

- Fix pubsub deleteWhenUnused typo (deletedWhenUnused)
- Add LOG_LEVEL environment variable support across services
- CLI: Rename --repeat to --times, add --max-files option
- Increase files pubsub prefetchCount from 25 to 50
- Add MAX_PARALLEL_WORKFLOWS configuration
- Fix DotNetAssemblyAnalysis null handling with field validators
- Update dashboard to show cumulative files/findings over time
- Add RUST_LOG environment variable support to noseyparker
- Update CHANGELOG for 2.1.4 release notes

* Dapr 1.16.2 and use db transactions

- Upgrade all Dapr containers from 1.16.1 to 1.16.2
- Reduce enrichment parallelism default from 25 to 5 workflows
- Reduce healthcheck intervals from 10s to 5s for alerting and document conversion
- Fixed DPAPI eventing to use new pubsubs
- Refactor file_linking database operations to use atomic upserts and avoid deadlocks
- Add WriteOnceViolationError handling in DPAPI masterkey analyzer
- Wrap database operations in transactions for enrichment storage and plaintext indexing
- Fix postgres notification handler closure variable capture

* remove unused start_time

* Scheduler persistence, workflow concurrency tuning, and config cleanup

- Add volume for Dapr scheduler and init service
- Add scheduler dependency to file enrichment service
- Add async workflow client libraries
- Format and cleanup compose.yaml (spacing, indentation, empty lines)

* Migrate file_enrichment to async Dapr client and optimize Dockerfile

- Use async DaprClient where possible in file_enrichment
- Improve Dockerfile caching
- Add asyncpg connection pool helper and fix typo in secret store name
- Include VS Code debug configuration for document_conversion
- Remove unused dapr_client from DpapiBlobAnalyzer
- Clean up activity return types and better handle exceptions

* Enrichment tracking for NoseyParker and logging cleanup

- Add workflow_id to NoseyParkerInput and NoseyParkerOutput models
- Remove workflow lookup query in noseyparker subscription handler
- Adjust jaeger_perf_stats.sh output formatting and precision
- Add type hints for async functions

* noseyparker scanner perf, tracing for update_enrichment_results

---------

Co-authored-by: Lee Chagolla-Christensen <lee@localhost>
2025-11-04 14:06:59 -08:00

129 lines
5.6 KiB
C#

using Dapr;
using Dapr.Client;
using ILSpyDecompilerService.Models;
using ILSpyDecompilerService.Services;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.Logging;
using Newtonsoft.Json;
using System;
using System.IO;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
namespace ILSpyDecompilerService.Controllers
{
[ApiController]
[Route("[controller]")]
public class DecompilerController : ControllerBase
{
private readonly ILogger<DecompilerController> _logger;
private readonly DaprClient _daprClient;
private readonly MinioService _minioService;
private readonly DecompilerEngine _decompilerEngine;
private readonly AssemblyAnalysisService _assemblyAnalysisService;
private readonly SemaphoreSlim _processingSemaphore;
private const string PubSubName = "dotnet";
private const string InputTopicName = "dotnet_input";
private const string OutputTopicName = "dotnet_output";
public DecompilerController(
ILogger<DecompilerController> logger,
DaprClient daprClient,
MinioService minioService,
DecompilerEngine decompilerEngine,
AssemblyAnalysisService assemblyAnalysisService,
IConfiguration configuration)
{
_logger = logger;
_daprClient = daprClient;
_minioService = minioService;
_decompilerEngine = decompilerEngine;
_assemblyAnalysisService = assemblyAnalysisService;
// Get max concurrent processing from environment variable, default to 5
var maxConcurrentProcessing = configuration.GetValue<int>("MAX_CONCURRENT_PROCESSING", 5);
_processingSemaphore = new SemaphoreSlim(maxConcurrentProcessing, maxConcurrentProcessing);
_logger.LogInformation("Maximum concurrent processing set to: {MaxConcurrentProcessing}", maxConcurrentProcessing);
}
[Topic(PubSubName, InputTopicName)]
[HttpPost("process")]
public async Task<IActionResult> ProcessDecompilationRequest([FromBody] InputMessage inputMessage)
{
var rawObjectJson = JsonConvert.SerializeObject(inputMessage, Formatting.Indented);
// _logger.LogDebug("Raw input object: {RawObject}", rawObjectJson);
// Wait for semaphore to limit concurrent processing
await _processingSemaphore.WaitAsync();
string downloadedFilePath = null;
string outputDirectory = null;
string zipFilePath = null;
try
{
_logger.LogInformation("Processing decompilation request for object ID: {ObjectId}", inputMessage.ObjectId);
if (string.IsNullOrEmpty(inputMessage.ObjectId))
{
_logger.LogError("Invalid input message - ObjectId is null or empty");
return BadRequest("ObjectId is required");
}
var objectId = inputMessage.ObjectId;
// Download file from Minio
downloadedFilePath = await _minioService.DownloadFileAsync(objectId);
_logger.LogDebug("File downloaded to: {downloadedFilePath}", downloadedFilePath);
// Perform assembly analysis
_logger.LogInformation("Starting assembly analysis for object ID: {ObjectId}", objectId);
var analysisResult = _assemblyAnalysisService.AnalyzeAssembly(downloadedFilePath);
_logger.LogDebug("Assembly analysis completed for: {ObjectId}", objectId);
// Decompile assembly
outputDirectory = Path.Combine(Path.GetTempPath(), $"{objectId}_source");
await _decompilerEngine.DecompileAssemblyAsync(downloadedFilePath, outputDirectory);
_logger.LogDebug("File decompiled to: {outputDirectory}", outputDirectory);
// Create ZIP file
var newObjectId = Guid.NewGuid().ToString();
zipFilePath = Path.Combine(Path.GetTempPath(), newObjectId);
await _decompilerEngine.CreateZipFromDirectoryAsync(outputDirectory, zipFilePath);
// Upload ZIP to Minio
await _minioService.UploadFileAsync(zipFilePath, newObjectId);
_logger.LogDebug("Zip uploaded to: {newObjectId}", newObjectId);
// Publish result with both decompilation and analysis data
var outputMessage = new OutputMessage
{
ObjectId = objectId,
Decompilation = newObjectId,
Analysis = analysisResult
};
await _daprClient.PublishEventAsync(PubSubName, OutputTopicName, outputMessage);
_logger.LogInformation("Successfully processed decompilation request and published result: {NewObjectId}", newObjectId);
return Ok(new { success = true, outputId = newObjectId, analysisResult = analysisResult });
}
catch (Exception ex)
{
_logger.LogError(ex, "Failed to process decompilation request for object ID: {ObjectId}", inputMessage?.ObjectId);
return StatusCode(500, new { success = false, error = ex.Message });
}
finally
{
// Cleanup temporary files
_decompilerEngine.CleanupTemporaryFiles(downloadedFilePath, outputDirectory, zipFilePath);
// Release semaphore
_processingSemaphore.Release();
}
}
}
}