From 1bc253968d78b75e1f80e504e797d41f2d875864 Mon Sep 17 00:00:00 2001 From: "Chris Davies (MSTIC)" Date: Tue, 11 Dec 2018 17:37:21 +0000 Subject: [PATCH 1/4] This isn't finished, but I think represents a better way of handling warnings and errors that doesn't treat warnings as errors --- YaraSharp/Compiler.cpp | 29 ++++++++++++++--------------- YaraSharp/Compiler.h | 3 ++- YaraSharp/YaraSharp.cpp | 8 ++++++++ 3 files changed, 24 insertions(+), 16 deletions(-) diff --git a/YaraSharp/Compiler.cpp b/YaraSharp/Compiler.cpp index 009d7ec..57ed47c 100644 --- a/YaraSharp/Compiler.cpp +++ b/YaraSharp/Compiler.cpp @@ -10,7 +10,11 @@ namespace YaraSharp ErrorUtility::ThrowOnError(yr_compiler_create(&TestCompiler)); Compiler = TestCompiler; - Errors = gcnew List(); + // Set up and initialize the error dictionary for errors and warnings + // Which are populated by the callback handler + Errors = gcnew Dictionary^>(); + Errors->Add(YARA_ERROR_LEVEL_ERROR, gcnew List()); + Errors->Add(YARA_ERROR_LEVEL_WARNING, gcnew List()); SetCompilerExternals(ExternalVariables); SetCompilerCallback(); @@ -52,7 +56,11 @@ namespace YaraSharp } List^ CCompiler::GetErrors() { - return this->Errors; + return this->Errors[YARA_ERROR_LEVEL_ERROR]; + } + List^ CCompiler::GetWarnings() + { + return this->Errors[YARA_ERROR_LEVEL_WARNING]; } // Set externals @@ -99,20 +107,11 @@ namespace YaraSharp UNREFERENCED_PARAMETER(UserData); String^ errorLevel; - - if (ErrorLevel == YARA_ERROR_LEVEL_ERROR) - { - errorLevel = "ERROR"; - } - else if (ErrorLevel == YARA_ERROR_LEVEL_WARNING) - { - errorLevel = "WARNING"; - } - - auto msg = String::Format("{0}: {1} on line {2} in file: {3}", - errorLevel, marshal_as(Message), LineNumber, + + auto msg = String::Format("{0} on line {1} in file: {2}", + marshal_as(Message), LineNumber, Filename ? marshal_as(Filename) : "[none]"); - Errors->Add(msg); + Errors[ErrorLevel]->Add(msg); } } \ No newline at end of file diff --git a/YaraSharp/Compiler.h b/YaraSharp/Compiler.h index 1b41407..664255f 100644 --- a/YaraSharp/Compiler.h +++ b/YaraSharp/Compiler.h @@ -8,7 +8,7 @@ namespace YaraSharp public ref class CCompiler sealed { initonly YR_COMPILER* Compiler; - initonly List^ Errors; + initonly Dictionary^>^ Errors; public: CCompiler(Dictionary^ ExternalVariables); @@ -16,6 +16,7 @@ namespace YaraSharp CRules^ GetRules(); List^ GetErrors(); + List^ GetWarnings(); int AddFile(String^ FilePath); void AddFiles(List^ FilePathList); diff --git a/YaraSharp/YaraSharp.cpp b/YaraSharp/YaraSharp.cpp index 4f4ae3c..5babdaa 100644 --- a/YaraSharp/YaraSharp.cpp +++ b/YaraSharp/YaraSharp.cpp @@ -19,6 +19,10 @@ namespace YaraSharp CompilationErrors->Add(FilePathList[i], TestCompiler->GetErrors()); FilePathList->Remove(FilePathList[i--]); } + else if (TestCompiler->GetWarnings()->Count > 0) + { + CompilationErrors->Add(FilePathList[i], TestCompiler->GetWarnings()); + } delete TestCompiler; } @@ -41,6 +45,10 @@ namespace YaraSharp delete TestCompiler; break; } + else if (TestCompiler->GetWarnings()->Count > 0) + { + CompilationErrors->Add(FilePath, TestCompiler->GetWarnings()); + } } } From 6b81bc13c5c8f20533f604e1b4eb4791ed48aa6a Mon Sep 17 00:00:00 2001 From: "Chris Davies (MSTIC)" Date: Tue, 11 Dec 2018 22:28:00 +0000 Subject: [PATCH 2/4] Completed changes --- YaraSharp/Compiler.cpp | 35 ++++++++++++++++++++-------- YaraSharp/Compiler.h | 3 +-- YaraSharp/YaraSharp.cpp | 51 ++++++++++++++++------------------------- 3 files changed, 46 insertions(+), 43 deletions(-) diff --git a/YaraSharp/Compiler.cpp b/YaraSharp/Compiler.cpp index 57ed47c..ec3da2d 100644 --- a/YaraSharp/Compiler.cpp +++ b/YaraSharp/Compiler.cpp @@ -15,7 +15,7 @@ namespace YaraSharp Errors = gcnew Dictionary^>(); Errors->Add(YARA_ERROR_LEVEL_ERROR, gcnew List()); Errors->Add(YARA_ERROR_LEVEL_WARNING, gcnew List()); - + SetCompilerExternals(ExternalVariables); SetCompilerCallback(); } @@ -27,12 +27,15 @@ namespace YaraSharp { FILE* File; Errors->Clear(); + Errors->Add(YARA_ERROR_LEVEL_ERROR, gcnew List()); + Errors->Add(YARA_ERROR_LEVEL_WARNING, gcnew List()); + auto NativePath = marshal_as(FilePath); auto FileOpenError = fopen_s(&File, NativePath.c_str(), "r"); if (FileOpenError) - ErrorUtility::ThrowOnError(String::Format("Îøèáêà îòêðûòèÿ ôàéëà: {0}", FileOpenError)); + ErrorUtility::ThrowOnError(String::Format("Error opening file: {0}", FileOpenError)); auto errors = yr_compiler_add_file(Compiler, File, nullptr, NativePath.c_str()); @@ -40,11 +43,13 @@ namespace YaraSharp return errors; } + void CCompiler::AddFiles(List^ FilePathList) { for each (auto FilePath in FilePathList) AddFile(FilePath); } + CRules^ CCompiler::GetRules() { YR_RULES* Rules; @@ -54,14 +59,18 @@ namespace YaraSharp return gcnew CRules(Rules); } - List^ CCompiler::GetErrors() + + List^ CCompiler::GetErrors(Boolean IncludeWarnings) { + if (IncludeWarnings) + { + List^ errorsAndWarnings = gcnew List(this->Errors[YARA_ERROR_LEVEL_ERROR]); + errorsAndWarnings->AddRange(this->Errors[YARA_ERROR_LEVEL_WARNING]); + return errorsAndWarnings; + } + return this->Errors[YARA_ERROR_LEVEL_ERROR]; } - List^ CCompiler::GetWarnings() - { - return this->Errors[YARA_ERROR_LEVEL_WARNING]; - } // Set externals void CCompiler::SetCompilerExternals(Dictionary^ ExternalVariables) @@ -101,15 +110,21 @@ namespace YaraSharp YR_COMPILER_CALLBACK_FUNC CallbackPointer = (YR_COMPILER_CALLBACK_FUNC)(Marshal::GetFunctionPointerForDelegate(CompilerCallback)).ToPointer(); yr_compiler_set_callback(Compiler, CallbackPointer, NULL); } + void CCompiler::HandleCompilerCallback(int ErrorLevel, const char* Filename, int LineNumber, const char* Message, void* UserData) { UNREFERENCED_PARAMETER(ErrorLevel); UNREFERENCED_PARAMETER(UserData); - String^ errorLevel; + String^ errorLevel = "ERROR"; + + if (ErrorLevel == YARA_ERROR_LEVEL_WARNING) + { + errorLevel = "WARNING"; + } - auto msg = String::Format("{0} on line {1} in file: {2}", - marshal_as(Message), LineNumber, + auto msg = String::Format("{0}: {1} on line {2} in file: {3}", + errorLevel, marshal_as(Message), LineNumber, Filename ? marshal_as(Filename) : "[none]"); Errors[ErrorLevel]->Add(msg); diff --git a/YaraSharp/Compiler.h b/YaraSharp/Compiler.h index 664255f..59a0cd6 100644 --- a/YaraSharp/Compiler.h +++ b/YaraSharp/Compiler.h @@ -15,8 +15,7 @@ namespace YaraSharp ~CCompiler(); CRules^ GetRules(); - List^ GetErrors(); - List^ GetWarnings(); + List^ GetErrors(Boolean IncludeWarnings); int AddFile(String^ FilePath); void AddFiles(List^ FilePathList); diff --git a/YaraSharp/YaraSharp.cpp b/YaraSharp/YaraSharp.cpp index 5babdaa..9d03ac3 100644 --- a/YaraSharp/YaraSharp.cpp +++ b/YaraSharp/YaraSharp.cpp @@ -6,52 +6,41 @@ namespace YaraSharp // Check list of yara files Dictionary^>^ CYaraSharp::CheckYaraRules([Out] List^ FilePathList, Dictionary^ ExternalVariables) { - Dictionary^>^ CompilationErrors = - gcnew Dictionary^>(); + Dictionary^>^ CompilationErrors = gcnew Dictionary^>(); // Iterative check for (int i = 0; i < FilePathList->Count; i++) { CCompiler^ TestCompiler = gcnew CCompiler(ExternalVariables); - if (TestCompiler->AddFile(FilePathList[i]) || TestCompiler->GetErrors()->Count > 0) + if (TestCompiler->AddFile(FilePathList[i]) || TestCompiler->GetErrors(false)->Count > 0) { - CompilationErrors->Add(FilePathList[i], TestCompiler->GetErrors()); + if (CompilationErrors->ContainsKey(FilePathList[i])) + { + CompilationErrors[FilePathList[i]]->AddRange(TestCompiler->GetErrors(false)); + } + else + { + CompilationErrors->Add(FilePathList[i], TestCompiler->GetErrors(false)); + } + FilePathList->Remove(FilePathList[i--]); } - else if (TestCompiler->GetWarnings()->Count > 0) + else if (TestCompiler->GetErrors(true)->Count > 0) { - CompilationErrors->Add(FilePathList[i], TestCompiler->GetWarnings()); + if (CompilationErrors->ContainsKey(FilePathList[i])) + { + CompilationErrors[FilePathList[i]]->AddRange(TestCompiler->GetErrors(true)); + } + else + { + CompilationErrors->Add(FilePathList[i], TestCompiler->GetErrors(true)); + } } delete TestCompiler; } - // Simultaneous check - bool SuccessFlag = false; - while (!SuccessFlag) - { - SuccessFlag = true; - CCompiler^ TestCompiler = gcnew CCompiler(ExternalVariables); - for each (auto FilePath in FilePathList) - { - if (TestCompiler->AddFile(FilePath) || TestCompiler->GetErrors()->Count > 0) - { - CompilationErrors->Add(FilePath, TestCompiler->GetErrors()); - FilePathList->Remove(FilePath); - SuccessFlag = false; - - // New compiler must be created if we fail - delete TestCompiler; - break; - } - else if (TestCompiler->GetWarnings()->Count > 0) - { - CompilationErrors->Add(FilePath, TestCompiler->GetWarnings()); - } - } - } - return CompilationErrors; } From ef6c2298f59d22394cd323fa82a63cdeb958f79a Mon Sep 17 00:00:00 2001 From: "Chris Davies (MSTIC)" Date: Tue, 11 Dec 2018 22:42:59 +0000 Subject: [PATCH 3/4] Removed unnecessary check for errors --- YaraSharp/YaraSharp.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/YaraSharp/YaraSharp.cpp b/YaraSharp/YaraSharp.cpp index 9d03ac3..d1b47fc 100644 --- a/YaraSharp/YaraSharp.cpp +++ b/YaraSharp/YaraSharp.cpp @@ -13,7 +13,7 @@ namespace YaraSharp { CCompiler^ TestCompiler = gcnew CCompiler(ExternalVariables); - if (TestCompiler->AddFile(FilePathList[i]) || TestCompiler->GetErrors(false)->Count > 0) + if (TestCompiler->AddFile(FilePathList[i])) { if (CompilationErrors->ContainsKey(FilePathList[i])) { From adee785997830fe1c722c86973763b5c2a2585a3 Mon Sep 17 00:00:00 2001 From: "Chris Davies (MSTIC)" Date: Tue, 11 Dec 2018 23:24:15 +0000 Subject: [PATCH 4/4] Added unit test project, complete with first unit test --- YaraSharp.sln | 14 ++++ YaraSharpUnitTests/Properties/AssemblyInfo.cs | 20 +++++ .../TestData/CheckWarningStillScans.txt | 1 + .../TestData/CheckWarningStillScans.yar | 10 +++ YaraSharpUnitTests/WarningsAndErrors.cs | 43 ++++++++++ YaraSharpUnitTests/YaraSharpUnitTests.csproj | 81 +++++++++++++++++++ YaraSharpUnitTests/packages.config | 5 ++ 7 files changed, 174 insertions(+) create mode 100644 YaraSharpUnitTests/Properties/AssemblyInfo.cs create mode 100644 YaraSharpUnitTests/TestData/CheckWarningStillScans.txt create mode 100644 YaraSharpUnitTests/TestData/CheckWarningStillScans.yar create mode 100644 YaraSharpUnitTests/WarningsAndErrors.cs create mode 100644 YaraSharpUnitTests/YaraSharpUnitTests.csproj create mode 100644 YaraSharpUnitTests/packages.config diff --git a/YaraSharp.sln b/YaraSharp.sln index d788bac..17d3c7c 100644 --- a/YaraSharp.sln +++ b/YaraSharp.sln @@ -20,6 +20,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "YaraTest", "YaraTest\YaraTest.csproj", "{602143B0-7EA3-4843-8CEE-228BC66F0437}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "YaraSharpUnitTests", "YaraSharpUnitTests\YaraSharpUnitTests.csproj", "{77A34A2C-411B-4958-B553-0151A11BA09B}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -62,6 +64,18 @@ Global {602143B0-7EA3-4843-8CEE-228BC66F0437}.Release|x64.Build.0 = Release|Any CPU {602143B0-7EA3-4843-8CEE-228BC66F0437}.Release|x86.ActiveCfg = Release|Any CPU {602143B0-7EA3-4843-8CEE-228BC66F0437}.Release|x86.Build.0 = Release|Any CPU + {77A34A2C-411B-4958-B553-0151A11BA09B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {77A34A2C-411B-4958-B553-0151A11BA09B}.Debug|Any CPU.Build.0 = Debug|Any CPU + {77A34A2C-411B-4958-B553-0151A11BA09B}.Debug|x64.ActiveCfg = Debug|Any CPU + {77A34A2C-411B-4958-B553-0151A11BA09B}.Debug|x64.Build.0 = Debug|Any CPU + {77A34A2C-411B-4958-B553-0151A11BA09B}.Debug|x86.ActiveCfg = Debug|Any CPU + {77A34A2C-411B-4958-B553-0151A11BA09B}.Debug|x86.Build.0 = Debug|Any CPU + {77A34A2C-411B-4958-B553-0151A11BA09B}.Release|Any CPU.ActiveCfg = Release|Any CPU + {77A34A2C-411B-4958-B553-0151A11BA09B}.Release|Any CPU.Build.0 = Release|Any CPU + {77A34A2C-411B-4958-B553-0151A11BA09B}.Release|x64.ActiveCfg = Release|Any CPU + {77A34A2C-411B-4958-B553-0151A11BA09B}.Release|x64.Build.0 = Release|Any CPU + {77A34A2C-411B-4958-B553-0151A11BA09B}.Release|x86.ActiveCfg = Release|Any CPU + {77A34A2C-411B-4958-B553-0151A11BA09B}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/YaraSharpUnitTests/Properties/AssemblyInfo.cs b/YaraSharpUnitTests/Properties/AssemblyInfo.cs new file mode 100644 index 0000000..ef3a7ce --- /dev/null +++ b/YaraSharpUnitTests/Properties/AssemblyInfo.cs @@ -0,0 +1,20 @@ +using System.Reflection; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +[assembly: AssemblyTitle("YaraSharpUnitTests")] +[assembly: AssemblyDescription("")] +[assembly: AssemblyConfiguration("")] +[assembly: AssemblyCompany("")] +[assembly: AssemblyProduct("YaraSharpUnitTests")] +[assembly: AssemblyCopyright("Copyright © 2018")] +[assembly: AssemblyTrademark("")] +[assembly: AssemblyCulture("")] + +[assembly: ComVisible(false)] + +[assembly: Guid("77a34a2c-411b-4958-b553-0151a11ba09b")] + +// [assembly: AssemblyVersion("1.0.*")] +[assembly: AssemblyVersion("1.0.0.0")] +[assembly: AssemblyFileVersion("1.0.0.0")] diff --git a/YaraSharpUnitTests/TestData/CheckWarningStillScans.txt b/YaraSharpUnitTests/TestData/CheckWarningStillScans.txt new file mode 100644 index 0000000..5de30e5 --- /dev/null +++ b/YaraSharpUnitTests/TestData/CheckWarningStillScans.txt @@ -0,0 +1 @@ +this is a slow regex \ No newline at end of file diff --git a/YaraSharpUnitTests/TestData/CheckWarningStillScans.yar b/YaraSharpUnitTests/TestData/CheckWarningStillScans.yar new file mode 100644 index 0000000..06e0665 --- /dev/null +++ b/YaraSharpUnitTests/TestData/CheckWarningStillScans.yar @@ -0,0 +1,10 @@ +// This file contains a rule that should trigger a warning. This should not prevent scanning from being successful + +rule WarningRule +{ + strings: + $error_str = /this.+?is.+?a.+?slow.+?regex/ + + condition: + $error_str +} diff --git a/YaraSharpUnitTests/WarningsAndErrors.cs b/YaraSharpUnitTests/WarningsAndErrors.cs new file mode 100644 index 0000000..2243a03 --- /dev/null +++ b/YaraSharpUnitTests/WarningsAndErrors.cs @@ -0,0 +1,43 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Reflection; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using YaraSharp; + +namespace YaraSharpUnitTests +{ + [TestClass] + public class WarningsAndErrors + { + private static string BaseDirectory = Environment.CurrentDirectory; + private static string TestDataDirectory = Path.Combine(BaseDirectory, "TestData"); + + /// + /// Ensures warnings are correctly reported along with errors, + /// but does not prevent scanning from working + /// + [TestMethod] + public void CheckWarningsStillScan() + { + string inputFileBase = "CheckWarningStillScans"; + string yaraRuleFile = Path.Combine(TestDataDirectory, $"{inputFileBase}.yar"); + string yaraInputFile = Path.Combine(TestDataDirectory, $"{inputFileBase}.txt"); + + CYaraSharp yaraInstance = new CYaraSharp(); + + using (CContext context = new CContext()) + { + using (CCompiler compiler = new CCompiler(null)) + { + compiler.AddFile(yaraRuleFile); + List compilerErrors = compiler.GetErrors(true); + + CScanner scanner = new CScanner(compiler.GetRules(), null); + Assert.IsTrue(scanner.ScanFile(yaraInputFile).Any(r => r.Rule.Identifier == "WarningRule")); + } + } + } + } +} diff --git a/YaraSharpUnitTests/YaraSharpUnitTests.csproj b/YaraSharpUnitTests/YaraSharpUnitTests.csproj new file mode 100644 index 0000000..4cd73dc --- /dev/null +++ b/YaraSharpUnitTests/YaraSharpUnitTests.csproj @@ -0,0 +1,81 @@ + + + + + + Debug + AnyCPU + {77A34A2C-411B-4958-B553-0151A11BA09B} + Library + Properties + YaraSharpUnitTests + YaraSharpUnitTests + v4.5 + 512 + {3AC096D0-A1C2-E12C-1390-A8335801FDAB};{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC} + 15.0 + $(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion) + $(ProgramFiles)\Common Files\microsoft shared\VSTT\$(VisualStudioVersion)\UITestExtensionPackages + False + UnitTest + + + + + true + full + false + bin\Debug\ + DEBUG;TRACE + prompt + 4 + + + pdbonly + true + bin\Release\ + TRACE + prompt + 4 + + + + ..\packages\MSTest.TestFramework.1.3.2\lib\net45\Microsoft.VisualStudio.TestPlatform.TestFramework.dll + + + ..\packages\MSTest.TestFramework.1.3.2\lib\net45\Microsoft.VisualStudio.TestPlatform.TestFramework.Extensions.dll + + + + + + + + + + + Always + + + + Always + + + + + {c005e3b9-2474-4fea-b5f4-7bb897111e8f} + YaraSharp + + + + + + + + This project references NuGet package(s) that are missing on this computer. Use NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}. + + + + + + \ No newline at end of file diff --git a/YaraSharpUnitTests/packages.config b/YaraSharpUnitTests/packages.config new file mode 100644 index 0000000..d2da0eb --- /dev/null +++ b/YaraSharpUnitTests/packages.config @@ -0,0 +1,5 @@ + + + + + \ No newline at end of file