From cf8c1dde882e28590de19059c1935afd0ee40e5d Mon Sep 17 00:00:00 2001 From: Zac Brown Date: Fri, 24 Mar 2017 14:44:17 -0700 Subject: [PATCH] Add support for ends_with/iends_with property value matching in filter code. (#18) * Create a debug version of the Lobsters nuspec. * Create UsingMessageAnalyzerToFindETWSources.md * Add guide for using Message Analyzer Signed-off-by: Zac Brown * Update links to images. Signed-off-by: Zac Brown * Add link to Message Analyzer howto in README. * Force usage of TDH to get property size of array types. Should this prove to be a performance penalty in the longterm, we will revisit optimizing for heuristic sizing. Signed-off-by: Zac Brown (ODSP SECURITY) * More general solution for handling when property Flags are set on EVENT_PROPERTY_INFO. 1) If flags are not set, try to get the length from the property 'length' field. Otherwise, try the heuristic. 2) if we couldn't get a length, fall back to calling TDH. Signed-off-by: Zac Brown (ODSP SECURITY) * Add support for EndsWith/IEndsWith in property value matching. Signed-off-by: Zac Brown (ODSP SECURITY) * Fix broken test in GitHub build. Signed-off-by: Zac Brown (ODSP SECURITY) * Add tests for ends_with/iends_with. Signed-off-by: Zac Brown (ODSP SECURITY) --- O365.Security.Native.ETW.Debug.nuspec | 4 +- O365.Security.Native.ETW.nuspec | 4 +- .../Filtering/AnsiString.hpp | 26 ++++++++ .../Filtering/CountedString.hpp | 26 ++++++++ .../Filtering/UnicodeString.hpp | 26 ++++++++ krabs/krabs/filtering/comparers.hpp | 16 +++++ krabs/krabs/filtering/predicates.hpp | 28 ++++++++ krabsetw.nuspec | 4 +- tests/krabstests/test_filter.cpp | 65 ++++++++++++++++++- tests/krabstests/test_record_builder.cpp | 12 ++-- 10 files changed, 198 insertions(+), 13 deletions(-) diff --git a/O365.Security.Native.ETW.Debug.nuspec b/O365.Security.Native.ETW.Debug.nuspec index d81ab86..6da321d 100644 --- a/O365.Security.Native.ETW.Debug.nuspec +++ b/O365.Security.Native.ETW.Debug.nuspec @@ -2,7 +2,7 @@ O365.Security.Native.ETW.Debug - 1.0.4 + 1.0.5 O365.Security.Native.ETW Debug - managed wrappers for krabsetw Microsoft Corporation,OneDrive/SharePoint Security Microsoft Corporation,OneDrive/SharePoint Security @@ -11,7 +11,7 @@ false O365.Security.Native.ETW Debug is a managed wrapper around the krabsetw ETW library. Also known as "Lobsters." This is the Debug build. O365.Security.Native.ETW Debug is a managed wrapper around the krabsetw ETW library. Also known as "Lobsters." This is the Debug build. - Support fixed-length strings in collection_view. + Add support for EndsWith/IEndsWith property value matching. Copyright 2016 Microsoft Corporation ETW krabs lobsters managed cppcli diff --git a/O365.Security.Native.ETW.nuspec b/O365.Security.Native.ETW.nuspec index 4abea1f..9e38359 100644 --- a/O365.Security.Native.ETW.nuspec +++ b/O365.Security.Native.ETW.nuspec @@ -2,7 +2,7 @@ O365.Security.Native.ETW - 1.0.4 + 1.0.5 O365.Security.Native.ETW - managed wrappers for krabsetw Microsoft Corporation,OneDrive/SharePoint Security Microsoft Corporation,OneDrive/SharePoint Security @@ -11,7 +11,7 @@ false O365.Security.Native.ETW is a managed wrapper around the krabsetw ETW library. Also known as "Lobsters." O365.Security.Native.ETW is a managed wrapper around the krabsetw ETW library. Also known as "Lobsters." - Support fixed-length strings in collection_view. + Add support for EndsWith/IEndsWith property value matching. Copyright 2016 Microsoft Corporation ETW krabs lobsters managed cppcli diff --git a/O365.Security.Native.ETW/Filtering/AnsiString.hpp b/O365.Security.Native.ETW/Filtering/AnsiString.hpp index aa65b3f..607d8a7 100644 --- a/O365.Security.Native.ETW/Filtering/AnsiString.hpp +++ b/O365.Security.Native.ETW/Filtering/AnsiString.hpp @@ -100,5 +100,31 @@ namespace O365 { namespace Security { namespace ETW { msclr::interop::marshal_as(name), msclr::interop::marshal_as(value))); } + + /// + /// Accept event if ANSI string property ends with the specified string + /// + /// represents the property name + /// represents the value to match on + /// a predicate that accepts an event if the value ends with the specified string + static Predicate^ EndsWith(String^ name, String^ value) + { + return gcnew Predicate(krabs::predicates::property_ends_with>( + msclr::interop::marshal_as(name), + msclr::interop::marshal_as(value))); + } + + /// + /// Accept event if ANSI string property ends with (case invariant) the specified string + /// + /// represents the property name + /// represents the value to match on + /// a predicate that accepts an event if the value ends with (case invariant) the specified string + static Predicate^ IEndsWith(String^ name, String^ value) + { + return gcnew Predicate(krabs::predicates::property_iends_with>( + msclr::interop::marshal_as(name), + msclr::interop::marshal_as(value))); + } }; } } } diff --git a/O365.Security.Native.ETW/Filtering/CountedString.hpp b/O365.Security.Native.ETW/Filtering/CountedString.hpp index b6aa787..661de1b 100644 --- a/O365.Security.Native.ETW/Filtering/CountedString.hpp +++ b/O365.Security.Native.ETW/Filtering/CountedString.hpp @@ -100,5 +100,31 @@ namespace O365 { namespace Security { namespace ETW { msclr::interop::marshal_as(name), msclr::interop::marshal_as(value))); } + + /// + /// Accept event if counted string property ends with the specified string + /// + /// represents the property name + /// represents the value to match on + /// a predicate that accepts an event if the value ends with the specified string + static Predicate^ EndsWith(String^ name, String^ value) + { + return gcnew Predicate(krabs::predicates::property_ends_with( + msclr::interop::marshal_as(name), + msclr::interop::marshal_as(value))); + } + + /// + /// Accept event if counted string property ends with (case invariant) the specified string + /// + /// represents the property name + /// represents the value to match on + /// a predicate that accepts an event if the value ends with (case invariant) the specified string + static Predicate^ IEndsWith(String^ name, String^ value) + { + return gcnew Predicate(krabs::predicates::property_iends_with( + msclr::interop::marshal_as(name), + msclr::interop::marshal_as(value))); + } }; } } } diff --git a/O365.Security.Native.ETW/Filtering/UnicodeString.hpp b/O365.Security.Native.ETW/Filtering/UnicodeString.hpp index 1cd4c4f..a16448d 100644 --- a/O365.Security.Native.ETW/Filtering/UnicodeString.hpp +++ b/O365.Security.Native.ETW/Filtering/UnicodeString.hpp @@ -100,5 +100,31 @@ namespace O365 { namespace Security { namespace ETW { msclr::interop::marshal_as(name), msclr::interop::marshal_as(value))); } + + /// + /// Accept event if unicode string property ends with the specified string + /// + /// name of the property to match against + /// the value to match against + /// a predicate representing that the named property ends with the specified value + static Predicate^ EndsWith(String^ name, String^ value) + { + return gcnew Predicate(krabs::predicates::property_ends_with( + msclr::interop::marshal_as(name), + msclr::interop::marshal_as(value))); + } + + /// + /// Accept event if unicode string property ends with (case invariant) the specified string + /// + /// name of the property to match against + /// the value to match against + /// a predicate representing that the named property ends with (case invariant) the specified value + static Predicate^ IEndsWith(String^ name, String^ value) + { + return gcnew Predicate(krabs::predicates::property_iends_with( + msclr::interop::marshal_as(name), + msclr::interop::marshal_as(value))); + } }; } } } diff --git a/krabs/krabs/filtering/comparers.hpp b/krabs/krabs/filtering/comparers.hpp index 2c92d23..e5107f1 100644 --- a/krabs/krabs/filtering/comparers.hpp +++ b/krabs/krabs/filtering/comparers.hpp @@ -71,6 +71,22 @@ namespace krabs { namespace predicates { } }; + /** + * Iterator based ends_with + */ + template + struct ends_with + { + template + bool operator()(Iter1 begin1, Iter1 end1, Iter2 begin2, Iter2 end2) + { + auto r1 = boost::make_iterator_range(begin1, end1); + auto r2 = boost::make_iterator_range(begin2, end2); + + return boost::ends_with(r1, r2, Comparer()); + } + }; + // Custom Comparison // -------------------------------------------------------------------- diff --git a/krabs/krabs/filtering/predicates.hpp b/krabs/krabs/filtering/predicates.hpp index eec3e2d..dc5cab2 100644 --- a/krabs/krabs/filtering/predicates.hpp +++ b/krabs/krabs/filtering/predicates.hpp @@ -385,6 +385,34 @@ namespace krabs { namespace predicates { return{ prop, expected, Adapter(), Comparer() }; } + /** + * Accepts events if property ends with expected value + */ + template < + typename Adapter = adapters::generic_string, + typename T, + typename Comparer = ends_with>> + details::property_view_predicate property_ends_with( + const std::wstring &prop, + const T& expected) + { + return{ prop, expected, Adapter(), Comparer() }; + } + + /** + * Accepts events if property case insensitive ends with expected value + */ + template < + typename Adapter = adapters::generic_string, + typename T, + typename Comparer = ends_with>> + details::property_view_predicate property_iends_with( + const std::wstring &prop, + const T& expected) + { + return{ prop, expected, Adapter(), Comparer() }; + } + /** * * Accepts an event if its two component filters both accept the event. diff --git a/krabsetw.nuspec b/krabsetw.nuspec index 3957652..6a0d870 100644 --- a/krabsetw.nuspec +++ b/krabsetw.nuspec @@ -2,7 +2,7 @@ krabsetw - 1.0.4 + 1.0.5 Krabs ETW Wrappers Microsoft Corporation,OneDrive/SharePoint Security Microsoft Corporation,OneDrive/SharePoint Security @@ -11,7 +11,7 @@ false Krabs ETW provides a modern C++ wrapper around the low-level ETW trace consumption functions Krabs ETW provides a modern C++ wrapper around the low-level ETW trace consumption functions - Support fixed-length strings in collection_view. + Add support for ends_with/iends_with property value matching Copyright 2016 Microsoft Corporation ETW krabs krabsetw native headers cpp diff --git a/tests/krabstests/test_filter.cpp b/tests/krabstests/test_filter.cpp index a9a0673..61161a1 100644 --- a/tests/krabstests/test_filter.cpp +++ b/tests/krabstests/test_filter.cpp @@ -309,7 +309,10 @@ namespace krabstests Assert::IsTrue(filter(record)); } - TEST_METHOD(property_istarts_with_should_not_match_counted_string_with_events__that_start_with_expected) + /* the ">" unicode character at the start corresponds to 0x003E, or 62. Therefore this function should only compare + * the first 31 characters of the string, which ends at "... is 31 chara", and therefore should not match "charac" + */ + TEST_METHOD(property_istarts_with_should_not_match_counted_string_with_events_that_start_with_expected) { auto filter = krabs::predicates::property_istarts_with(L"UserData", std::wstring(L"stRING is")); Assert::IsFalse(filter(record)); @@ -327,6 +330,66 @@ namespace krabstests Assert::IsFalse(filter(record)); } + TEST_METHOD(property_ends_with_should_match_properties_that_ends_with_expected) + { + auto filter = krabs::predicates::property_ends_with(L"ContextInfo", std::wstring(L"baz bingo")); + Assert::IsTrue(filter(record)); + } + + TEST_METHOD(property_ends_with_should_not_match_properties_that_doesnt_ends_with_expected) + { + auto filter = krabs::predicates::property_ends_with(L"ContextInfo", std::wstring(L"Foo bar")); + Assert::IsFalse(filter(record)); + } + + TEST_METHOD(property_ends_with_should_match_counted_string_properties_that_ends_with_expected) + { + auto filter = krabs::predicates::property_ends_with(L"UserData", std::wstring(L"is 31 chara")); + Assert::IsTrue(filter(record)); + } + + TEST_METHOD(property_ends_with_should_not_match_counted_string_properties_that_ends_with_expected) + { + auto filter = krabs::predicates::property_ends_with(L"UserData", std::wstring(L"a lot after")); + Assert::IsFalse(filter(record)); + } + + TEST_METHOD(property_ends_with_should_not_match_events_that_go_past_given_counted_string_len) + { + auto filter = krabs::predicates::property_ends_with(L"UserData", std::wstring(L"this counted string is 31 charac")); + Assert::IsFalse(filter(record)); + } + + TEST_METHOD(property_iends_with_should_match_events_that_start_with_expected) + { + auto filter = krabs::predicates::property_iends_with(L"ContextInfo", std::wstring(L"baZ bINgo")); + Assert::IsTrue(filter(record)); + } + + TEST_METHOD(property_iends_with_should_not_match_events_that_do_not_match_expected) + { + auto filter = krabs::predicates::property_iends_with(L"ContextInfo", std::wstring(L"fOo BAr")); + Assert::IsFalse(filter(record)); + } + + TEST_METHOD(property_iends_with_should_match_counted_string_with_events_that_end_with_expected) + { + auto filter = krabs::predicates::property_iends_with(L"UserData", std::wstring(L"iS 31 chaRa")); + Assert::IsTrue(filter(record)); + } + + TEST_METHOD(property_iends_with_should_not_match_counted_string_with_events_that_with_with_expected) + { + auto filter = krabs::predicates::property_iends_with(L"UserData", std::wstring(L"a LOT aFteR")); + Assert::IsFalse(filter(record)); + } + + TEST_METHOD(property_iends_with_should_not_match_events_that_go_past_given_counted_string_len) + { + auto filter = krabs::predicates::property_iends_with(L"UserData", std::wstring(L"thIS coUNteD stRINg is 31 chArac")); + Assert::IsFalse(filter(record)); + } + TEST_METHOD(and_should_match_an_event_if_both_components_match) { auto filter = krabs::predicates::and_filter(krabs::predicates::any_event, krabs::predicates::any_event); diff --git a/tests/krabstests/test_record_builder.cpp b/tests/krabstests/test_record_builder.cpp index 9ed2173..d05356d 100644 --- a/tests/krabstests/test_record_builder.cpp +++ b/tests/krabstests/test_record_builder.cpp @@ -102,19 +102,19 @@ namespace krabstests TEST_METHOD(pack_incomplete_should_fill_enough_bytes_for_nonstring_types_when_incomplete) { - krabs::guid group_policy(L"{AEA1B4FA-97D1-45F2-A64C-4D69FFFD92C9}"); - krabs::testing::record_builder builder(group_policy, krabs::id(1500), krabs::version(0)); + krabs::guid wininet(L"{43D1A55C-76D6-4F7E-995C-64C711E5CAFE}"); + krabs::testing::record_builder builder(wininet, krabs::id(1057), krabs::version(0)); builder.add_properties() - (L"SupportInfo2", (unsigned int)3921) - (L"DCName", L"www.microsoft.com"); + (L"URL", "https://microsoft.com") + (L"Status", (unsigned int)300); auto record = builder.pack_incomplete(); krabs::schema schema(record); krabs::parser parser(schema); - Assert::AreEqual(parser.parse(L"SupportInfo2"), (unsigned int)3921); - Assert::AreEqual(parser.parse(L"DCName"), std::wstring(L"www.microsoft.com")); + Assert::AreEqual(parser.parse(L"Status"), (unsigned int)300); + Assert::AreEqual(parser.parse(L"URL"), std::string("https://microsoft.com")); } TEST_METHOD(pack_incomplete_should_correctly_handle_no_set_props)