Add support for ends_with/iends_with property value matching in filter code. (#18)

* Create a debug version of the Lobsters nuspec.

* Create UsingMessageAnalyzerToFindETWSources.md

* Add guide for using Message Analyzer

Signed-off-by: Zac Brown <zac@zacbrown.io>

* Update links to images.

Signed-off-by: Zac Brown <zac@zacbrown.io>

* Add link to Message Analyzer howto in README.

* Force usage of TDH to get property size of array types. Should this prove to be a performance penalty in the longterm, we will revisit optimizing for heuristic sizing.

Signed-off-by: Zac Brown (ODSP SECURITY) <zbrown@microsoft.com>

* More general solution for handling when property Flags are set on EVENT_PROPERTY_INFO.

1) If flags are not set, try to get the length from the property 'length' field. Otherwise, try the heuristic.
2) if we couldn't get a length, fall back to calling TDH.

Signed-off-by: Zac Brown (ODSP SECURITY) <zbrown@microsoft.com>

* Add support for EndsWith/IEndsWith in property value matching.

Signed-off-by: Zac Brown (ODSP SECURITY) <zbrown@microsoft.com>

* Fix broken test in GitHub build.

Signed-off-by: Zac Brown (ODSP SECURITY) <zbrown@microsoft.com>

* Add tests for ends_with/iends_with.

Signed-off-by: Zac Brown (ODSP SECURITY) <zbrown@microsoft.com>

O365.Security.Native.ETW.Debug.nuspec

@@ -2,7 +2,7 @@
 <package xmlns="http://schemas.microsoft.com/packaging/2011/08/nuspec.xsd">
     <metadata>
         <id>O365.Security.Native.ETW.Debug</id>
-        <version>1.0.4</version>
+        <version>1.0.5</version>
         <title>O365.Security.Native.ETW Debug - managed wrappers for krabsetw</title>
         <authors>Microsoft Corporation,OneDrive/SharePoint Security</authors>
         <owners>Microsoft Corporation,OneDrive/SharePoint Security</owners>
@@ -11,7 +11,7 @@
         <requireLicenseAcceptance>false</requireLicenseAcceptance>
         <description>O365.Security.Native.ETW Debug is a managed wrapper around the krabsetw ETW library. Also known as "Lobsters." This is the Debug build.</description>
         <summary>O365.Security.Native.ETW Debug is a managed wrapper around the krabsetw ETW library. Also known as "Lobsters." This is the Debug build.</summary>
-        <releaseNotes>Support fixed-length strings in collection_view.</releaseNotes>
+        <releaseNotes>Add support for EndsWith/IEndsWith property value matching.</releaseNotes>
         <copyright>Copyright 2016 Microsoft Corporation</copyright>
         <language />
         <tags>ETW krabs lobsters managed cppcli</tags>

O365.Security.Native.ETW.nuspec

@@ -2,7 +2,7 @@
 <package xmlns="http://schemas.microsoft.com/packaging/2011/08/nuspec.xsd">
     <metadata>
         <id>O365.Security.Native.ETW</id>
-        <version>1.0.4</version>
+        <version>1.0.5</version>
         <title>O365.Security.Native.ETW - managed wrappers for krabsetw</title>
         <authors>Microsoft Corporation,OneDrive/SharePoint Security</authors>
         <owners>Microsoft Corporation,OneDrive/SharePoint Security</owners>
@@ -11,7 +11,7 @@
         <requireLicenseAcceptance>false</requireLicenseAcceptance>
         <description>O365.Security.Native.ETW is a managed wrapper around the krabsetw ETW library. Also known as "Lobsters."</description>
         <summary>O365.Security.Native.ETW is a managed wrapper around the krabsetw ETW library. Also known as "Lobsters."</summary>
-        <releaseNotes>Support fixed-length strings in collection_view.</releaseNotes>
+        <releaseNotes>Add support for EndsWith/IEndsWith property value matching.</releaseNotes>
         <copyright>Copyright 2016 Microsoft Corporation</copyright>
         <language />
         <tags>ETW krabs lobsters managed cppcli</tags>

O365.Security.Native.ETW/Filtering/AnsiString.hpp

@@ -100,5 +100,31 @@ namespace O365 { namespace Security { namespace ETW {
                 msclr::interop::marshal_as<std::wstring>(name),
                 msclr::interop::marshal_as<std::string>(value)));
         }
+
+        /// <summary>
+        /// Accept event if ANSI string property ends with the specified string
+        /// </summary>
+        /// <param name="name">represents the property name</param>
+        /// <param name="value">represents the value to match on</param>
+        /// <returns>a predicate that accepts an event if the value ends with the specified string</returns>
+        static Predicate^ EndsWith(String^ name, String^ value)
+        {
+            return gcnew Predicate(krabs::predicates::property_ends_with<adapt::generic_string<char>>(
+                msclr::interop::marshal_as<std::wstring>(name),
+                msclr::interop::marshal_as<std::string>(value)));
+        }
+
+        /// <summary>
+        /// Accept event if ANSI string property ends with (case invariant) the specified string
+        /// </summary>
+        /// <param name="name">represents the property name</param>
+        /// <param name="value">represents the value to match on</param>
+        /// <returns>a predicate that accepts an event if the value ends with (case invariant) the specified string</returns>
+        static Predicate^ IEndsWith(String^ name, String^ value)
+        {
+            return gcnew Predicate(krabs::predicates::property_iends_with<adapt::generic_string<char>>(
+                msclr::interop::marshal_as<std::wstring>(name),
+                msclr::interop::marshal_as<std::string>(value)));
+        }
     };
 } } }

O365.Security.Native.ETW/Filtering/CountedString.hpp

@@ -100,5 +100,31 @@ namespace O365 { namespace Security { namespace ETW {
                 msclr::interop::marshal_as<std::wstring>(name),
                 msclr::interop::marshal_as<std::wstring>(value)));
         }
+
+        /// <summary>
+        /// Accept event if counted string property ends with the specified string
+        /// </summary>
+        /// <param name="name">represents the property name</param>
+        /// <param name="value">represents the value to match on</param>
+        /// <returns>a predicate that accepts an event if the value ends with the specified string</returns>
+        static Predicate^ EndsWith(String^ name, String^ value)
+        {
+            return gcnew Predicate(krabs::predicates::property_ends_with<adapt::counted_string>(
+                msclr::interop::marshal_as<std::wstring>(name),
+                msclr::interop::marshal_as<std::wstring>(value)));
+        }
+
+        /// <summary>
+        /// Accept event if counted string property ends with (case invariant) the specified string
+        /// </summary>
+        /// <param name="name">represents the property name</param>
+        /// <param name="value">represents the value to match on</param>
+        /// <returns>a predicate that accepts an event if the value ends with (case invariant) the specified string</returns>
+        static Predicate^ IEndsWith(String^ name, String^ value)
+        {
+            return gcnew Predicate(krabs::predicates::property_iends_with<adapt::counted_string>(
+                msclr::interop::marshal_as<std::wstring>(name),
+                msclr::interop::marshal_as<std::wstring>(value)));
+        }
     };
 } } }

O365.Security.Native.ETW/Filtering/UnicodeString.hpp

@@ -100,5 +100,31 @@ namespace O365 { namespace Security { namespace ETW {
                 msclr::interop::marshal_as<std::wstring>(name),
                 msclr::interop::marshal_as<std::wstring>(value)));
         }
+
+        /// <summary>
+        /// Accept event if unicode string property ends with the specified string
+        /// </summary>
+        /// <param name="name">name of the property to match against</param>
+        /// <param name="value">the value to match against</param>
+        /// <returns>a predicate representing that the named property ends with the specified value</returns>
+        static Predicate^ EndsWith(String^ name, String^ value)
+        {
+            return gcnew Predicate(krabs::predicates::property_ends_with(
+                msclr::interop::marshal_as<std::wstring>(name),
+                msclr::interop::marshal_as<std::wstring>(value)));
+        }
+
+        /// <summary>
+        /// Accept event if unicode string property ends with (case invariant) the specified string
+        /// </summary>
+        /// <param name="name">name of the property to match against</param>
+        /// <param name="value">the value to match against</param>
+        /// <returns>a predicate representing that the named property ends with (case invariant) the specified value</returns>
+        static Predicate^ IEndsWith(String^ name, String^ value)
+        {
+            return gcnew Predicate(krabs::predicates::property_iends_with(
+                msclr::interop::marshal_as<std::wstring>(name),
+                msclr::interop::marshal_as<std::wstring>(value)));
+        }
     };
 } } }

krabs/krabs/filtering/comparers.hpp

@@ -71,6 +71,22 @@ namespace krabs { namespace predicates {
             }
         };
 
+        /**
+        * Iterator based ends_with
+        */
+        template <typename Comparer>
+        struct ends_with
+        {
+            template <typename Iter1, typename Iter2>
+            bool operator()(Iter1 begin1, Iter1 end1, Iter2 begin2, Iter2 end2)
+            {
+                auto r1 = boost::make_iterator_range(begin1, end1);
+                auto r2 = boost::make_iterator_range(begin2, end2);
+
+                return boost::ends_with(r1, r2, Comparer());
+            }
+        };
+
         // Custom Comparison
         // --------------------------------------------------------------------
 

krabs/krabs/filtering/predicates.hpp

@@ -385,6 +385,34 @@ namespace krabs { namespace predicates {
         return{ prop, expected, Adapter(), Comparer() };
     }
 
+    /**
+    * Accepts events if property ends with expected value
+    */
+    template <
+        typename Adapter = adapters::generic_string<wchar_t>,
+        typename T,
+        typename Comparer = ends_with<std::equal_to<typename Adapter::value_type>>>
+        details::property_view_predicate<T, Adapter, Comparer> property_ends_with(
+            const std::wstring &prop,
+            const T& expected)
+    {
+        return{ prop, expected, Adapter(), Comparer() };
+    }
+
+    /**
+    * Accepts events if property case insensitive ends with expected value
+    */
+    template <
+        typename Adapter = adapters::generic_string<wchar_t>,
+        typename T,
+        typename Comparer = ends_with<iequal_to<typename Adapter::value_type>>>
+        details::property_view_predicate<T, Adapter, Comparer> property_iends_with(
+            const std::wstring &prop,
+            const T& expected)
+    {
+        return{ prop, expected, Adapter(), Comparer() };
+    }
+
     /**
      * <summary>
      *   Accepts an event if its two component filters both accept the event.

krabsetw.nuspec

@@ -2,7 +2,7 @@
 <package xmlns="http://schemas.microsoft.com/packaging/2011/08/nuspec.xsd">
     <metadata>
         <id>krabsetw</id>
-        <version>1.0.4</version>
+        <version>1.0.5</version>
         <title>Krabs ETW Wrappers</title>
         <authors>Microsoft Corporation,OneDrive/SharePoint Security</authors>
         <owners>Microsoft Corporation,OneDrive/SharePoint Security</owners>
@@ -11,7 +11,7 @@
         <requireLicenseAcceptance>false</requireLicenseAcceptance>
         <description>Krabs ETW provides a modern C++ wrapper around the low-level ETW trace consumption functions</description>
         <summary>Krabs ETW provides a modern C++ wrapper around the low-level ETW trace consumption functions</summary>
-        <releaseNotes>Support fixed-length strings in collection_view.</releaseNotes>
+        <releaseNotes>Add support for ends_with/iends_with property value matching</releaseNotes>
         <copyright>Copyright 2016 Microsoft Corporation</copyright>
         <language />
         <tags>ETW krabs krabsetw native headers cpp</tags>

tests/krabstests/test_filter.cpp

@@ -309,7 +309,10 @@ namespace krabstests
             Assert::IsTrue(filter(record));
         }
 
-        TEST_METHOD(property_istarts_with_should_not_match_counted_string_with_events__that_start_with_expected)
+        /* the ">" unicode character at the start corresponds to 0x003E, or 62. Therefore this function should only compare
+        * the first 31 characters of the string, which ends at "... is 31 chara", and therefore should not match "charac"
+        */
+        TEST_METHOD(property_istarts_with_should_not_match_counted_string_with_events_that_start_with_expected)
         {
             auto filter = krabs::predicates::property_istarts_with<adpt::counted_string>(L"UserData", std::wstring(L"stRING is"));
             Assert::IsFalse(filter(record));
@@ -327,6 +330,66 @@ namespace krabstests
             Assert::IsFalse(filter(record));
         }
 
+        TEST_METHOD(property_ends_with_should_match_properties_that_ends_with_expected)
+        {
+            auto filter = krabs::predicates::property_ends_with(L"ContextInfo", std::wstring(L"baz bingo"));
+            Assert::IsTrue(filter(record));
+        }
+
+        TEST_METHOD(property_ends_with_should_not_match_properties_that_doesnt_ends_with_expected)
+        {
+            auto filter = krabs::predicates::property_ends_with(L"ContextInfo", std::wstring(L"Foo bar"));
+            Assert::IsFalse(filter(record));
+        }
+
+        TEST_METHOD(property_ends_with_should_match_counted_string_properties_that_ends_with_expected)
+        {
+            auto filter = krabs::predicates::property_ends_with<adpt::counted_string>(L"UserData", std::wstring(L"is 31 chara"));
+            Assert::IsTrue(filter(record));
+        }
+
+        TEST_METHOD(property_ends_with_should_not_match_counted_string_properties_that_ends_with_expected)
+        {
+            auto filter = krabs::predicates::property_ends_with<adpt::counted_string>(L"UserData", std::wstring(L"a lot after"));
+            Assert::IsFalse(filter(record));
+        }
+
+        TEST_METHOD(property_ends_with_should_not_match_events_that_go_past_given_counted_string_len)
+        {
+            auto filter = krabs::predicates::property_ends_with<adpt::counted_string>(L"UserData", std::wstring(L"this counted string is 31 charac"));
+            Assert::IsFalse(filter(record));
+        }
+
+        TEST_METHOD(property_iends_with_should_match_events_that_start_with_expected)
+        {
+            auto filter = krabs::predicates::property_iends_with(L"ContextInfo", std::wstring(L"baZ bINgo"));
+            Assert::IsTrue(filter(record));
+        }
+
+        TEST_METHOD(property_iends_with_should_not_match_events_that_do_not_match_expected)
+        {
+            auto filter = krabs::predicates::property_iends_with(L"ContextInfo", std::wstring(L"fOo BAr"));
+            Assert::IsFalse(filter(record));
+        }
+
+        TEST_METHOD(property_iends_with_should_match_counted_string_with_events_that_end_with_expected)
+        {
+            auto filter = krabs::predicates::property_iends_with<adpt::counted_string>(L"UserData", std::wstring(L"iS 31 chaRa"));
+            Assert::IsTrue(filter(record));
+        }
+
+        TEST_METHOD(property_iends_with_should_not_match_counted_string_with_events_that_with_with_expected)
+        {
+            auto filter = krabs::predicates::property_iends_with<adpt::counted_string>(L"UserData", std::wstring(L"a LOT aFteR"));
+            Assert::IsFalse(filter(record));
+        }
+
+        TEST_METHOD(property_iends_with_should_not_match_events_that_go_past_given_counted_string_len)
+        {
+            auto filter = krabs::predicates::property_iends_with<adpt::counted_string>(L"UserData", std::wstring(L"thIS coUNteD stRINg is 31 chArac"));
+            Assert::IsFalse(filter(record));
+        }
+
         TEST_METHOD(and_should_match_an_event_if_both_components_match)
         {
             auto filter = krabs::predicates::and_filter(krabs::predicates::any_event, krabs::predicates::any_event);

tests/krabstests/test_record_builder.cpp

@@ -102,19 +102,19 @@ namespace krabstests
 
         TEST_METHOD(pack_incomplete_should_fill_enough_bytes_for_nonstring_types_when_incomplete)
         {
-            krabs::guid group_policy(L"{AEA1B4FA-97D1-45F2-A64C-4D69FFFD92C9}");
-            krabs::testing::record_builder builder(group_policy, krabs::id(1500), krabs::version(0));
+            krabs::guid wininet(L"{43D1A55C-76D6-4F7E-995C-64C711E5CAFE}");
+            krabs::testing::record_builder builder(wininet, krabs::id(1057), krabs::version(0));
 
             builder.add_properties()
-                (L"SupportInfo2", (unsigned int)3921)
-                (L"DCName", L"www.microsoft.com");
+                (L"URL", "https://microsoft.com")
+                (L"Status", (unsigned int)300);
 
             auto record = builder.pack_incomplete();
             krabs::schema schema(record);
             krabs::parser parser(schema);
 
-            Assert::AreEqual(parser.parse<unsigned int>(L"SupportInfo2"), (unsigned int)3921);
-            Assert::AreEqual(parser.parse<std::wstring>(L"DCName"), std::wstring(L"www.microsoft.com"));
+            Assert::AreEqual(parser.parse<unsigned int>(L"Status"), (unsigned int)300);
+            Assert::AreEqual(parser.parse<std::string>(L"URL"), std::string("https://microsoft.com"));
         }
 
         TEST_METHOD(pack_incomplete_should_correctly_handle_no_set_props)