[Bug] ESQL validation keep Clause Reported Missing Metadata Fields (#5717)

* Update Keep Field to Handle Comments

* Update for handling inline comments

Co-authored-by: Mika Ayenson, PhD <Mikaayenson@users.noreply.github.com>

---------

Co-authored-by: Mika Ayenson, PhD <Mikaayenson@users.noreply.github.com>
Co-authored-by: Jonhnathan <26856693+w0rk3r@users.noreply.github.com>

(cherry picked from commit f74c04d11a)

detection_rules/rule.py

@@ -1000,7 +1000,8 @@ class ESQLRuleData(QueryRuleData):
         # Ensure that keep clause includes metadata fields on non-aggregate queries
         aggregate_pattern = re.compile(r"\|\s*stats\b(?:\s+([^\|]+?))?(?:\s+by\s+([^\|]+))?", re.IGNORECASE | re.DOTALL)
         if not aggregate_pattern.search(query_lower):
-            keep_fields = [field.strip() for field in keep_match.group(1).split(",")]
+            raw_keep = re.sub(r"//.*", "", keep_match.group(1))
+            keep_fields = [field.strip() for field in raw_keep.split(",") if field.strip()]
             if "*" not in keep_fields:
                 required_metadata = {"_id", "_version", "_index"}
                 if not required_metadata.issubset(set(map(str.strip, keep_fields))):

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "detection_rules"
-version = "1.5.42"
+version = "1.5.43"
 description = "Detection Rules is the home for rules used by Elastic Security. This repository is used for the development, maintenance, testing, validation, and release of rules for Elastic Security’s Detection Engine."
 readme = "README.md"
 requires-python = ">=3.12"