From 43675dc5562a8a9a40499f65c97c4dd7a7e87d4b Mon Sep 17 00:00:00 2001 From: research-bot Date: Tue, 3 Sep 2024 11:32:03 +0530 Subject: [PATCH 01/17] adding data-sources-dependabot --- .github/workflows/datasource-dependabot.yml | 54 ++++++++++++++++++ .github/workflows/update_data_sources_ta.py | 62 +++++++++++++++++++++ 2 files changed, 116 insertions(+) create mode 100644 .github/workflows/datasource-dependabot.yml create mode 100644 .github/workflows/update_data_sources_ta.py diff --git a/.github/workflows/datasource-dependabot.yml b/.github/workflows/datasource-dependabot.yml new file mode 100644 index 0000000000..8552c6fb45 --- /dev/null +++ b/.github/workflows/datasource-dependabot.yml @@ -0,0 +1,54 @@ +name: Splunk TA Update + +on: + workflow_dispatch: # Manually trigger the workflow + schedule: + - cron: '55 06 * * *' # Runs daily at midnight + +jobs: + modify-code: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + ref: 'develop' + token: ${{ secrets.DATA_SOURCES_DEPENDABOT }} # Add this line to use the PAT for checkout + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' # or the version your script requires + + - name: Install Python Dependencies and ContentCTL and Atomic Red Team + run: | + pip install contentctl>=4.0.0 + git clone --depth=1 --single-branch --branch=master https://github.com/redcanaryco/atomic-red-team.git + + - name: Run ContentCTL Data source TA validation + run: | + contentctl validate --data-source-TA-validation >> data_source_validation.log + + - name: Check for changes + id: changes + run: | + git fetch origin develop + if git diff --exit-code origin/develop -- configs/attack_range_default.yml; then + echo "No changes detected in configs/attack_range_default.yml compared to develop branch" + echo "changes_detected=false" >> $GITHUB_ENV + else + echo "Changes detected in configs/attack_range_default.yml compared to develop branch" + echo "changes_detected=true" >> $GITHUB_ENV + fi + + - name: Create Pull Request + if: env.changes_detected == 'true' + uses: peter-evans/create-pull-request@v6 + with: + token: ${{ secrets.GH_PAT }} + commit-message: Updated TAs + branch: auto-ta-update-${{ github.run_number }} + base: develop + title: Automated Splunk TA Update ${{ github.run_number }} + body: "This PR contains updates to Splunk TAs made by GitHub Actions workflow." \ No newline at end of file diff --git a/.github/workflows/update_data_sources_ta.py b/.github/workflows/update_data_sources_ta.py new file mode 100644 index 0000000000..83239de4bc --- /dev/null +++ b/.github/workflows/update_data_sources_ta.py @@ -0,0 +1,62 @@ +import os +import yaml +from collections import OrderedDict + +# Custom YAML loader to preserve the order of keys +class OrderedLoader(yaml.SafeLoader): + pass + +def construct_mapping(loader, node): + loader.flatten_mapping(node) + return OrderedDict(loader.construct_pairs(node)) + +OrderedLoader.add_constructor( + yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, + construct_mapping +) + +# Custom YAML dumper to preserve the order of keys +class OrderedDumper(yaml.SafeDumper): + pass + +def dict_representer(dumper, data): + return dumper.represent_dict(data.items()) + +OrderedDumper.add_representer(OrderedDict, dict_representer) + +# Define the paths +log_file_path = 'data_source_validation.log' +data_sources_dir = 'data_sources' + +# Read the log file to find version mismatches +with open(log_file_path, 'r') as log_file: + log_lines = log_file.readlines() + +# Parse the log file to find the TA name and the latest version +for i, line in enumerate(log_lines): + if 'Version mismatch' in line: + ta_name = log_lines[i].split("'")[3].strip() + latest_version = log_lines[i + 1].split(':')[1].strip() + print(f"Found version mismatch for TA: {ta_name}, updating to version: {latest_version}") + + # Update the YAML files in the data sources directory + for filename in os.listdir(data_sources_dir): + if filename.endswith('.yml'): + file_path = os.path.join(data_sources_dir, filename) + with open(file_path, 'r') as yml_file: + data = yaml.load(yml_file, Loader=OrderedLoader) + + # Check if the TA name matches and update the version + updated = False + for ta in data.get('supported_TA', []): + if ta['name'] == ta_name: + if ta['version'] != latest_version: + ta['version'] = latest_version + updated = True + + # Write the updated data back to the YAML file + if updated: + with open(file_path, 'w') as yml_file: + yaml.dump(data, yml_file, Dumper=OrderedDumper) + +print("Version updates completed.") \ No newline at end of file From 63d82962a2ba5a6f87c3d044c5b652831a9b7ef2 Mon Sep 17 00:00:00 2001 From: research-bot Date: Tue, 3 Sep 2024 11:36:15 +0530 Subject: [PATCH 02/17] trigger conditions --- .github/workflows/datasource-dependabot.yml | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/.github/workflows/datasource-dependabot.yml b/.github/workflows/datasource-dependabot.yml index 8552c6fb45..b19d83f090 100644 --- a/.github/workflows/datasource-dependabot.yml +++ b/.github/workflows/datasource-dependabot.yml @@ -1,6 +1,9 @@ name: Splunk TA Update on: + push: + branches: + - update_data_sources workflow_dispatch: # Manually trigger the workflow schedule: - cron: '55 06 * * *' # Runs daily at midnight @@ -14,7 +17,7 @@ jobs: uses: actions/checkout@v4 with: ref: 'develop' - token: ${{ secrets.DATA_SOURCES_DEPENDABOT }} # Add this line to use the PAT for checkout + token: ${{ secrets.DATA_SOURCES_DEPENDABOT }} - name: Set up Python uses: actions/setup-python@v5 @@ -27,23 +30,17 @@ jobs: git clone --depth=1 --single-branch --branch=master https://github.com/redcanaryco/atomic-red-team.git - name: Run ContentCTL Data source TA validation + id: validate run: | contentctl validate --data-source-TA-validation >> data_source_validation.log - - name: Check for changes - id: changes + - name: Update Data Sources if Validation Fails + if: failure() run: | - git fetch origin develop - if git diff --exit-code origin/develop -- configs/attack_range_default.yml; then - echo "No changes detected in configs/attack_range_default.yml compared to develop branch" - echo "changes_detected=false" >> $GITHUB_ENV - else - echo "Changes detected in configs/attack_range_default.yml compared to develop branch" - echo "changes_detected=true" >> $GITHUB_ENV - fi + python .github/workflows/update_data_sources_ta.py - name: Create Pull Request - if: env.changes_detected == 'true' + if: failure() # Only create a PR if the validation step failed uses: peter-evans/create-pull-request@v6 with: token: ${{ secrets.GH_PAT }} From 7cf933cd5c452230c251f6d9c2aae5e558a8ace0 Mon Sep 17 00:00:00 2001 From: research-bot Date: Tue, 3 Sep 2024 11:40:50 +0530 Subject: [PATCH 03/17] updates to name --- .github/workflows/datasource-dependabot.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/datasource-dependabot.yml b/.github/workflows/datasource-dependabot.yml index b19d83f090..b01dd24669 100644 --- a/.github/workflows/datasource-dependabot.yml +++ b/.github/workflows/datasource-dependabot.yml @@ -9,7 +9,7 @@ on: - cron: '55 06 * * *' # Runs daily at midnight jobs: - modify-code: + data-source-validation-and-update: runs-on: ubuntu-latest steps: @@ -22,17 +22,19 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: '3.11' # or the version your script requires + python-version: '3.11' + architecture: 'x64' # or the version your script requires - name: Install Python Dependencies and ContentCTL and Atomic Red Team run: | pip install contentctl>=4.0.0 - git clone --depth=1 --single-branch --branch=master https://github.com/redcanaryco/atomic-red-team.git + - name: Run ContentCTL Data source TA validation id: validate run: | contentctl validate --data-source-TA-validation >> data_source_validation.log + continue-on-error: true - name: Update Data Sources if Validation Fails if: failure() From 2061218e7fbd2b1c69bc08915b6a2c9d5ade7844 Mon Sep 17 00:00:00 2001 From: research-bot Date: Tue, 3 Sep 2024 11:46:02 +0530 Subject: [PATCH 04/17] remove > --- .github/workflows/datasource-dependabot.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/datasource-dependabot.yml b/.github/workflows/datasource-dependabot.yml index b01dd24669..de5525891c 100644 --- a/.github/workflows/datasource-dependabot.yml +++ b/.github/workflows/datasource-dependabot.yml @@ -33,7 +33,7 @@ jobs: - name: Run ContentCTL Data source TA validation id: validate run: | - contentctl validate --data-source-TA-validation >> data_source_validation.log + contentctl validate --data-source-TA-validation > data_source_validation.log continue-on-error: true - name: Update Data Sources if Validation Fails From b15a7a10be61477f8eacff1485a339b1445ff840 Mon Sep 17 00:00:00 2001 From: research-bot Date: Tue, 3 Sep 2024 11:55:12 +0530 Subject: [PATCH 05/17] remove output --- .github/workflows/datasource-dependabot.yml | 2 +- contentctl | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) create mode 160000 contentctl diff --git a/.github/workflows/datasource-dependabot.yml b/.github/workflows/datasource-dependabot.yml index de5525891c..132760bcef 100644 --- a/.github/workflows/datasource-dependabot.yml +++ b/.github/workflows/datasource-dependabot.yml @@ -33,7 +33,7 @@ jobs: - name: Run ContentCTL Data source TA validation id: validate run: | - contentctl validate --data-source-TA-validation > data_source_validation.log + contentctl validate --data-source-TA-validation continue-on-error: true - name: Update Data Sources if Validation Fails diff --git a/contentctl b/contentctl new file mode 160000 index 0000000000..b3e7330c2b --- /dev/null +++ b/contentctl @@ -0,0 +1 @@ +Subproject commit b3e7330c2bc71ee5054c8a9bc46f4456c7972d55 From 8a8358a88fa2c6535cec3ad43e41e4fc7cff2223 Mon Sep 17 00:00:00 2001 From: research-bot Date: Tue, 3 Sep 2024 11:58:42 +0530 Subject: [PATCH 06/17] stder --- .github/workflows/datasource-dependabot.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/datasource-dependabot.yml b/.github/workflows/datasource-dependabot.yml index 132760bcef..a920f892e2 100644 --- a/.github/workflows/datasource-dependabot.yml +++ b/.github/workflows/datasource-dependabot.yml @@ -33,7 +33,7 @@ jobs: - name: Run ContentCTL Data source TA validation id: validate run: | - contentctl validate --data-source-TA-validation + contentctl validate --data-source-TA-validation 2>&1 | tee data_source_validation.log continue-on-error: true - name: Update Data Sources if Validation Fails From b8aa86d2cf1185f5f70b3ab6cf2f97e6f4b4fbf4 Mon Sep 17 00:00:00 2001 From: research-bot Date: Tue, 3 Sep 2024 12:01:20 +0530 Subject: [PATCH 07/17] cat cat --- .github/workflows/datasource-dependabot.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/datasource-dependabot.yml b/.github/workflows/datasource-dependabot.yml index a920f892e2..eb16b2b62c 100644 --- a/.github/workflows/datasource-dependabot.yml +++ b/.github/workflows/datasource-dependabot.yml @@ -28,21 +28,18 @@ jobs: - name: Install Python Dependencies and ContentCTL and Atomic Red Team run: | pip install contentctl>=4.0.0 - - name: Run ContentCTL Data source TA validation id: validate run: | contentctl validate --data-source-TA-validation 2>&1 | tee data_source_validation.log - continue-on-error: true + cat data_source_validation.log - name: Update Data Sources if Validation Fails - if: failure() run: | python .github/workflows/update_data_sources_ta.py - name: Create Pull Request - if: failure() # Only create a PR if the validation step failed uses: peter-evans/create-pull-request@v6 with: token: ${{ secrets.GH_PAT }} From 9dbbe3647907714af9e33c1df092f768e84e7599 Mon Sep 17 00:00:00 2001 From: research-bot Date: Tue, 3 Sep 2024 12:09:14 +0530 Subject: [PATCH 08/17] remove last step for debug --- .github/workflows/datasource-dependabot.yml | 26 +++++++++++++-------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/.github/workflows/datasource-dependabot.yml b/.github/workflows/datasource-dependabot.yml index eb16b2b62c..2b6ee60ac1 100644 --- a/.github/workflows/datasource-dependabot.yml +++ b/.github/workflows/datasource-dependabot.yml @@ -32,19 +32,25 @@ jobs: - name: Run ContentCTL Data source TA validation id: validate run: | + pw contentctl validate --data-source-TA-validation 2>&1 | tee data_source_validation.log - cat data_source_validation.log + continue-on-error: true + + - name: Print Validation Log + run: cat data_source_validation.log - name: Update Data Sources if Validation Fails run: | + pwd + ls -la .github/workflows/ python .github/workflows/update_data_sources_ta.py - - name: Create Pull Request - uses: peter-evans/create-pull-request@v6 - with: - token: ${{ secrets.GH_PAT }} - commit-message: Updated TAs - branch: auto-ta-update-${{ github.run_number }} - base: develop - title: Automated Splunk TA Update ${{ github.run_number }} - body: "This PR contains updates to Splunk TAs made by GitHub Actions workflow." \ No newline at end of file + # - name: Create Pull Request + # uses: peter-evans/create-pull-request@v6 + # with: + # token: ${{ secrets.GH_PAT }} + # commit-message: Updated TAs + # branch: auto-ta-update-${{ github.run_number }} + # base: develop + # title: Automated Splunk TA Update ${{ github.run_number }} + # body: "This PR contains updates to Splunk TAs made by GitHub Actions workflow." \ No newline at end of file From 9f11d1adee3997064468d757aacea1cd8bb550d7 Mon Sep 17 00:00:00 2001 From: research-bot Date: Tue, 3 Sep 2024 12:12:16 +0530 Subject: [PATCH 09/17] pwd --- .github/workflows/datasource-dependabot.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/datasource-dependabot.yml b/.github/workflows/datasource-dependabot.yml index 2b6ee60ac1..4053e08728 100644 --- a/.github/workflows/datasource-dependabot.yml +++ b/.github/workflows/datasource-dependabot.yml @@ -32,7 +32,7 @@ jobs: - name: Run ContentCTL Data source TA validation id: validate run: | - pw + pwd contentctl validate --data-source-TA-validation 2>&1 | tee data_source_validation.log continue-on-error: true From 67f00707ad5983fb3c2459feb4a75e45d0a3922c Mon Sep 17 00:00:00 2001 From: research-bot Date: Tue, 3 Sep 2024 12:14:02 +0530 Subject: [PATCH 10/17] branch name shenanigans --- .github/workflows/datasource-dependabot.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/datasource-dependabot.yml b/.github/workflows/datasource-dependabot.yml index 4053e08728..261b649566 100644 --- a/.github/workflows/datasource-dependabot.yml +++ b/.github/workflows/datasource-dependabot.yml @@ -16,7 +16,7 @@ jobs: - name: Checkout repository uses: actions/checkout@v4 with: - ref: 'develop' + ref: 'update_data_sources' token: ${{ secrets.DATA_SOURCES_DEPENDABOT }} - name: Set up Python From 9d619417b2038100dde9f7c3ced122ddeed72328 Mon Sep 17 00:00:00 2001 From: research-bot Date: Tue, 3 Sep 2024 12:18:41 +0530 Subject: [PATCH 11/17] git status --- .github/workflows/datasource-dependabot.yml | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/.github/workflows/datasource-dependabot.yml b/.github/workflows/datasource-dependabot.yml index 261b649566..59a9ecb1dd 100644 --- a/.github/workflows/datasource-dependabot.yml +++ b/.github/workflows/datasource-dependabot.yml @@ -44,13 +44,14 @@ jobs: pwd ls -la .github/workflows/ python .github/workflows/update_data_sources_ta.py + git status - # - name: Create Pull Request - # uses: peter-evans/create-pull-request@v6 - # with: - # token: ${{ secrets.GH_PAT }} - # commit-message: Updated TAs - # branch: auto-ta-update-${{ github.run_number }} - # base: develop - # title: Automated Splunk TA Update ${{ github.run_number }} - # body: "This PR contains updates to Splunk TAs made by GitHub Actions workflow." \ No newline at end of file + - name: Create Pull Request + uses: peter-evans/create-pull-request@v6 + with: + token: ${{ secrets.DATA_SOURCES_DEPENDABOT }} + commit-message: Updated TAs + branch: auto-ta-update-${{ github.run_number }} + base: develop + title: Automated Splunk TA Update ${{ github.run_number }} + body: "This PR contains updates to Splunk TAs made by GitHub Actions workflow." \ No newline at end of file From 18eccfc430ebe17134f1f285c846e2b0db546e32 Mon Sep 17 00:00:00 2001 From: research-bot Date: Tue, 3 Sep 2024 12:23:58 +0530 Subject: [PATCH 12/17] adding only datasources --- .github/workflows/datasource-dependabot.yml | 4 ++- .github/workflows/datasource-ta-check.yml | 27 --------------------- 2 files changed, 3 insertions(+), 28 deletions(-) delete mode 100644 .github/workflows/datasource-ta-check.yml diff --git a/.github/workflows/datasource-dependabot.yml b/.github/workflows/datasource-dependabot.yml index 59a9ecb1dd..115235f5fb 100644 --- a/.github/workflows/datasource-dependabot.yml +++ b/.github/workflows/datasource-dependabot.yml @@ -54,4 +54,6 @@ jobs: branch: auto-ta-update-${{ github.run_number }} base: develop title: Automated Splunk TA Update ${{ github.run_number }} - body: "This PR contains updates to Splunk TAs made by GitHub Actions workflow." \ No newline at end of file + body: "This PR contains updates to Splunk TAs made by GitHub Actions workflow." + paths: | + security_content/data_sources/** \ No newline at end of file diff --git a/.github/workflows/datasource-ta-check.yml b/.github/workflows/datasource-ta-check.yml deleted file mode 100644 index 20e50d5841..0000000000 --- a/.github/workflows/datasource-ta-check.yml +++ /dev/null @@ -1,27 +0,0 @@ -name: datasource-ta-check -on: - pull_request_target: - push: - branches: - - develop - -jobs: - datasource-ta-check: - runs-on: ubuntu-latest - continue-on-error: true - steps: - - name: Check out the repository code - uses: actions/checkout@v4 - - - uses: actions/setup-python@v5 - with: - python-version: '3.11' - architecture: 'x64' - - - name: Install Python Dependencies and contentctl - run: | - pip install contentctl>=4.0.0 - - - name: Run datasource TA check - run: | - contentctl validate --data-source-TA-validation \ No newline at end of file From d7e01bd8b9f9a78bdf0d7e517352bee3916a8144 Mon Sep 17 00:00:00 2001 From: research-bot Date: Tue, 3 Sep 2024 12:29:47 +0530 Subject: [PATCH 13/17] remove an added file --- .github/workflows/datasource-dependabot.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/datasource-dependabot.yml b/.github/workflows/datasource-dependabot.yml index 115235f5fb..730fbde68d 100644 --- a/.github/workflows/datasource-dependabot.yml +++ b/.github/workflows/datasource-dependabot.yml @@ -27,7 +27,7 @@ jobs: - name: Install Python Dependencies and ContentCTL and Atomic Red Team run: | - pip install contentctl>=4.0.0 + pip install "contentctl>=4.0.0" - name: Run ContentCTL Data source TA validation id: validate @@ -37,12 +37,12 @@ jobs: continue-on-error: true - name: Print Validation Log - run: cat data_source_validation.log + run: | + cat data_source_validation.log + rm -f =4.0.0 - name: Update Data Sources if Validation Fails run: | - pwd - ls -la .github/workflows/ python .github/workflows/update_data_sources_ta.py git status From 9d889a265726a37e9f591b3c8487bfdae4f3e4fd Mon Sep 17 00:00:00 2001 From: research-bot Date: Tue, 3 Sep 2024 12:30:50 +0530 Subject: [PATCH 14/17] indent --- .github/workflows/datasource-dependabot.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/datasource-dependabot.yml b/.github/workflows/datasource-dependabot.yml index 730fbde68d..6a6ecbf1c9 100644 --- a/.github/workflows/datasource-dependabot.yml +++ b/.github/workflows/datasource-dependabot.yml @@ -38,8 +38,8 @@ jobs: - name: Print Validation Log run: | - cat data_source_validation.log - rm -f =4.0.0 + cat data_source_validation.log + rm -f =4.0.0 - name: Update Data Sources if Validation Fails run: | From aa403af48438b5401039108154e7e686ba44f16c Mon Sep 17 00:00:00 2001 From: research-bot Date: Tue, 3 Sep 2024 12:35:00 +0530 Subject: [PATCH 15/17] remove prints and change brnach name --- .github/workflows/datasource-dependabot.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/datasource-dependabot.yml b/.github/workflows/datasource-dependabot.yml index 6a6ecbf1c9..b61cf6cdde 100644 --- a/.github/workflows/datasource-dependabot.yml +++ b/.github/workflows/datasource-dependabot.yml @@ -1,9 +1,6 @@ name: Splunk TA Update on: - push: - branches: - - update_data_sources workflow_dispatch: # Manually trigger the workflow schedule: - cron: '55 06 * * *' # Runs daily at midnight @@ -16,7 +13,7 @@ jobs: - name: Checkout repository uses: actions/checkout@v4 with: - ref: 'update_data_sources' + ref: 'develop' token: ${{ secrets.DATA_SOURCES_DEPENDABOT }} - name: Set up Python From eeb8a932ac07e6ba1358f57cc64d44f933019a30 Mon Sep 17 00:00:00 2001 From: research-bot Date: Tue, 3 Sep 2024 18:50:47 +0530 Subject: [PATCH 16/17] rm contentctl --- contentctl | 1 - 1 file changed, 1 deletion(-) delete mode 160000 contentctl diff --git a/contentctl b/contentctl deleted file mode 160000 index b3e7330c2b..0000000000 --- a/contentctl +++ /dev/null @@ -1 +0,0 @@ -Subproject commit b3e7330c2bc71ee5054c8a9bc46f4456c7972d55 From df72a5e60520be4077d273167574d3e27372f11e Mon Sep 17 00:00:00 2001 From: research-bot Date: Wed, 4 Sep 2024 00:29:03 +0530 Subject: [PATCH 17/17] trigger