mirror of
https://github.com/splunk/security_content
synced 2026-06-08 17:32:49 +00:00
Merge pull request #2323 from splunk/eliminate_duplicate_python_requirements_files
Eliminate duplicate python requirements files
This commit is contained in:
Bhavin Patel
@@ -273,16 +273,16 @@ jobs:
|
||||
run: |
|
||||
#Get the virtualenv set up
|
||||
rm -rf venv
|
||||
python3 -m venv --clear venv
|
||||
python -m venv --clear venv
|
||||
source venv/bin/activate
|
||||
python3 -m pip install --upgrade pip
|
||||
python3 -m pip install wheel
|
||||
python3 -m pip install -q -r requirements.txt
|
||||
python -m pip install --upgrade pip
|
||||
python -m pip install wheel
|
||||
python -m pip install -q -r requirements.txt
|
||||
|
||||
- name: run reporting
|
||||
run: |
|
||||
source venv/bin/activate
|
||||
python3 contentctl.py -p . reporting
|
||||
python contentctl.py -p . reporting
|
||||
|
||||
#Official, Verified Amazon-AWS Github Account Provided Action
|
||||
- uses: aws-actions/configure-aws-credentials@v1
|
||||
@@ -329,11 +329,11 @@ jobs:
|
||||
run: |
|
||||
#Get the virtualenv set up
|
||||
rm -rf venv
|
||||
python3 -m venv --clear venv
|
||||
python -m venv --clear venv
|
||||
source venv/bin/activate
|
||||
python3 -m pip install --upgrade pip
|
||||
python3 -m pip install wheel
|
||||
python3 -m pip install -q -r requirements.txt
|
||||
python -m pip install --upgrade pip
|
||||
python -m pip install wheel
|
||||
python -m pip install -q -r requirements.txt
|
||||
|
||||
- name: Get branch and PR required for detection testing main.py
|
||||
id: vars
|
||||
@@ -343,7 +343,7 @@ jobs:
|
||||
- name: Run reporting
|
||||
run: |
|
||||
source venv/bin/activate
|
||||
python3 contentctl.py -p . reporting
|
||||
python contentctl.py -p . reporting
|
||||
|
||||
- name: Update github with new docs and package bits
|
||||
run: |
|
||||
@@ -452,16 +452,16 @@ jobs:
|
||||
run: |
|
||||
#Get the virtualenv set up
|
||||
rm -rf venv
|
||||
python3 -m venv --clear venv
|
||||
python -m venv --clear venv
|
||||
source venv/bin/activate
|
||||
python3 -m pip install --upgrade pip
|
||||
python3 -m pip install wheel
|
||||
python3 -m pip install -q -r requirements.txt
|
||||
python -m pip install --upgrade pip
|
||||
python -m pip install wheel
|
||||
python -m pip install -q -r requirements.txt
|
||||
|
||||
- name: Create YML to JSON Folder
|
||||
run: |
|
||||
source venv/bin/activate
|
||||
python3 contentctl.py --path . generate --product API --output dist/api
|
||||
python contentctl.py --path . generate --product API --output dist/api
|
||||
|
||||
- name: Generate content version and timestamp JSON
|
||||
run : |
|
||||
|
||||
@@ -23,11 +23,11 @@ jobs:
|
||||
run: |
|
||||
#Get the virtualenv set up
|
||||
rm -rf venv
|
||||
python3 -m venv --clear venv
|
||||
python -m venv --clear venv
|
||||
source venv/bin/activate
|
||||
python3 -m pip install --upgrade pip
|
||||
python3 -m pip install wheel
|
||||
python3 -m pip install -q -r requirements.txt
|
||||
python -m pip install --upgrade pip
|
||||
python -m pip install wheel
|
||||
python -m pip install -q -r requirements.txt
|
||||
|
||||
- name: test code with pytest
|
||||
run: |
|
||||
|
||||
@@ -58,20 +58,20 @@ jobs:
|
||||
- uses: actions/setup-python@v2
|
||||
with:
|
||||
python-version: '3.9' #Available versions here - https://github.com/actions/python-versions/releases easy to change/make a matrix/use pypy
|
||||
architecture: 'x64' # optional x64 or x86. Defaults to x64 if not specified
|
||||
architecture: 'x64' # optional x64 or x86. Defaults to x64 if not specified
|
||||
cache: 'pip'
|
||||
|
||||
- name: Install Python Dependencies
|
||||
run: |
|
||||
cd bin/docker_detection_tester
|
||||
python3 -m venv .venv
|
||||
python -m venv .venv
|
||||
source .venv/bin/activate
|
||||
python3 -m pip install wheel
|
||||
python3 -m pip install -r requirements.txt
|
||||
python -m pip install wheel
|
||||
python -m pip install -r requirements.txt
|
||||
|
||||
- name: Run the CI
|
||||
run: |
|
||||
cd bin/docker_detection_tester
|
||||
source .venv/bin/activate
|
||||
cd bin/docker_detection_tester
|
||||
echo "github.event.issue.pull_request : [${{ github.event.issue.pull_request }}]"
|
||||
echo "github.event.pull_request.number : [${{ github.event.pull_request.number }}]"
|
||||
echo "steps.vars.outputs.branch : [${{ steps.vars.outputs.branch }}]"
|
||||
@@ -152,22 +152,23 @@ jobs:
|
||||
- uses: actions/setup-python@v2
|
||||
with:
|
||||
python-version: '3.9' #Available versions here - https://github.com/actions/python-versions/releases easy to change/make a matrix/use pypy
|
||||
architecture: 'x64' # optional x64 or x86. Defaults to x64 if not specified
|
||||
architecture: 'x64' # optional x64 or x86. Defaults to x64 if not specified
|
||||
cache: 'pip'
|
||||
|
||||
- name: Install Python Dependencies
|
||||
run: |
|
||||
cd bin/docker_detection_tester
|
||||
python3 -m venv .venv
|
||||
python -m venv .venv
|
||||
source .venv/bin/activate
|
||||
python3 -m pip install wheel
|
||||
python3 -m pip install -r requirements.txt
|
||||
python -m pip install wheel
|
||||
python -m pip install -r requirements.txt
|
||||
|
||||
- name: Run the CI
|
||||
run: |
|
||||
cd bin/docker_detection_tester
|
||||
source .venv/bin/activate
|
||||
cd bin/docker_detection_tester
|
||||
|
||||
python3 detection_testing_execution.py run -c prior_config/${{ matrix.manifest_filename}}
|
||||
|
||||
python detection_testing_execution.py run -c prior_config/${{ matrix.manifest_filename}}
|
||||
|
||||
|
||||
- name: Upload Test Results Files
|
||||
@@ -255,20 +256,20 @@ jobs:
|
||||
- uses: actions/setup-python@v2
|
||||
with:
|
||||
python-version: '3.9' #Available versions here - https://github.com/actions/python-versions/releases easy to change/make a matrix/use pypy
|
||||
architecture: 'x64' # optional x64 or x86. Defaults to x64 if not specified
|
||||
architecture: 'x64' # optional x64 or x86. Defaults to x64 if not specified
|
||||
cache: 'pip'
|
||||
|
||||
- name: Install Python Dependencies
|
||||
run: |
|
||||
cd bin/docker_detection_tester
|
||||
python3 -m venv .venv
|
||||
python -m venv .venv
|
||||
source .venv/bin/activate
|
||||
python3 -m pip install wheel
|
||||
python3 -m pip install -r requirements.txt
|
||||
python -m pip install wheel
|
||||
python -m pip install -r requirements.txt
|
||||
|
||||
- name: Merge Detections into single File
|
||||
run: |
|
||||
cd bin/docker_detection_tester
|
||||
source .venv/bin/activate
|
||||
cd bin/docker_detection_tester
|
||||
python summarize_json.py --files results_*/summary.json --output_filename summary_test_results.json
|
||||
|
||||
|
||||
@@ -340,4 +341,4 @@ jobs:
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -1,16 +0,0 @@
|
||||
FROM ubuntu:18.04
|
||||
|
||||
RUN apt-get update
|
||||
RUN DEBIAN_FRONTEND="noninteractive" apt-get -y install tzdata
|
||||
RUN apt-get install -y python3-dev git python-dev unzip python3-pip awscli
|
||||
RUN apt-get install -y python-gitdb
|
||||
RUN apt-get install -y wget unzip
|
||||
RUN apt-get install -y git
|
||||
|
||||
ADD . /app
|
||||
|
||||
WORKDIR /app
|
||||
RUN pip3 install -r requirements.txt
|
||||
|
||||
ENTRYPOINT ["python3", "detection_testing_execution.py"]
|
||||
CMD ["-b", "automated_detections_testing_2"]
|
||||
@@ -173,14 +173,9 @@ def generate_escu_app(persist_security_content: bool = False) -> str:
|
||||
print("****GENERATING ESCU APP****")
|
||||
os.chdir("security_content")
|
||||
if persist_security_content is False:
|
||||
commands = ["python3 -m venv .venv",
|
||||
". ./.venv/bin/activate",
|
||||
"python -m pip install wheel",
|
||||
"python -m pip install -r requirements.txt",
|
||||
"python3 ../../../contentctl.py --path . --skip_enrichment generate --product ESCU --output dist/escu"]
|
||||
commands = ["python ../../../contentctl.py --path . --skip_enrichment generate --product ESCU --output dist/escu"]
|
||||
else:
|
||||
commands = [". ./.venv/bin/activate",
|
||||
"python3 ../../../contentctl.py --path . --skip_enrichment generate --product ESCU --output dist/escu"]
|
||||
commands = ["python ../../../contentctl.py --path . --skip_enrichment generate --product ESCU --output dist/escu"]
|
||||
ret = subprocess.run("; ".join(commands),
|
||||
shell=True, capture_output=True)
|
||||
if ret.returncode != 0:
|
||||
@@ -199,7 +194,7 @@ def generate_escu_app(persist_security_content: bool = False) -> str:
|
||||
output_file_path_from_slim_latest = os.path.join(
|
||||
"upload", output_file_name)
|
||||
output_file_path_from_security_content = os.path.join(
|
||||
"slim_packaging", "slim_latest", output_file_path_from_slim_latest)
|
||||
"slim_packaging", output_file_path_from_slim_latest)
|
||||
output_file_path_from_root = os.path.join(
|
||||
"security_content", output_file_path_from_security_content)
|
||||
|
||||
@@ -215,42 +210,17 @@ def generate_escu_app(persist_security_content: bool = False) -> str:
|
||||
sys.exit(1)
|
||||
|
||||
# There remove the latest file if it exists
|
||||
commands = ["cd slim_packaging/slim_latest",
|
||||
". ./.venv/bin/activate",
|
||||
"cp -R ../../dist/escu DA-ESS-ContentUpdate",
|
||||
commands = ["cd slim_packaging",
|
||||
"cp -R ../dist/escu DA-ESS-ContentUpdate",
|
||||
"slim package -o upload DA-ESS-ContentUpdate",
|
||||
"cp upload/DA-ESS-ContentUpdate*.tar.gz %s" % (output_file_path_from_slim_latest)]
|
||||
|
||||
else:
|
||||
os.mkdir("slim_packaging")
|
||||
|
||||
try:
|
||||
SPLUNK_PACKAGING_TOOLKIT_URL = "https://download.splunk.com/misc/packaging-toolkit/splunk-packaging-toolkit-0.9.0.tar.gz"
|
||||
SPLUNK_PACKAGING_TOOLKIT_FILENAME = 'splunk-packaging-toolkit-latest.tar.gz'
|
||||
print("Downloading the Splunk Packaging Toolkit from %s..." %
|
||||
(SPLUNK_PACKAGING_TOOLKIT_URL), end='')
|
||||
response = get(SPLUNK_PACKAGING_TOOLKIT_URL)
|
||||
response.raise_for_status()
|
||||
with open(SPLUNK_PACKAGING_TOOLKIT_FILENAME, 'wb') as slim_file:
|
||||
slim_file.write(response.content)
|
||||
print("Done")
|
||||
except Exception as e:
|
||||
print("Error downloading the Splunk Packaging Toolkit: [%s].\n\tQuitting..." %
|
||||
(str(e)), file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
commands = ["rm -rf slim_packaging/slim_latest",
|
||||
"mkdir slim_packaging/slim_latest",
|
||||
"mkdir slim_packaging",
|
||||
"cd slim_packaging",
|
||||
"tar -zxf ../splunk-packaging-toolkit-latest.tar.gz -C slim_latest --strip-components=1",
|
||||
"cd slim_latest",
|
||||
"python3 -m venv .venv",
|
||||
". ./.venv/bin/activate",
|
||||
"python -m pip install --upgrade pip",
|
||||
"python -m pip install wheel",
|
||||
"python -m pip install semantic_version",
|
||||
"python -m pip install .",
|
||||
"cp -R ../../dist/escu DA-ESS-ContentUpdate",
|
||||
"cp -R ../dist/escu DA-ESS-ContentUpdate",
|
||||
"slim package -o upload DA-ESS-ContentUpdate",
|
||||
"cp upload/DA-ESS-ContentUpdate*.tar.gz %s" % (output_file_path_from_slim_latest)]
|
||||
|
||||
|
||||
@@ -80,15 +80,18 @@ def get_number_of_indexed_events(splunk_host, splunk_port, splunk_password, inde
|
||||
search = f'''search index="{index}" host="{event_host}" | stats count'''
|
||||
kwargs = {"exec_mode":"blocking"}
|
||||
try:
|
||||
search_result = service.jobs.create(search, **kwargs)
|
||||
job = service.jobs.create(search, **kwargs)
|
||||
|
||||
#This returns the count in string form, not as an int. For example:
|
||||
#OrderedDict([('count', '59630')])
|
||||
search_results = list(results.ResultsReader(search_result.results()))
|
||||
if len(search_results) != 1:
|
||||
results_stream = job.results(output_mode='json')
|
||||
count = None
|
||||
for res in results.JSONResultsReader(results_stream):
|
||||
if 'count' in res:
|
||||
count = int(res['count'],10)
|
||||
if count is None:
|
||||
raise Exception(f"Expected the get_number_of_indexed_events search to only return 1 count, but got {len(search_results)} instead.")
|
||||
|
||||
count = int(search_results[0]['count'])
|
||||
return count
|
||||
|
||||
except Exception as e:
|
||||
@@ -122,59 +125,6 @@ def wait_for_indexing_to_complete(splunk_host, splunk_port, splunk_password, sou
|
||||
time.sleep(check_interval_seconds)
|
||||
|
||||
|
||||
'''
|
||||
def wait_for_indexing_to_complete(splunk_host, splunk_port, splunk_password, sourcetype:str, index:str, check_interval_seconds:int=10):
|
||||
|
||||
startTime = timeit.default_timer()
|
||||
previous_count = -1
|
||||
time.sleep(check_interval_seconds/2)
|
||||
while True:
|
||||
#print("waiting for search...")
|
||||
try:
|
||||
service = client.connect(
|
||||
host=splunk_host,
|
||||
port=splunk_port,
|
||||
username='admin',
|
||||
password=splunk_password
|
||||
)
|
||||
except Exception as e:
|
||||
raise(Exception("Unable to connect to Splunk instance: " + str(e)))
|
||||
|
||||
search = 'search index="%s" sourcetype="%s" | stats count'%(index,sourcetype)
|
||||
kwargs = {"exec_mode":"blocking"}
|
||||
try:
|
||||
search_result = service.jobs.create(search, **kwargs)
|
||||
except Exception as e:
|
||||
print("Error while waiting for indexing of data to complete: %s"%(str(e)))
|
||||
#return False
|
||||
|
||||
#This returns the count in string form, not as an int. For example:
|
||||
#OrderedDict([('count', '59630')])
|
||||
try:
|
||||
for result in results.ResultsReader(search_result.results()):
|
||||
count = int(result['count'])
|
||||
print("count is %d, previous count is %d"%(count,previous_count))
|
||||
if previous_count == -1:
|
||||
if count == 0:
|
||||
pass
|
||||
else:
|
||||
previous_count = count
|
||||
else:
|
||||
if count == previous_count:
|
||||
#After waiting for the check interval, we return the same number of results. The indexing must be complete
|
||||
stopTime = timeit.default_timer()
|
||||
#print("Indexing completed after: %s "%(datetime.timedelta(seconds=stopTime-startTime)))
|
||||
return True
|
||||
else:
|
||||
previous_count = count
|
||||
|
||||
except Exception as e:
|
||||
print("Error trying to get the count while waiting for indexing to complete: %s"%(str(e)))
|
||||
#return False
|
||||
time.sleep(check_interval_seconds)
|
||||
'''
|
||||
|
||||
|
||||
def test_baseline_search(splunk_host, splunk_port, splunk_password, search, pass_condition, baseline_name, baseline_file, earliest_time, latest_time)->dict:
|
||||
try:
|
||||
service = client.connect(
|
||||
@@ -280,6 +230,8 @@ def test_detection_search(splunk_host:str, splunk_port:int, splunk_password:str,
|
||||
|
||||
try:
|
||||
job = service.jobs.create(splunk_search, **kwargs)
|
||||
results_stream = job.results(output_mode='json')
|
||||
|
||||
except Exception as e:
|
||||
|
||||
error_message = "Unable to execute detection: %s"%(str(e))
|
||||
@@ -301,7 +253,7 @@ def test_detection_search(splunk_host:str, splunk_port:int, splunk_password:str,
|
||||
if int(job['resultCount']) != 1:
|
||||
#print("Test failed for detection: " + detection_name)
|
||||
if attempts_remaining > 0:
|
||||
print(f"Execution of test failed for [{detection_name}]. Sleeping for [{failure_sleep_interval_seconds} seconds] and trying again...")
|
||||
print(f"Execution of test failed for [{detection_name}]. Sleeping for [{failure_sleep_interval_seconds} seconds] and trying up to {attempts_remaining} more times...")
|
||||
time.sleep(failure_sleep_interval_seconds)
|
||||
return test_detection_search(splunk_host, splunk_port, splunk_password, search, pass_condition, detection_name, detection_file,
|
||||
earliest_time, latest_time, attempts_remaining=attempts_remaining,
|
||||
@@ -347,7 +299,8 @@ def delete_attack_data(splunk_host:str, splunk_password:str, splunk_port:int, wa
|
||||
try:
|
||||
|
||||
job = service.jobs.create(splunk_search, **kwargs)
|
||||
reader = results.ResultsReader(job)
|
||||
results_stream = job.results(output_mode='json')
|
||||
reader = results.JSONResultsReader(results_stream)
|
||||
|
||||
|
||||
except Exception as e:
|
||||
|
||||
@@ -1,17 +0,0 @@
|
||||
psutil==5.8.0
|
||||
attackcti==0.3.4.3
|
||||
GitPython==3.1.14
|
||||
Jinja2==3.0.0
|
||||
PyYAML==5.4
|
||||
requests==2.25.1
|
||||
six==1.16.0
|
||||
|
||||
splunk-sdk==1.6.12
|
||||
splunk-packaging-toolkit==1.0.1
|
||||
#newest version of docker for managing the splunk containers
|
||||
#we will freeze at a specific version later
|
||||
docker==5.0.3
|
||||
|
||||
#For help getting and parsing the configuration
|
||||
jsonschema==4.2.1
|
||||
wrapt_timeout_decorator==1.3.1
|
||||
+9
-1
@@ -1,12 +1,20 @@
|
||||
attackcti
|
||||
docker
|
||||
gitpython
|
||||
jinja2
|
||||
jsonschema
|
||||
mock
|
||||
psutil
|
||||
pycvesearch
|
||||
pydantic
|
||||
pytest
|
||||
PyYAML
|
||||
pyyaml
|
||||
questionary
|
||||
requests
|
||||
six
|
||||
splunk-packaging-toolkit
|
||||
splunk-sdk
|
||||
wrapt_timeout_decorator
|
||||
xmltodict
|
||||
splunk-sdk
|
||||
https://download.splunk.com/misc/packaging-toolkit/splunk-packaging-toolkit-1.0.1.tar.gz
|
||||
|
||||
Reference in New Issue
Block a user