From 2347743eedab41a78d548b4116acc3fc7898e146 Mon Sep 17 00:00:00 2001 From: mjobanputra Date: Thu, 21 Oct 2021 18:50:54 +0530 Subject: [PATCH 1/5] Add one stage in gitlab CI file which will trigger on tag push, which will publish build to pre_qa artifactory --- .gitlab-ci.yml | 22 ++ .../publish_build_to_pre_qa.py | 315 ++++++++++++++++++ 2 files changed, 337 insertions(+) create mode 100644 security_content_automation/publish_build_to_pre_qa/publish_build_to_pre_qa.py diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6d426ee40d..1a358a03ed 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -11,6 +11,7 @@ stages: - publish_smoketest_runner - publish_deployer - smoketest_staging + - publish_build_to_pre_qa publish_deployer: @@ -59,3 +60,24 @@ smoketest_staging: only: - /^ssa.*$/ - develop + +publish_build_to_pre_qa: + stage: publish_build_to_pre_qa + artifacts: + when: always + paths: + - artifacts/* + image: python:3.8-alpine + before_script: + - apk add --update --no-cache make curl bash git + - curl -L https://github.com/screwdriver-cd/gitversion/releases/download/v1.1.1/gitversion_linux_amd64 -o /usr/local/bin/gitversion && chmod +x /usr/local/bin/gitversion + - eval $(ssh-agent -s) + script: + - mkdir -p artifacts + - pip install requests + - python security_content_automation/publish_build_to_pre_qa/publish_build_to_pre_qa.py --version $CI_COMMIT_REF_NAME --builds DA-ESS_AmazonWebServices_Content DA-ESS-ContentUpdate + after_script: + - cp publish_build_to_pre_qa.log artifacts/publish_build_to_pre_qa.log + only: + refs: + - tags diff --git a/security_content_automation/publish_build_to_pre_qa/publish_build_to_pre_qa.py b/security_content_automation/publish_build_to_pre_qa/publish_build_to_pre_qa.py new file mode 100644 index 0000000000..e7d7f197cb --- /dev/null +++ b/security_content_automation/publish_build_to_pre_qa/publish_build_to_pre_qa.py @@ -0,0 +1,315 @@ +import os +import json +import subprocess +from base64 import b64encode, b64decode +import logging +import time +import re +import argparse + +import requests + +logging.basicConfig( + level=logging.DEBUG, + format="%(asctime)s [%(levelname)s] %(message)s", + handlers=[ + logging.FileHandler("publish_build_to_pre_qa.log") + ], +) + + +class RetryConstant: + RETRY_COUNT = 3 + RETRY_INTERVAL = 15 + + +class PublishArtifactory: + def __init__(self, release_id, artifactory_list): + # Tag version + self.release_id = release_id + # Jfrog artifactory endpoint + self.endpoint = os.environ.get("JFROG_ENDPOINT") + # Jfrog artifactory repository + self.repository = os.environ.get("JFROG_REPOSITORY") + # Pre-QA dir name + self.pre_qa_repository_dir_name = os.environ.get("JFROG_DIR") + # Pre-QA artifactory repository endpoint + self.pre_qa_endpoint = f"{self.endpoint}/artifactory/{self.repository}" + # Pre-QA artifactory API endpoint + self.pre_qa_api_endpoint = ( + f"{self.endpoint}/artifactory/api/storage/{self.repository}" + ) + # Artifactory name list, will push to pre-qa + self.artifactory_files = artifactory_list + # Github tag endpoint endpoint for download assets + self.git_tags_api_endpoint = ( + f"https://api.github.com/repos/splunk/security_content/releases/tags/" + f"{self.release_id}" + ) + # Max artifact in builds dir + self.max_artifacts_in_build_dir = 5 + + def __fetch_and_download_artifactory_from_git(self): + """ + This method will download build from github + :return: Downloaded build list + """ + token = b64encode( + str.encode( + f"{os.environ.get('GIT_USERNAME')}:{b64decode(os.environ.get('GIT_ACCESS_TOKEN')).decode()}" + ) + ).decode("ascii") + headers = { + "accept": "application/vnd.github.v3+json", + "Authorization": f"Basic %s" % token, + } + try: + # Git API fetch tag asset details + response = requests.get(f"{self.git_tags_api_endpoint}", headers=headers) + + if response.status_code == 200: + response_content = json.loads(response.content) + logging.debug( + f"Response status code - {response.status_code}, Response content - {response_content}, " + f"Request endpoint- {self.git_tags_api_endpoint}" + ) + else: + raise Exception(response.content) + except Exception as error: + error_message = f"Error while fetching Git file content: {self.git_tags_api_endpoint}, Reason: {error}" + logging.error(error_message) + raise type(error)(error_message) from error + + artifactory_list = {} + for asset in response_content["assets"]: + if asset["name"].split("-v")[0] in self.artifactory_files: + artifactory_list[asset["name"]] = asset["url"] + + logging.debug(f"Artifactory list - {artifactory_list}") + + # Download artifactory from github using CURL + for key, value in artifactory_list.items(): + download_url = f"curl -vLJO -H 'Authorization: Basic {token}' -H 'Accept: application/octet-stream' {value}" + status, output = subprocess.getstatusoutput(download_url) + logging.debug( + f"Download URL - {download_url}, status - {status}, output-{output}" + ) + if status == 0: + logging.debug(f"{key} build successfully downloaded from github") + else: + error_message = ( + f"Error occur while downloading build from github, Reason: {output}" + ) + logging.error(error_message) + raise Exception(error_message) + + return artifactory_list.keys() + + def __delete_exiting_artifactory_from_pre_qa(self, artifactory): + """ + This method will delete the existing build from JFROG artifactory latest and builds dir + :param artifactory: Artifactory name + :return: None + """ + # Delete the existing build from pre-qa latest dir + pre_qa_latest_endpoint = f"{self.pre_qa_endpoint}/{self.pre_qa_repository_dir_name}/{artifactory}/latest" + delete_pre_qa_latest_response = requests.request( + "DELETE", + pre_qa_latest_endpoint, + auth=( + os.environ.get("JFROG_ARTIFACTORY_USERNAME"), + b64decode(os.environ.get("JFROG_ARTIFACTORY_PASSWORD")).decode(), + ), + ) + if delete_pre_qa_latest_response.status_code == 204: + logging.info( + f"{pre_qa_latest_endpoint} Successfully delete existing build from jfrog artifactory" + ) + elif delete_pre_qa_latest_response.status_code == 404: + logging.info(f"{pre_qa_latest_endpoint} Nothing to delete") + else: + error_message = ( + f"Error occur while deleting build from jfrog artifactory latest dir, endpoint: {pre_qa_latest_endpoint}" + f", Reason: {delete_pre_qa_latest_response.content}" + ) + logging.error(error_message) + raise Exception(error_message) + + # Get all artifact from builds dir + pre_qa_builds_endpoint = f"{self.pre_qa_api_endpoint}/{self.pre_qa_repository_dir_name}/{artifactory}/builds/" + response = requests.request( + "GET", + pre_qa_builds_endpoint, + auth=( + os.environ.get("JFROG_ARTIFACTORY_USERNAME"), + b64decode(os.environ.get("JFROG_ARTIFACTORY_PASSWORD")).decode(), + ), + ) + + logging.debug( + f"Response status code - {response.status_code}, Response content - {response.content}, " + f"Request endpoint- {self.pre_qa_endpoint}" + ) + + if response.status_code == 200: + delete_artifact_count = len( + json.loads(response.content).get("children") + ) - (self.max_artifacts_in_build_dir - 1) + if delete_artifact_count > 0: + artifactory_list = json.loads(response.content).get("children") + temp_artifactory_dict = {} + + for obj in artifactory_list: + temp_artifactory_dict[ + re.search("\d+(\.\d+){2,}", obj["uri"]).group() + ] = obj["uri"] + + # Sort the artifactory list + sorted_temp_artifactory_dict = dict( + sorted(temp_artifactory_dict.items()) + ) + + delete_artifactory_dict = { + obj: sorted_temp_artifactory_dict[obj] + for obj in list(sorted_temp_artifactory_dict)[ + :delete_artifact_count + ] + } + + delete_artifactory_list = list(delete_artifactory_dict.values()) + + # Delete the artifactory list from builds dir + for obj in delete_artifactory_list: + url = f"{self.pre_qa_endpoint}/{self.pre_qa_repository_dir_name}/{artifactory}/builds{obj}" + delete_response = requests.request( + "DELETE", + url, + auth=( + os.environ.get("JFROG_ARTIFACTORY_USERNAME"), + b64decode( + os.environ.get("JFROG_ARTIFACTORY_PASSWORD") + ).decode(), + ), + ) + if delete_response.status_code == 204: + logging.info( + f"{url} Successfully delete existing build from jfrog artifactory" + ) + return + else: + error_message = ( + f"Error occur while deleting build from jfrog artifactory builds dir, " + f"Reason: {delete_response.content}, endpoint: {url}" + ) + logging.error(error_message) + raise Exception(error_message) + + def __publish_single_artifactory_to_pre_qa(self, artifactory, endpoint): + """ + This method will deploy locally downloaded build to jfrog artifactory server. + :param artifactory: Local downloaded artifactory path + :param endpoint: Jfrog artifactory path where we deploy the artifactory + :return: None + """ + retries = 0 + while retries < RetryConstant.RETRY_COUNT: + deploy_url = f"curl -u '{os.environ.get('JFROG_ARTIFACTORY_USERNAME')}:" \ + f"{b64decode(os.environ.get('JFROG_ARTIFACTORY_PASSWORD')).decode()}' -H 'Connection: " \ + f"keep-alive' --compressed -v --keepalive-time 2000 -X PUT {endpoint} -T {artifactory}" + + status, output = subprocess.getstatusoutput(deploy_url) + logging.debug( + f"Deploy URL - {deploy_url}, status - {status}, output-{output}" + ) + if status == 0: + logging.debug(f"{artifactory} build deploy successfully to {endpoint}") + break + else: + error_message = ( + f"Error occur while downloading build from github, Reason: {output}" + ) + logging.error(error_message) + time.sleep(RetryConstant.RETRY_INTERVAL) + retries = retries + 1 + + if retries == RetryConstant.RETRY_COUNT: + error_message = ( + "Max retries occur while deploying build to jfrog artifactory" + ) + raise Exception(error_message) + + def __publish_artifactory_to_pre_qa(self, artifactory_list): + """ + This method will deploy the artifactory jfrog artifactory server and delete the existing build from latest + and builds dir + :param artifactory_list: The list of locally downloaded artifactory + :return: None + """ + for artifactory in artifactory_list: + try: + # Current artifactory dir name + current_artifactory_dir_name = ( + artifactory.replace("_", "-") + .lower() + .split(re.search("-v\d+(\.\d+){2,}", artifactory).group())[0] + ) + + # Delete existing builds from Pre-qa artifactory latest and builds + self.__delete_exiting_artifactory_from_pre_qa( + current_artifactory_dir_name + ) + + # Push build to Pre-qa builds + builds_endpoint = f"{self.pre_qa_endpoint}/{self.pre_qa_repository_dir_name}/" \ + f"{current_artifactory_dir_name}/builds/{artifactory}" + self.__publish_single_artifactory_to_pre_qa( + artifactory, builds_endpoint + ) + + # Push build to Pre-qa latest + latest_endpoint = f"{self.pre_qa_endpoint}/{self.pre_qa_repository_dir_name}/" \ + f"{current_artifactory_dir_name}/latest/{artifactory}" + self.__publish_single_artifactory_to_pre_qa( + artifactory, latest_endpoint + ) + except Exception as error: + error_message = f"Error occur while publishing build to PRE-QA artifactory, reason: {error}" + raise Exception(error_message) + + @staticmethod + def __remove_downloaded_file(artifactory_list): + """ + This method will delete locally downloaded artifactory + :param artifactory_list: List of artifactory + :return: None + """ + for file in artifactory_list: + os.remove(file) + logging.info(f"{file} successfully remove file from local") + + def main(self): + """ + This is wrapper method of above listed method + :return: None + """ + artifactory_list = self.__fetch_and_download_artifactory_from_git() + self.__publish_artifactory_to_pre_qa(artifactory_list) + self.__remove_downloaded_file(artifactory_list) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Tag version") + parser.add_argument( + "--version", dest="version", type=str, help="Tag version number", required=True + ) + parser.add_argument( + "--builds", dest="builds", nargs="+", type=str, help="List of builds", required=True + ) + args = parser.parse_args() + # Validate tag version + if args.version and bool(re.search("v\d+(\.\d+){2,}", args.version)) and args.builds: + PublishArtifactory( + args.version, args.builds + ).main() + else: + raise Exception("Tagged version is not correct") From 1212dc33e2e4cc28e0f9d26692ff0526338a0c02 Mon Sep 17 00:00:00 2001 From: mjobanputra Date: Fri, 29 Oct 2021 16:39:26 +0530 Subject: [PATCH 2/5] Resolve review comment changes --- .../publish_build_to_pre_qa/constant.py | 9 ++ .../publish_build_to_pre_qa.py | 98 +++++++++++-------- 2 files changed, 66 insertions(+), 41 deletions(-) create mode 100644 security_content_automation/publish_build_to_pre_qa/constant.py diff --git a/security_content_automation/publish_build_to_pre_qa/constant.py b/security_content_automation/publish_build_to_pre_qa/constant.py new file mode 100644 index 0000000000..5d3fc35667 --- /dev/null +++ b/security_content_automation/publish_build_to_pre_qa/constant.py @@ -0,0 +1,9 @@ +class RetryConstant: + RETRY_COUNT = 3 + RETRY_INTERVAL = 15 + + +class JfrogArtifactoryConstant: + JFROG_ENDPOINT = "https://repo.splunk.com" + JFROG_REPOSITORY = "Solutions" + PRE_QA_DIR = "DA/Pre-QA" diff --git a/security_content_automation/publish_build_to_pre_qa/publish_build_to_pre_qa.py b/security_content_automation/publish_build_to_pre_qa/publish_build_to_pre_qa.py index e7d7f197cb..a1f810ceca 100644 --- a/security_content_automation/publish_build_to_pre_qa/publish_build_to_pre_qa.py +++ b/security_content_automation/publish_build_to_pre_qa/publish_build_to_pre_qa.py @@ -9,30 +9,25 @@ import argparse import requests +from constant import RetryConstant, JfrogArtifactoryConstant + logging.basicConfig( level=logging.DEBUG, format="%(asctime)s [%(levelname)s] %(message)s", - handlers=[ - logging.FileHandler("publish_build_to_pre_qa.log") - ], + handlers=[logging.FileHandler("publish_build_to_pre_qa.log")], ) -class RetryConstant: - RETRY_COUNT = 3 - RETRY_INTERVAL = 15 - - class PublishArtifactory: def __init__(self, release_id, artifactory_list): # Tag version self.release_id = release_id # Jfrog artifactory endpoint - self.endpoint = os.environ.get("JFROG_ENDPOINT") + self.endpoint = JfrogArtifactoryConstant.JFROG_ENDPOINT # Jfrog artifactory repository - self.repository = os.environ.get("JFROG_REPOSITORY") + self.repository = JfrogArtifactoryConstant.JFROG_REPOSITORY # Pre-QA dir name - self.pre_qa_repository_dir_name = os.environ.get("JFROG_DIR") + self.pre_qa_repository_dir_name = JfrogArtifactoryConstant.PRE_QA_DIR # Pre-QA artifactory repository endpoint self.pre_qa_endpoint = f"{self.endpoint}/artifactory/{self.repository}" # Pre-QA artifactory API endpoint @@ -51,7 +46,7 @@ class PublishArtifactory: def __fetch_and_download_artifactory_from_git(self): """ - This method will download build from github + Download build from github tags assets :return: Downloaded build list """ token = b64encode( @@ -88,14 +83,16 @@ class PublishArtifactory: logging.debug(f"Artifactory list - {artifactory_list}") # Download artifactory from github using CURL - for key, value in artifactory_list.items(): - download_url = f"curl -vLJO -H 'Authorization: Basic {token}' -H 'Accept: application/octet-stream' {value}" + for assets_name, assets_url in artifactory_list.items(): + download_url = f"curl -vLJO -H 'Authorization: Basic {token}' -H 'Accept: application/octet-stream' {assets_url}" status, output = subprocess.getstatusoutput(download_url) logging.debug( f"Download URL - {download_url}, status - {status}, output-{output}" ) if status == 0: - logging.debug(f"{key} build successfully downloaded from github") + logging.debug( + f"{assets_name} build successfully downloaded from github" + ) else: error_message = ( f"Error occur while downloading build from github, Reason: {output}" @@ -107,7 +104,7 @@ class PublishArtifactory: def __delete_exiting_artifactory_from_pre_qa(self, artifactory): """ - This method will delete the existing build from JFROG artifactory latest and builds dir + Delete the existing build from JFROG artifactory latest dir and builds dir of given product :param artifactory: Artifactory name :return: None """ @@ -152,11 +149,11 @@ class PublishArtifactory: ) if response.status_code == 200: - delete_artifact_count = len( - json.loads(response.content).get("children") - ) - (self.max_artifacts_in_build_dir - 1) + artifactory_list = json.loads(response.content).get("children") + delete_artifact_count = len(artifactory_list) - ( + self.max_artifacts_in_build_dir - 1 + ) if delete_artifact_count > 0: - artifactory_list = json.loads(response.content).get("children") temp_artifactory_dict = {} for obj in artifactory_list: @@ -206,16 +203,18 @@ class PublishArtifactory: def __publish_single_artifactory_to_pre_qa(self, artifactory, endpoint): """ - This method will deploy locally downloaded build to jfrog artifactory server. + Deploy build to jfrog artifactory server :param artifactory: Local downloaded artifactory path :param endpoint: Jfrog artifactory path where we deploy the artifactory :return: None """ retries = 0 while retries < RetryConstant.RETRY_COUNT: - deploy_url = f"curl -u '{os.environ.get('JFROG_ARTIFACTORY_USERNAME')}:" \ - f"{b64decode(os.environ.get('JFROG_ARTIFACTORY_PASSWORD')).decode()}' -H 'Connection: " \ - f"keep-alive' --compressed -v --keepalive-time 2000 -X PUT {endpoint} -T {artifactory}" + deploy_url = ( + f"curl -u '{os.environ.get('JFROG_ARTIFACTORY_USERNAME')}:" + f"{b64decode(os.environ.get('JFROG_ARTIFACTORY_PASSWORD')).decode()}' -H 'Connection: " + f"keep-alive' --compressed -v --keepalive-time 2000 -X PUT {endpoint} -T {artifactory}" + ) status, output = subprocess.getstatusoutput(deploy_url) logging.debug( @@ -240,8 +239,7 @@ class PublishArtifactory: def __publish_artifactory_to_pre_qa(self, artifactory_list): """ - This method will deploy the artifactory jfrog artifactory server and delete the existing build from latest - and builds dir + Deploy build to jfrog artifactory server and delete the existing builds :param artifactory_list: The list of locally downloaded artifactory :return: None """ @@ -260,15 +258,19 @@ class PublishArtifactory: ) # Push build to Pre-qa builds - builds_endpoint = f"{self.pre_qa_endpoint}/{self.pre_qa_repository_dir_name}/" \ - f"{current_artifactory_dir_name}/builds/{artifactory}" + builds_endpoint = ( + f"{self.pre_qa_endpoint}/{self.pre_qa_repository_dir_name}/" + f"{current_artifactory_dir_name}/builds/{artifactory}" + ) self.__publish_single_artifactory_to_pre_qa( artifactory, builds_endpoint ) # Push build to Pre-qa latest - latest_endpoint = f"{self.pre_qa_endpoint}/{self.pre_qa_repository_dir_name}/" \ - f"{current_artifactory_dir_name}/latest/{artifactory}" + latest_endpoint = ( + f"{self.pre_qa_endpoint}/{self.pre_qa_repository_dir_name}/" + f"{current_artifactory_dir_name}/latest/{artifactory}" + ) self.__publish_single_artifactory_to_pre_qa( artifactory, latest_endpoint ) @@ -279,17 +281,17 @@ class PublishArtifactory: @staticmethod def __remove_downloaded_file(artifactory_list): """ - This method will delete locally downloaded artifactory + Delete Github downloaded build :param artifactory_list: List of artifactory :return: None """ for file in artifactory_list: os.remove(file) - logging.info(f"{file} successfully remove file from local") + logging.info(f"{file} successfully delete github downloaded build") def main(self): """ - This is wrapper method of above listed method + Wrapper method of above listed method :return: None """ artifactory_list = self.__fetch_and_download_artifactory_from_git() @@ -298,18 +300,32 @@ class PublishArtifactory: if __name__ == "__main__": - parser = argparse.ArgumentParser(description="Tag version") + parser = argparse.ArgumentParser(description="Github version Tag") parser.add_argument( - "--version", dest="version", type=str, help="Tag version number", required=True + "--version", + dest="version", + type=str, + help="Security content repo new releases tag version number", + required=True, ) parser.add_argument( - "--builds", dest="builds", nargs="+", type=str, help="List of builds", required=True + "--builds", + dest="builds", + nargs="+", + type=str, + help="List of builds, need to fetch from Security content " + "github assets and deploy to Pre-QA Dir of artifactory", + required=True, ) args = parser.parse_args() # Validate tag version - if args.version and bool(re.search("v\d+(\.\d+){2,}", args.version)) and args.builds: - PublishArtifactory( - args.version, args.builds - ).main() + if ( + args.version + and bool(re.search("v\d+(\.\d+){2,}", args.version)) + and args.builds + ): + PublishArtifactory(args.version, args.builds).main() else: - raise Exception("Tagged version is not correct") + raise Exception( + f"Github release tagged version is not correct, Tag version: {args.version}" + ) From b4122efa5c992a7ee04ed6f95de5d529ba2e6550 Mon Sep 17 00:00:00 2001 From: mjobanputra Date: Mon, 1 Nov 2021 18:52:33 +0530 Subject: [PATCH 3/5] Change the pipeline trigger conditions --- .gitlab-ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1a358a03ed..13c1671a13 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -78,6 +78,6 @@ publish_build_to_pre_qa: - python security_content_automation/publish_build_to_pre_qa/publish_build_to_pre_qa.py --version $CI_COMMIT_REF_NAME --builds DA-ESS_AmazonWebServices_Content DA-ESS-ContentUpdate after_script: - cp publish_build_to_pre_qa.log artifacts/publish_build_to_pre_qa.log - only: - refs: - - tags + rules: + - if: '$CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+\.[0-9]$/' + when: always From 92612b77372907cf0e9e793e88a690c9b0cc0eb8 Mon Sep 17 00:00:00 2001 From: mjobanputra Date: Fri, 12 Nov 2021 14:36:05 +0530 Subject: [PATCH 4/5] Update docstring --- .../publish_build_to_pre_qa.py | 37 ++++++++++--------- 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/security_content_automation/publish_build_to_pre_qa/publish_build_to_pre_qa.py b/security_content_automation/publish_build_to_pre_qa/publish_build_to_pre_qa.py index a1f810ceca..99901dacd0 100644 --- a/security_content_automation/publish_build_to_pre_qa/publish_build_to_pre_qa.py +++ b/security_content_automation/publish_build_to_pre_qa/publish_build_to_pre_qa.py @@ -26,22 +26,22 @@ class PublishArtifactory: self.endpoint = JfrogArtifactoryConstant.JFROG_ENDPOINT # Jfrog artifactory repository self.repository = JfrogArtifactoryConstant.JFROG_REPOSITORY - # Pre-QA dir name + # Pre-QA directory name self.pre_qa_repository_dir_name = JfrogArtifactoryConstant.PRE_QA_DIR - # Pre-QA artifactory repository endpoint + # Pre-QA artifactory endpoint self.pre_qa_endpoint = f"{self.endpoint}/artifactory/{self.repository}" # Pre-QA artifactory API endpoint self.pre_qa_api_endpoint = ( f"{self.endpoint}/artifactory/api/storage/{self.repository}" ) - # Artifactory name list, will push to pre-qa + # Artifactory name list, will push to Pre-QA self.artifactory_files = artifactory_list - # Github tag endpoint endpoint for download assets + # Github tags API endpoint for download assets self.git_tags_api_endpoint = ( f"https://api.github.com/repos/splunk/security_content/releases/tags/" f"{self.release_id}" ) - # Max artifact in builds dir + # Max artifact in builds directory self.max_artifacts_in_build_dir = 5 def __fetch_and_download_artifactory_from_git(self): @@ -82,7 +82,7 @@ class PublishArtifactory: logging.debug(f"Artifactory list - {artifactory_list}") - # Download artifactory from github using CURL + # Download build from github for assets_name, assets_url in artifactory_list.items(): download_url = f"curl -vLJO -H 'Authorization: Basic {token}' -H 'Accept: application/octet-stream' {assets_url}" status, output = subprocess.getstatusoutput(download_url) @@ -104,11 +104,11 @@ class PublishArtifactory: def __delete_exiting_artifactory_from_pre_qa(self, artifactory): """ - Delete the existing build from JFROG artifactory latest dir and builds dir of given product - :param artifactory: Artifactory name + Delete the existing build from JFROG artifactory latest directory and builds directory of given product + :param artifactory: Product name :return: None """ - # Delete the existing build from pre-qa latest dir + # Delete the existing build from product latest directory pre_qa_latest_endpoint = f"{self.pre_qa_endpoint}/{self.pre_qa_repository_dir_name}/{artifactory}/latest" delete_pre_qa_latest_response = requests.request( "DELETE", @@ -132,7 +132,7 @@ class PublishArtifactory: logging.error(error_message) raise Exception(error_message) - # Get all artifact from builds dir + # Get artifact details from product builds directory pre_qa_builds_endpoint = f"{self.pre_qa_api_endpoint}/{self.pre_qa_repository_dir_name}/{artifactory}/builds/" response = requests.request( "GET", @@ -175,7 +175,7 @@ class PublishArtifactory: delete_artifactory_list = list(delete_artifactory_dict.values()) - # Delete the artifactory list from builds dir + # Delete older build from product builds directory for obj in delete_artifactory_list: url = f"{self.pre_qa_endpoint}/{self.pre_qa_repository_dir_name}/{artifactory}/builds{obj}" delete_response = requests.request( @@ -195,13 +195,14 @@ class PublishArtifactory: return else: error_message = ( - f"Error occur while deleting build from jfrog artifactory builds dir, " + f"Error occur while deleting build from jfrog artifactory builds directory, " f"Reason: {delete_response.content}, endpoint: {url}" ) logging.error(error_message) raise Exception(error_message) - def __publish_single_artifactory_to_pre_qa(self, artifactory, endpoint): + @staticmethod + def __publish_single_artifactory_to_pre_qa(artifactory, endpoint): """ Deploy build to jfrog artifactory server :param artifactory: Local downloaded artifactory path @@ -245,19 +246,19 @@ class PublishArtifactory: """ for artifactory in artifactory_list: try: - # Current artifactory dir name + # product directory name current_artifactory_dir_name = ( artifactory.replace("_", "-") .lower() .split(re.search("-v\d+(\.\d+){2,}", artifactory).group())[0] ) - # Delete existing builds from Pre-qa artifactory latest and builds + # Delete existing builds from product latest and builds directory self.__delete_exiting_artifactory_from_pre_qa( current_artifactory_dir_name ) - # Push build to Pre-qa builds + # Push build to product builds directory builds_endpoint = ( f"{self.pre_qa_endpoint}/{self.pre_qa_repository_dir_name}/" f"{current_artifactory_dir_name}/builds/{artifactory}" @@ -266,7 +267,7 @@ class PublishArtifactory: artifactory, builds_endpoint ) - # Push build to Pre-qa latest + # Push build to product latest directory latest_endpoint = ( f"{self.pre_qa_endpoint}/{self.pre_qa_repository_dir_name}/" f"{current_artifactory_dir_name}/latest/{artifactory}" @@ -314,7 +315,7 @@ if __name__ == "__main__": nargs="+", type=str, help="List of builds, need to fetch from Security content " - "github assets and deploy to Pre-QA Dir of artifactory", + "github assets and deploy to Pre-QA directory of artifactory", required=True, ) args = parser.parse_args() From a13e6356d43c2a627df5693e206f092302c271ca Mon Sep 17 00:00:00 2001 From: mjobanputra Date: Wed, 17 Nov 2021 14:56:48 +0530 Subject: [PATCH 5/5] Update log message --- .../publish_build_to_pre_qa/publish_build_to_pre_qa.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/security_content_automation/publish_build_to_pre_qa/publish_build_to_pre_qa.py b/security_content_automation/publish_build_to_pre_qa/publish_build_to_pre_qa.py index 99901dacd0..bfe782e601 100644 --- a/security_content_automation/publish_build_to_pre_qa/publish_build_to_pre_qa.py +++ b/security_content_automation/publish_build_to_pre_qa/publish_build_to_pre_qa.py @@ -226,7 +226,7 @@ class PublishArtifactory: break else: error_message = ( - f"Error occur while downloading build from github, Reason: {output}" + f"Error occur while deploy build to pre-qa, Reason: {output}" ) logging.error(error_message) time.sleep(RetryConstant.RETRY_INTERVAL)